diff --git a/.eslintignore b/.eslintignore index 6803a9b63d3..fd409251590 100644 --- a/.eslintignore +++ b/.eslintignore @@ -7,3 +7,6 @@ versions acmeair-nodejs vendor integration-tests/esbuild/out.js +integration-tests/esbuild/aws-sdk-out.js +packages/dd-trace/src/appsec/blocked_templates.js +packages/dd-trace/src/payload-tagging/jsonpath-plus.js diff --git a/.eslintrc.json b/.eslintrc.json index fe6ef889976..13031ec7db1 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -1,6 +1,6 @@ { "parserOptions": { - "ecmaVersion": 2020 + "ecmaVersion": 2021 }, "extends": [ "eslint:recommended", @@ -13,7 +13,12 @@ ], "env": { "node": true, - "es2020": true + "es2021": true + }, + "settings": { + "node": { + "version": ">=16.0.0" + } }, "rules": { "max-len": [2, 120, 2], @@ -34,6 +39,8 @@ "mocha/no-exports": 0, "mocha/no-skipped-tests": 0, "n/no-restricted-require": [2, ["diagnostics_channel"]], - "object-curly-newline": ["error", {"multiline": true, "consistent": true }] + "n/no-callback-literal": 0, + "object-curly-newline": ["error", {"multiline": true, "consistent": true }], + "import/no-absolute-path": 0 } } diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index b4638274d1e..00000000000 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,34 +0,0 @@ ---- -name: Bug report -about: Create a report to help us improve -title: '' -labels: bug -assignees: '' - ---- - - -**Expected behaviour** - - -**Actual behaviour** - - -**Steps to reproduce** - - -**Environment** - -* **Operation system:** -* **Node.js version:** -* **Tracer version:** -* **Agent version:** -* **Relevant library versions:** - - diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000000..b5a5eb1d199 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,9 @@ +blank_issues_enabled: true +contact_links: + - name: Bug Report + url: https://help.datadoghq.com/hc/en-us/requests/new?tf_1260824651490=pt_product_type:apm&tf_1900004146284=pt_apm_language:node + about: This option creates an expedited Bug Report via the helpdesk (no login required). This will allow us to look up your account and allows you to provide additional information in private. Please do not create a GitHub issue to report a bug. + - name: Feature Request + url: https://help.datadoghq.com/hc/en-us/requests/new?tf_1260824651490=pt_product_type:apm&tf_1900004146284=pt_apm_language:node&tf_1260825272270=pt_apm_category_feature_request + about: This option creates an expedited Feature Request via the helpdesk (no login required). This helps with prioritization and allows you to provide additional information in private. Please do not create a GitHub issue to request a feature. + diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md deleted file mode 100644 index 47b34bf567b..00000000000 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -name: Feature request -about: Suggest an idea for this project -title: '' -labels: feature-request -assignees: '' - ---- diff --git a/.github/actions/injection/action.yml b/.github/actions/injection/action.yml deleted file mode 100644 index 36949930f70..00000000000 --- a/.github/actions/injection/action.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: Build/Publish Injection Image -inputs: - init-image-version: - description: Image version to use for publishing - required: true -runs: - using: composite - steps: - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # 2.0.0 - with: - version: v0.9.1 # https://github.com/docker/buildx/issues/1533 - - name: Build injection image and push to github packages - shell: bash - run: | - cp dd-trace-*.tgz lib-injection/dd-trace.tgz - docker buildx create --name lib-injection - docker buildx use lib-injection - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # https://stackoverflow.com/questions/72167570/docker-buildx-nodejs-fail - docker buildx build --platform=linux/amd64,linux/arm/v7,linux/arm64/v8 -t ghcr.io/datadog/dd-trace-js/dd-lib-js-init:${{ inputs.init-image-version }} --push lib-injection - diff --git a/.github/actions/install/action.yml b/.github/actions/install/action.yml new file mode 100644 index 00000000000..0401dd02e81 --- /dev/null +++ b/.github/actions/install/action.yml @@ -0,0 +1,7 @@ +name: Install dependencies +runs: + using: composite + steps: # retry in case of server error from registry + - run: yarn install --ignore-engines || yarn install --ignore-engines + shell: bash + diff --git a/.github/actions/node/latest/action.yml b/.github/actions/node/latest/action.yml index 74e5d531f94..9e4c62ceca5 100644 --- a/.github/actions/node/latest/action.yml +++ b/.github/actions/node/latest/action.yml @@ -4,4 +4,4 @@ runs: steps: - uses: actions/setup-node@v3 with: - node-version: 'latest' + node-version: '22' # Update this line to the latest Node.js version diff --git a/.github/actions/node/oldest/action.yml b/.github/actions/node/oldest/action.yml index 0dbaafccab8..a679a468d29 100644 --- a/.github/actions/node/oldest/action.yml +++ b/.github/actions/node/oldest/action.yml @@ -1,7 +1,7 @@ -name: Node 16 +name: Node 18 runs: using: composite steps: - uses: actions/setup-node@v3 with: - node-version: '16' + node-version: '18' diff --git a/.github/actions/node/setup/action.yml b/.github/actions/node/setup/action.yml index 46e5c70e944..c00c299f594 100644 --- a/.github/actions/node/setup/action.yml +++ b/.github/actions/node/setup/action.yml @@ -5,4 +5,4 @@ runs: - uses: actions/setup-node@v3 with: cache: yarn - node-version: '16' + node-version: '18' diff --git a/.github/actions/plugins/test-and-upstream/action.yml b/.github/actions/plugins/test-and-upstream/action.yml new file mode 100644 index 00000000000..d847de98c0e --- /dev/null +++ b/.github/actions/plugins/test-and-upstream/action.yml @@ -0,0 +1,20 @@ +name: Plugin Tests +runs: + using: composite + steps: + - uses: ./.github/actions/testagent/start + - uses: ./.github/actions/node/setup + - uses: ./.github/actions/install + - uses: ./.github/actions/node/oldest + - run: yarn test:plugins:ci + shell: bash + - run: yarn test:plugins:upstream + shell: bash + - uses: ./.github/actions/node/latest + - run: yarn test:plugins:ci + shell: bash + - run: yarn test:plugins:upstream + shell: bash + - uses: codecov/codecov-action@v3 + - if: always() + uses: ./.github/actions/testagent/logs diff --git a/.github/actions/plugins/test/action.yml b/.github/actions/plugins/test/action.yml new file mode 100644 index 00000000000..f39da26b682 --- /dev/null +++ b/.github/actions/plugins/test/action.yml @@ -0,0 +1,16 @@ +name: Plugin Tests +runs: + using: composite + steps: + - uses: ./.github/actions/testagent/start + - uses: ./.github/actions/node/setup + - uses: ./.github/actions/install + - uses: ./.github/actions/node/oldest + - run: yarn test:plugins:ci + shell: bash + - uses: ./.github/actions/node/latest + - run: yarn test:plugins:ci + shell: bash + - uses: codecov/codecov-action@v3 + - if: always() + uses: ./.github/actions/testagent/logs diff --git a/.github/actions/plugins/upstream/action.yml b/.github/actions/plugins/upstream/action.yml new file mode 100644 index 00000000000..e1d74b574ee --- /dev/null +++ b/.github/actions/plugins/upstream/action.yml @@ -0,0 +1,16 @@ +name: Plugin Upstream Tests +runs: + using: composite + steps: + - uses: ./.github/actions/testagent/start + - uses: ./.github/actions/node/setup + - uses: ./.github/actions/install + - uses: ./.github/actions/node/oldest + - run: yarn test:plugins:upstream + shell: bash + - uses: ./.github/actions/node/latest + - run: yarn test:plugins:upstream + shell: bash + - uses: codecov/codecov-action@v3 + - if: always() + uses: ./.github/actions/testagent/logs diff --git a/.github/actions/testagent/logs/action.yml b/.github/actions/testagent/logs/action.yml index bb80d251848..a168e9008ae 100644 --- a/.github/actions/testagent/logs/action.yml +++ b/.github/actions/testagent/logs/action.yml @@ -12,7 +12,7 @@ runs: if [ -n "${{inputs.container-id}}" ]; then docker logs ${{inputs.container-id}} else - docker-compose logs testagent + docker compose logs testagent fi shell: bash - name: Get Tested Integrations from Test Agent diff --git a/.github/actions/testagent/start/action.yml b/.github/actions/testagent/start/action.yml index e5865983986..d17071f6680 100644 --- a/.github/actions/testagent/start/action.yml +++ b/.github/actions/testagent/start/action.yml @@ -3,6 +3,6 @@ description: "Starts the APM Test Agent image with environment." runs: using: composite steps: - - uses: actions/checkout@v2 - - run: docker-compose up -d testagent + - uses: actions/checkout@v4 + - run: docker compose up -d testagent shell: bash diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 4df80bee84f..86a60111c4f 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -23,10 +23,4 @@ ### Additional Notes -### Security -Datadog employees: -- [ ] If this PR touches code that signs or publishes builds or packages, or handles credentials of any kind, I've requested a review from `@DataDog/security-design-and-guidance`. -- [ ] This PR doesn't touch any of that. - -Unsure? Have a question? Request a review! diff --git a/.github/workflows/all-green.yml b/.github/workflows/all-green.yml new file mode 100644 index 00000000000..1086b83ee7f --- /dev/null +++ b/.github/workflows/all-green.yml @@ -0,0 +1,19 @@ +name: All Green +on: + pull_request: + push: + branches: + - master + +jobs: + + all-green: + runs-on: ubuntu-latest + permissions: + checks: read + contents: read + steps: + - uses: wechuli/allcheckspassed@v1 + with: + retries: 20 # once per minute, some checks take up to 15 min + checks_exclude: devflow.* diff --git a/.github/workflows/appsec.yml b/.github/workflows/appsec.yml index f37acbe97bc..f41b18f9d53 100644 --- a/.github/workflows/appsec.yml +++ b/.github/workflows/appsec.yml @@ -15,34 +15,36 @@ jobs: macos: runs-on: macos-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - - run: yarn install + - uses: ./.github/actions/install - run: yarn test:appsec:ci - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 ubuntu: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/16 + - uses: ./.github/actions/install + - uses: ./.github/actions/node/oldest - run: yarn test:appsec:ci - - uses: ./.github/actions/node/18 + - uses: ./.github/actions/node/20 - run: yarn test:appsec:ci - uses: ./.github/actions/node/latest - run: yarn test:appsec:ci - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 windows: runs-on: windows-latest steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/node/setup - - run: yarn install + - uses: actions/checkout@v4 + - uses: actions/setup-node@v3 + with: + node-version: '18' + - uses: ./.github/actions/install - run: yarn test:appsec:ci - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 ldapjs: runs-on: ubuntu-latest @@ -60,14 +62,14 @@ jobs: LDAP_USERS: 'user01,user02' LDAP_PASSWORDS: 'password1,password2' steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - - run: yarn install + - uses: ./.github/actions/install - uses: ./.github/actions/node/oldest - run: yarn test:appsec:plugins:ci - uses: ./.github/actions/node/latest - run: yarn test:appsec:plugins:ci - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 postgres: runs-on: ubuntu-latest @@ -83,18 +85,16 @@ jobs: PLUGINS: pg|knex SERVICES: postgres steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - - run: yarn install + - uses: ./.github/actions/install - uses: ./.github/actions/node/oldest - run: yarn test:appsec:plugins:ci - - uses: ./.github/actions/node/16 - - run: yarn test:appsec:plugins:ci - uses: ./.github/actions/node/18 - run: yarn test:appsec:plugins:ci - uses: ./.github/actions/node/20 - run: yarn test:appsec:plugins:ci - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 mysql: runs-on: ubuntu-latest @@ -110,30 +110,42 @@ jobs: PLUGINS: mysql|mysql2|sequelize SERVICES: mysql steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/16 - - run: yarn test:appsec:plugins:ci + - uses: ./.github/actions/install - uses: ./.github/actions/node/18 - run: yarn test:appsec:plugins:ci - uses: ./.github/actions/node/20 - run: yarn test:appsec:plugins:ci - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 express: runs-on: ubuntu-latest env: PLUGINS: express|body-parser|cookie-parser steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - - run: yarn install + - uses: ./.github/actions/install - uses: ./.github/actions/node/oldest - run: yarn test:appsec:plugins:ci - uses: ./.github/actions/node/latest - run: yarn test:appsec:plugins:ci - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 + + graphql: + runs-on: ubuntu-latest + env: + PLUGINS: apollo-server|apollo-server-express|apollo-server-fastify|apollo-server-core + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/node/setup + - uses: ./.github/actions/install + - uses: ./.github/actions/node/oldest + - run: yarn test:appsec:plugins:ci + - uses: ./.github/actions/node/latest + - run: yarn test:appsec:plugins:ci + - uses: codecov/codecov-action@v3 mongodb-core: runs-on: ubuntu-latest @@ -143,17 +155,17 @@ jobs: ports: - 27017:27017 env: - PLUGINS: express-mongo-sanitize + PLUGINS: express-mongo-sanitize|mquery SERVICES: mongo steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - - run: yarn install + - uses: ./.github/actions/install - uses: ./.github/actions/node/oldest - run: yarn test:appsec:plugins:ci - uses: ./.github/actions/node/latest - run: yarn test:appsec:plugins:ci - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 mongoose: runs-on: ubuntu-latest @@ -166,54 +178,89 @@ jobs: PLUGINS: mongoose SERVICES: mongo steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - - run: yarn install + - uses: ./.github/actions/install - uses: ./.github/actions/node/oldest - run: yarn test:appsec:plugins:ci - uses: ./.github/actions/node/latest - run: yarn test:appsec:plugins:ci - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 sourcing: runs-on: ubuntu-latest env: PLUGINS: cookie steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/16 - - run: yarn test:appsec:plugins:ci + - uses: ./.github/actions/install - uses: ./.github/actions/node/18 - run: yarn test:appsec:plugins:ci - uses: ./.github/actions/node/20 - run: yarn test:appsec:plugins:ci - uses: ./.github/actions/node/latest - run: yarn test:appsec:plugins:ci - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 next: strategy: matrix: - node-version: [16] - range: ['>=9.5 <11.1', '>=11.1 <13.2'] - include: - - node-version: 18 - range: '>=13.2' + version: + - 18 + - latest + range: ['9.5.0', '11.1.4', '13.2.0', '14.2.6'] runs-on: ubuntu-latest env: PLUGINS: next - RANGE: ${{ matrix.range }} + PACKAGE_VERSION_RANGE: ${{ matrix.range }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/setup - uses: actions/setup-node@v3 with: - node-version: ${{ matrix.node-version }} - - run: yarn install + cache: yarn + node-version: ${{ matrix.version }} + - uses: ./.github/actions/install - run: yarn test:appsec:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 + + lodash: + runs-on: ubuntu-latest + env: + PLUGINS: lodash + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/node/setup + - uses: ./.github/actions/install + - uses: ./.github/actions/node/oldest + - run: yarn test:appsec:plugins:ci + - uses: ./.github/actions/node/latest + - run: yarn test:appsec:plugins:ci + - uses: codecov/codecov-action@v3 + + integration: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: yarn install + - uses: ./.github/actions/node/oldest + - run: yarn test:integration:appsec + - uses: ./.github/actions/node/latest + - run: yarn test:integration:appsec + + passport: + runs-on: ubuntu-latest + env: + PLUGINS: passport-local|passport-http + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/node/setup + - uses: ./.github/actions/install + - uses: ./.github/actions/node/oldest + - run: yarn test:appsec:plugins:ci + - uses: ./.github/actions/node/latest + - run: yarn test:appsec:plugins:ci + - uses: codecov/codecov-action@v3 diff --git a/.github/workflows/ci-visibility-performance.yml b/.github/workflows/ci-visibility-performance.yml index c399c9b3096..2a24980b4d5 100644 --- a/.github/workflows/ci-visibility-performance.yml +++ b/.github/workflows/ci-visibility-performance.yml @@ -19,7 +19,7 @@ jobs: env: ROBOT_CI_GITHUB_PERSONAL_ACCESS_TOKEN: ${{ secrets.ROBOT_CI_GITHUB_PERSONAL_ACCESS_TOKEN }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/18 - name: CI Visibility Performance Overhead Test run: yarn bench:e2e:ci-visibility diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index af37ccf7d90..51af025df84 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -34,7 +34,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index e8661d9652b..b6241113c3a 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -15,11 +15,11 @@ jobs: shimmer: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - - run: yarn install + - uses: ./.github/actions/install - uses: ./.github/actions/node/oldest - run: yarn test:shimmer:ci - uses: ./.github/actions/node/latest - run: yarn test:shimmer:ci - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 diff --git a/.github/workflows/datadog-static-analysis.yml b/.github/workflows/datadog-static-analysis.yml new file mode 100644 index 00000000000..d392f617b9b --- /dev/null +++ b/.github/workflows/datadog-static-analysis.yml @@ -0,0 +1,24 @@ +name: Datadog Static Analysis + +on: + pull_request: + push: + branches: [master] + +jobs: + static-analysis: + runs-on: ubuntu-latest + name: Datadog Static Analyzer + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Check code meets quality and security standards + id: datadog-static-analysis + uses: DataDog/datadog-static-analyzer-github-action@v1 + with: + dd_api_key: ${{ secrets.DD_STATIC_ANALYSIS_API_KEY }} + dd_app_key: ${{ secrets.DD_STATIC_ANALYSIS_APP_KEY }} + dd_service: dd-trace-js + dd_env: ci + dd_site: datadoghq.com + cpu_count: 2 diff --git a/.github/workflows/debugger.yml b/.github/workflows/debugger.yml new file mode 100644 index 00000000000..b9543148382 --- /dev/null +++ b/.github/workflows/debugger.yml @@ -0,0 +1,33 @@ +name: Debugger + +on: + pull_request: + push: + branches: [master] + schedule: + - cron: '0 4 * * *' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref || github.run_id }} + cancel-in-progress: true + +jobs: + ubuntu: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/testagent/start + - uses: ./.github/actions/node/setup + - uses: ./.github/actions/install + - uses: ./.github/actions/node/18 + - run: yarn test:debugger:ci + - run: yarn test:integration:debugger + - uses: ./.github/actions/node/20 + - run: yarn test:debugger:ci + - run: yarn test:integration:debugger + - uses: ./.github/actions/node/latest + - run: yarn test:debugger:ci + - run: yarn test:integration:debugger + - if: always() + uses: ./.github/actions/testagent/logs + - uses: codecov/codecov-action@v3 diff --git a/.github/workflows/lambda.yml b/.github/workflows/lambda.yml index 2600cc157f0..f0ee5d05b72 100644 --- a/.github/workflows/lambda.yml +++ b/.github/workflows/lambda.yml @@ -15,12 +15,10 @@ jobs: ubuntu: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/16 - - run: yarn test:lambda:ci + - uses: ./.github/actions/install - uses: ./.github/actions/node/18 - run: yarn test:lambda:ci - uses: ./.github/actions/node/20 @@ -29,4 +27,4 @@ jobs: - run: yarn test:lambda:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 diff --git a/.github/workflows/package-size.yml b/.github/workflows/package-size.yml index a29c22f29cb..628614c7dc5 100644 --- a/.github/workflows/package-size.yml +++ b/.github/workflows/package-size.yml @@ -12,12 +12,14 @@ concurrency: jobs: package-size-report: runs-on: ubuntu-latest + permissions: + pull-requests: write steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Setup Node.js uses: actions/setup-node@v2 with: - node-version: '16' + node-version: '18' - run: yarn - name: Compute module size tree and report uses: qard/heaviest-objects-in-the-universe@v1 diff --git a/.github/workflows/plugins.yml b/.github/workflows/plugins.yml index 2e2aa3b5764..c71ff2a2441 100644 --- a/.github/workflows/plugins.yml +++ b/.github/workflows/plugins.yml @@ -11,100 +11,92 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref || github.run_id }} cancel-in-progress: true -env: - SLACK_REPORT_ENABLE: ${{ github.event.schedule }} # value is empty for non-nightly jobs - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - SLACK_MOREINFO: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} - # TODO: upstream jobs jobs: - aerospike-3: + aerospike-node-16: runs-on: ubuntu-latest - container: - image: ubuntu:18.04 services: aerospike: - image: aerospike:ce-6.4.0.3 - ports: - - 3000:3000 - testagent: - image: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:latest - env: - LOG_LEVEL: DEBUG - TRACE_LANGUAGE: javascript - DISABLED_CHECKS: trace_content_length - PORT: 9126 + image: aerospike:ce-5.7.0.15 ports: - - 9126:9126 + - "127.0.0.1:3000-3002:3000-3002" env: PLUGINS: aerospike SERVICES: aerospike - PACKAGE_VERSION_RANGE: '3.16.2 - 3.16.7' - DD_TEST_AGENT_URL: http://testagent:9126 - AEROSPIKE_HOST_ADDRESS: aerospike + PACKAGE_VERSION_RANGE: '>=4.0.0 <5.2.0' steps: - - uses: actions/checkout@v2 - - uses: actions/setup-node@v3 - with: - node-version: '14' + - uses: actions/checkout@v4 + - uses: ./.github/actions/testagent/start + - uses: ./.github/actions/node/setup - id: pkg run: | content=`cat ./package.json | tr '\n' ' '` - echo "::set-output name=json::$content" + echo "json=$content" >> $GITHUB_OUTPUT - id: extract run: | version="${{fromJson(steps.pkg.outputs.json).version}}" majorVersion=$(echo "$version" | cut -d '.' -f 1) echo "Major Version: $majorVersion" echo "MAJOR_VERSION=$majorVersion" >> $GITHUB_ENV - - name: Check package version - if: env.MAJOR_VERSION == '3' - run: | - echo "Package version is 3. Proceeding with the next steps." + - uses: ./.github/actions/node/oldest - name: Install dependencies - if: env.MAJOR_VERSION == '3' - run: | - apt-get update && \ - apt-get install -y \ - python3 python3-pip \ - wget \ - g++ libssl1.0.0 libssl-dev zlib1g-dev && \ - npm install -g yarn - - if: env.MAJOR_VERSION == '3' - run: yarn install --ignore-engines - - if: env.MAJOR_VERSION == '3' - uses: ./.github/actions/node/14 - - if: env.MAJOR_VERSION == '3' + if: env.MAJOR_VERSION == '4' + uses: ./.github/actions/install + - name: Run tests + if: env.MAJOR_VERSION == '4' run: yarn test:plugins:ci - - if: env.MAJOR_VERSION == '3' - uses: codecov/codecov-action@v2 - aerospike: + - if: always() + uses: ./.github/actions/testagent/logs + - uses: codecov/codecov-action@v3 + + aerospike-node-18-20: + strategy: + matrix: + node-version: [18] + range: ['5.2.0 - 5.7.0'] + include: + - node-version: 20 + range: '>=5.8.0' runs-on: ubuntu-latest services: aerospike: image: aerospike:ce-6.4.0.3 - ports: + ports: - "127.0.0.1:3000-3002:3000-3002" env: PLUGINS: aerospike SERVICES: aerospike - PACKAGE_VERSION_RANGE: '4.0.0 - 5.7.0' + PACKAGE_VERSION_RANGE: ${{ matrix.range }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - - run: yarn install --ignore-engines - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - run: echo "PACKAGE_VERSION_RANGE=>=5.8.0" >> "$GITHUB_ENV" - - uses: ./.github/actions/node/20 # currently the latest version of aerospike only supports node 20 - - run: yarn test:plugins:ci + - id: pkg + run: | + content=`cat ./package.json | tr '\n' ' '` + echo "json=$content" >> $GITHUB_OUTPUT + - id: extract + run: | + version="${{fromJson(steps.pkg.outputs.json).version}}" + majorVersion=$(echo "$version" | cut -d '.' -f 1) + echo "Major Version: $majorVersion" + echo "MAJOR_VERSION=$majorVersion" >> $GITHUB_ENV + - uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + - name: Install dependencies + if: env.MAJOR_VERSION == '5' + uses: ./.github/actions/install + - name: Run tests + if: env.MAJOR_VERSION == '5' + run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 - amqp10: # TODO: move rhea to its own job + - uses: codecov/codecov-action@v3 + + amqp10: runs-on: ubuntu-latest services: qpid: @@ -115,22 +107,12 @@ jobs: ports: - 5673:5672 env: - PLUGINS: amqp10|rhea + PLUGINS: amqp10 SERVICES: qpid + DD_DATA_STREAMS_ENABLED: true steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - run: yarn test:plugins:upstream - - uses: ./.github/actions/node/latest - - run: yarn test:plugins:ci - - run: yarn test:plugins:upstream - - if: always() - uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test-and-upstream amqplib: runs-on: ubuntu-latest @@ -143,27 +125,36 @@ jobs: PLUGINS: amqplib SERVICES: rabbitmq steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - run: yarn test:plugins:upstream - - uses: ./.github/actions/node/latest - - run: yarn test:plugins:ci - - run: yarn test:plugins:upstream - - if: always() - uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test-and-upstream + + apollo: + runs-on: ubuntu-latest + env: + PLUGINS: apollo + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test-and-upstream + + avsc: + runs-on: ubuntu-latest + env: + PLUGINS: avsc + DD_DATA_STREAMS_ENABLED: true + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test-and-upstream aws-sdk: + strategy: + matrix: + node-version: ['18', 'latest'] runs-on: ubuntu-latest services: localstack: - image: localstack/localstack:1.1.0 + image: localstack/localstack:3.0.2 env: - LOCALSTACK_SERVICES: dynamodb,kinesis,s3,sqs,sns,redshift,route53,logs,serverless + LOCALSTACK_SERVICES: dynamodb,kinesis,s3,sqs,sns,redshift,route53,logs,serverless,lambda,stepfunctions,events EXTRA_CORS_ALLOWED_HEADERS: x-amz-request-id,x-amzn-requestid,x-amz-id-2 EXTRA_CORS_EXPOSE_HEADERS: x-amz-request-id,x-amzn-requestid,x-amz-id-2 AWS_DEFAULT_REGION: us-east-1 @@ -172,101 +163,126 @@ jobs: START_WEB: '0' ports: - 4566:4566 + # we have two localstacks since upgrading localstack was causing lambda & S3 tests to fail + # To-Do: Debug localstack / lambda and localstack / S3 + localstack-legacy: + image: localstack/localstack:1.1.0 + ports: + - "127.0.0.1:4567:4567" # Edge + env: + LOCALSTACK_SERVICES: dynamodb,kinesis,s3,sqs,sns,redshift,route53,logs,serverless + EXTRA_CORS_ALLOWED_HEADERS: x-amz-request-id,x-amzn-requestid,x-amz-id-2 + EXTRA_CORS_EXPOSE_HEADERS: x-amz-request-id,x-amzn-requestid,x-amz-id-2 + AWS_DEFAULT_REGION: us-east-1 + FORCE_NONINTERACTIVE: 'true' + LAMBDA_EXECUTOR: local + START_WEB: '0' + GATEWAY_LISTEN: 127.0.0.1:4567 + EDGE_PORT: 4567 + EDGE_PORT_HTTP: 4567 env: PLUGINS: aws-sdk - SERVICES: localstack + SERVICES: localstack localstack-legacy + DD_DATA_STREAMS_ENABLED: true steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - uses: ./.github/actions/node/latest + - uses: ./.github/actions/install + - uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 axios: runs-on: ubuntu-latest env: PLUGINS: axios steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:upstream - - uses: ./.github/actions/node/latest - - run: yarn test:plugins:upstream - - if: always() - uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/upstream + + azure-functions: + runs-on: ubuntu-latest + env: + PLUGINS: azure-functions + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test bluebird: runs-on: ubuntu-latest env: PLUGINS: bluebird steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - uses: ./.github/actions/node/latest - - run: yarn test:plugins:ci - - uses: codecov/codecov-action@v2 - - if: always() - uses: ./.github/actions/testagent/logs + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test + + body-parser: + runs-on: ubuntu-latest + env: + PLUGINS: body-parser + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test bunyan: runs-on: ubuntu-latest env: PLUGINS: bunyan steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - run: yarn test:plugins:upstream - - uses: ./.github/actions/node/latest - - run: yarn test:plugins:ci - - run: yarn test:plugins:upstream - - if: always() - uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test-and-upstream cassandra: runs-on: ubuntu-latest services: cassandra: - image: spotify/cassandra - env: - CASSANDRA_TOKEN: '-9223372036854775808' + image: cassandra:3-focal ports: - 9042:9042 env: PLUGINS: cassandra-driver SERVICES: cassandra + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test + + child_process: + runs-on: ubuntu-latest + env: + PLUGINS: child_process steps: - uses: actions/checkout@v2 - - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - - run: yarn install + - uses: ./.github/actions/install - uses: ./.github/actions/node/oldest - run: yarn test:plugins:ci + - uses: ./.github/actions/node/20 + - run: yarn test:plugins:ci - uses: ./.github/actions/node/latest - run: yarn test:plugins:ci - - if: always() - uses: ./.github/actions/testagent/logs - uses: codecov/codecov-action@v2 + cookie-parser: + runs-on: ubuntu-latest + env: + PLUGINS: cookie-parser + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test + couchbase: + strategy: + matrix: + node-version: [16] + range: ['^2.6.12', '^3.0.7', '>=4.0.0 <4.2.0'] + include: + - node-version: 18 + range: '>=4.2.0' runs-on: ubuntu-latest services: couchbase: @@ -277,50 +293,34 @@ jobs: env: PLUGINS: couchbase SERVICES: couchbase + PACKAGE_VERSION_RANGE: ${{ matrix.range }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - uses: codecov/codecov-action@v2 + - uses: ./.github/actions/install + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + - run: yarn config set ignore-engines true + - run: yarn test:plugins:ci --ignore-engines + - uses: codecov/codecov-action@v3 connect: runs-on: ubuntu-latest env: PLUGINS: connect steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - run: yarn test:plugins:upstream - - uses: ./.github/actions/node/latest - - run: yarn test:plugins:ci - - run: yarn test:plugins:upstream - - if: always() - uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test-and-upstream cucumber: runs-on: ubuntu-latest env: PLUGINS: cucumber steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - uses: ./.github/actions/node/latest - - run: yarn test:plugins:ci - - if: always() - uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test # TODO: fix performance issues and test more Node versions cypress: @@ -328,26 +328,24 @@ jobs: env: PLUGINS: cypress steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - - run: yarn install + - uses: ./.github/actions/install - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 dns: runs-on: ubuntu-latest env: PLUGINS: dns steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/16 - - run: yarn test:plugins:ci + - uses: ./.github/actions/install - uses: ./.github/actions/node/18 - run: yarn test:plugins:ci - uses: ./.github/actions/node/20 @@ -356,13 +354,13 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 elasticsearch: runs-on: ubuntu-latest services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.14.0 + image: elasticsearch:7.17.22 env: discovery.type: single-node ports: @@ -371,85 +369,62 @@ jobs: PLUGINS: elasticsearch SERVICES: elasticsearch steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci + - uses: ./.github/actions/install - uses: ./.github/actions/node/latest - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 express: runs-on: ubuntu-latest env: - PLUGINS: express|body-parser|cookie-parser + PLUGINS: express steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - uses: ./.github/actions/node/latest - - run: yarn test:plugins:ci - - if: always() - uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test + + express-mongo-sanitize: + runs-on: ubuntu-latest + services: + mongodb: + image: circleci/mongo + ports: + - 27017:27017 + env: + PLUGINS: express-mongo-sanitize + PACKAGE_NAMES: express-mongo-sanitize + SERVICES: mongo + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test fastify: runs-on: ubuntu-latest env: PLUGINS: fastify steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - uses: ./.github/actions/node/latest - - run: yarn test:plugins:ci - - if: always() - uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test fetch: runs-on: ubuntu-latest env: PLUGINS: fetch steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - uses: ./.github/actions/node/latest - - run: yarn test:plugins:ci - - if: always() - uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test generic-pool: runs-on: ubuntu-latest env: PLUGINS: generic-pool steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - uses: ./.github/actions/node/latest - - run: yarn test:plugins:ci - - if: always() - uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test google-cloud-pubsub: runs-on: ubuntu-latest @@ -462,103 +437,62 @@ jobs: PLUGINS: google-cloud-pubsub SERVICES: gpubsub steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - uses: ./.github/actions/node/latest - - run: yarn test:plugins:ci - - if: always() - uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test graphql: runs-on: ubuntu-latest env: PLUGINS: graphql steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - run: yarn test:plugins:upstream - - uses: ./.github/actions/node/latest - - run: yarn test:plugins:ci - - run: yarn test:plugins:upstream - - if: always() - uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test-and-upstream grpc: runs-on: ubuntu-latest env: PLUGINS: grpc steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - uses: ./.github/actions/node/latest - - run: yarn test:plugins:ci - - if: always() - uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test hapi: runs-on: ubuntu-latest env: PLUGINS: hapi steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - uses: ./.github/actions/node/latest - - run: yarn test:plugins:ci - - if: always() - uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test http: + strategy: + matrix: + node-version: ['18', '20', 'latest'] runs-on: ubuntu-latest env: PLUGINS: http steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/16 - - run: yarn test:plugins:ci - - uses: ./.github/actions/node/18 - - run: yarn test:plugins:ci - - uses: ./.github/actions/node/20 - - run: yarn test:plugins:ci - - uses: ./.github/actions/node/latest + - uses: ./.github/actions/install + - uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 http2: runs-on: ubuntu-latest env: PLUGINS: http2 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/16 - - run: yarn test:plugins:ci + - uses: ./.github/actions/install - uses: ./.github/actions/node/18 - run: yarn test:plugins:ci - uses: ./.github/actions/node/20 @@ -567,7 +501,7 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 # TODO: fix performance issues and test more Node versions jest: @@ -575,27 +509,32 @@ jobs: env: PLUGINS: jest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - - run: yarn install + - uses: ./.github/actions/install - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 kafkajs: runs-on: ubuntu-latest services: kafka: - image: debezium/kafka:1.7 + image: apache/kafka-native:3.8.0-rc2 env: - CLUSTER_ID: 5Yr1SIgYQz-b-dgRabWx4g - NODE_ID: "1" - CREATE_TOPICS: "test-topic:1:1" - KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:9093 + KAFKA_PROCESS_ROLES: broker,controller + KAFKA_NODE_ID: '1' + KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093 + KAFKA_CONTROLLER_QUORUM_VOTERS: 1@127.0.0.1:9093 + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_CLUSTER_ID: r4zt_wrqTRuT7W2NJsB_GA KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://127.0.0.1:9092 - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: "0" + KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: '1' + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: '0' ports: - 9092:9092 - 9093:9093 @@ -603,53 +542,24 @@ jobs: PLUGINS: kafkajs SERVICES: kafka steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - uses: ./.github/actions/node/latest - - run: yarn test:plugins:ci - - if: always() - uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test knex: runs-on: ubuntu-latest env: PLUGINS: knex steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - uses: ./.github/actions/node/latest - - run: yarn test:plugins:ci - - if: always() - uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test koa: runs-on: ubuntu-latest env: PLUGINS: koa steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - run: yarn test:plugins:upstream - - uses: ./.github/actions/node/latest - - run: yarn test:plugins:ci - - run: yarn test:plugins:upstream - - if: always() - uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test-and-upstream limitd-client: runs-on: ubuntu-latest @@ -666,17 +576,25 @@ jobs: PLUGINS: limitd-client SERVICES: limitd steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - uses: ./.github/actions/node/latest - - run: yarn test:plugins:ci - - if: always() - uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test + + mariadb: + runs-on: ubuntu-latest + services: + mysql: + image: mariadb:10.4 + env: + MYSQL_ALLOW_EMPTY_PASSWORD: 'yes' + MYSQL_DATABASE: 'db' + ports: + - 3306:3306 + env: + PLUGINS: mariadb + SERVICES: mariadb + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test memcached: runs-on: ubuntu-latest @@ -689,68 +607,47 @@ jobs: PLUGINS: memcached SERVICES: memcached steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - uses: ./.github/actions/node/latest - - run: yarn test:plugins:ci - - if: always() - uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test microgateway-core: runs-on: ubuntu-latest env: PLUGINS: microgateway-core steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - uses: ./.github/actions/node/latest - - run: yarn test:plugins:ci - - if: always() - uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test mocha: runs-on: ubuntu-latest env: PLUGINS: mocha steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - uses: ./.github/actions/node/latest - - run: yarn test:plugins:ci - - uses: codecov/codecov-action@v2 - - if: always() - uses: ./.github/actions/testagent/logs + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test moleculer: runs-on: ubuntu-latest env: PLUGINS: moleculer steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - uses: ./.github/actions/node/latest - - run: yarn test:plugins:ci - - if: always() - uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test + + mongodb: + runs-on: ubuntu-latest + services: + mongodb: + image: circleci/mongo + ports: + - 27017:27017 + env: + PLUGINS: mongodb-core + PACKAGE_NAMES: mongodb + SERVICES: mongo + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test mongodb-core: runs-on: ubuntu-latest @@ -761,19 +658,11 @@ jobs: - 27017:27017 env: PLUGINS: mongodb-core|express-mongo-sanitize + PACKAGE_NAMES: mongodb-core,express-mongo-sanitize SERVICES: mongo steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - uses: ./.github/actions/node/latest - - run: yarn test:plugins:ci - - if: always() - uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test mongoose: runs-on: ubuntu-latest @@ -786,17 +675,8 @@ jobs: PLUGINS: mongoose SERVICES: mongo steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - uses: ./.github/actions/node/latest - - run: yarn test:plugins:ci - - if: always() - uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test mysql: runs-on: ubuntu-latest @@ -809,32 +689,38 @@ jobs: ports: - 3306:3306 env: - PLUGINS: mysql|mysql2|mariadb # TODO: move mysql2 to its own job + PLUGINS: mysql SERVICES: mysql steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - uses: ./.github/actions/node/latest - - run: yarn test:plugins:ci - - if: always() - uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test + + mysql2: + runs-on: ubuntu-latest + services: + mysql: + image: mariadb:10.4 + env: + MYSQL_ALLOW_EMPTY_PASSWORD: 'yes' + MYSQL_DATABASE: 'db' + ports: + - 3306:3306 + env: + PLUGINS: mysql2 + SERVICES: mysql2 + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test net: runs-on: ubuntu-latest env: PLUGINS: net steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/16 - - run: yarn test:plugins:ci + - uses: ./.github/actions/install - uses: ./.github/actions/node/18 - run: yarn test:plugins:ci - uses: ./.github/actions/node/20 @@ -843,50 +729,37 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 # TODO: fix performance issues and test more Node versions next: strategy: matrix: - node-version: [16] - range: ['>=9.5 <11.1', '>=11.1 <13.2'] - include: - - node-version: 18 - range: '>=13.2' + version: + - 18 + - latest + range: ['9.5.0', '11.1.4', '13.2.0', '14.2.6'] runs-on: ubuntu-latest env: PLUGINS: next - RANGE: ${{ matrix.range }} + PACKAGE_VERSION_RANGE: ${{ matrix.range }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node-version }} - - run: yarn install + - uses: ./.github/actions/install - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 openai: runs-on: ubuntu-latest env: PLUGINS: openai steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - uses: ./.github/actions/node/latest - - run: yarn test:plugins:ci - - if: always() - uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test opensearch: runs-on: ubuntu-latest @@ -902,17 +775,8 @@ jobs: PLUGINS: opensearch SERVICES: opensearch steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - uses: ./.github/actions/node/latest - - run: yarn test:plugins:ci - - if: always() - uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test # TODO: Install the Oracle client on the host and test Node >=16. # TODO: Figure out why nyc stopped working with EACCESS errors. @@ -928,11 +792,11 @@ jobs: - 1521:1521 - 5500:5500 testagent: - image: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:latest + image: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.16.0 env: LOG_LEVEL: DEBUG TRACE_LANGUAGE: javascript - DISABLED_CHECKS: trace_content_length + ENABLED_CHECKS: trace_stall,meta_tracer_version_header,trace_count_header,trace_peer_service PORT: 9126 ports: - 9126:9126 @@ -940,12 +804,18 @@ jobs: PLUGINS: oracledb SERVICES: oracledb DD_TEST_AGENT_URL: http://testagent:9126 + # Needed to fix issue with `actions/checkout@v3: https://github.com/actions/checkout/issues/1590 + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/node/setup - - run: yarn install --ignore-engines - - run: yarn services - - run: yarn test:plugins + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + cache: yarn + node-version: '16' + - uses: ./.github/actions/install + - run: yarn config set ignore-engines true + - run: yarn services --ignore-engines + - run: yarn test:plugins --ignore-engines - uses: codecov/codecov-action@v2 paperplane: @@ -953,15 +823,15 @@ jobs: env: PLUGINS: paperplane steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - - run: yarn install + - uses: ./.github/actions/install - uses: ./.github/actions/node/oldest - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 # TODO: re-enable upstream tests if it ever stops being flaky pino: @@ -969,16 +839,18 @@ jobs: env: PLUGINS: pino steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - - run: yarn install + - uses: ./.github/actions/install + - uses: ./.github/actions/node/20 + - run: yarn test:plugins:ci - uses: ./.github/actions/node/latest - run: yarn test:plugins:ci # - run: yarn test:plugins:upstream - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 postgres: runs-on: ubuntu-latest @@ -994,70 +866,41 @@ jobs: PLUGINS: pg SERVICES: postgres steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - uses: ./.github/actions/node/latest - - run: yarn test:plugins:ci - - if: always() - uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test promise: runs-on: ubuntu-latest env: PLUGINS: promise steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - run: yarn test:plugins:upstream - - uses: ./.github/actions/node/latest - - run: yarn test:plugins:ci - - run: yarn test:plugins:upstream - - if: always() - uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test-and-upstream promise-js: runs-on: ubuntu-latest env: PLUGINS: promise-js steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - uses: ./.github/actions/node/latest - - run: yarn test:plugins:ci - - if: always() - uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test + + protobufjs: + runs-on: ubuntu-latest + env: + PLUGINS: protobufjs + DD_DATA_STREAMS_ENABLED: true + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test-and-upstream q: runs-on: ubuntu-latest env: PLUGINS: q steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - uses: ./.github/actions/node/latest - - run: yarn test:plugins:ci - - if: always() - uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test redis: runs-on: ubuntu-latest @@ -1070,74 +913,63 @@ jobs: PLUGINS: redis|ioredis # TODO: move ioredis to its own job SERVICES: redis steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - uses: ./.github/actions/node/latest - - run: yarn test:plugins:ci - - if: always() - uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test restify: runs-on: ubuntu-latest env: PLUGINS: restify steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - uses: ./.github/actions/node/16 - - run: yarn test:plugins:ci - - if: always() - uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test + + rhea: + runs-on: ubuntu-latest + services: + qpid: + image: scholzj/qpid-cpp:1.38.0 + env: + QPIDD_ADMIN_USERNAME: admin + QPIDD_ADMIN_PASSWORD: admin + ports: + - 5673:5672 + env: + PLUGINS: rhea + SERVICES: qpid + DD_DATA_STREAMS_ENABLED: true + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test-and-upstream router: runs-on: ubuntu-latest env: PLUGINS: router steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - uses: ./.github/actions/node/latest - - run: yarn test:plugins:ci - - if: always() - uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test sharedb: runs-on: ubuntu-latest env: PLUGINS: sharedb steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci + - uses: ./.github/actions/install - uses: ./.github/actions/node/latest - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 tedious: runs-on: ubuntu-latest services: mssql: - image: mcr.microsoft.com/mssql/server:2017-latest-ubuntu + image: mcr.microsoft.com/mssql/server:2019-latest env: ACCEPT_EULA: 'Y' SA_PASSWORD: DD_HUNTER2 @@ -1148,49 +980,37 @@ jobs: PLUGINS: tedious SERVICES: mssql steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/16 - - run: yarn test:plugins:ci - - run: yarn test:plugins:upstream + - uses: ./.github/actions/install - uses: ./.github/actions/node/latest - run: yarn test:plugins:ci - run: yarn test:plugins:upstream - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 + + undici: + runs-on: ubuntu-latest + env: + PLUGINS: undici + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test when: runs-on: ubuntu-latest env: PLUGINS: when steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - uses: ./.github/actions/node/latest - - run: yarn test:plugins:ci - - if: always() - uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test winston: runs-on: ubuntu-latest env: PLUGINS: winston steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/testagent/start - - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - uses: ./.github/actions/node/latest - - run: yarn test:plugins:ci - - if: always() - uses: ./.github/actions/testagent/logs + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test diff --git a/.github/workflows/prepare-release-proposal.yml b/.github/workflows/prepare-release-proposal.yml new file mode 100644 index 00000000000..46e472e4e33 --- /dev/null +++ b/.github/workflows/prepare-release-proposal.yml @@ -0,0 +1,101 @@ +name: Prepare release proposal + +on: + workflow_dispatch: + +jobs: + create-proposal: + strategy: + matrix: + base-branch: + - v4.x + - v5.x + runs-on: ubuntu-latest + + permissions: write-all + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ matrix.base-branch }} + token: ${{ secrets.GH_ACCESS_TOKEN_RELEASE }} + + - name: Set up Git + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + - name: Pull master branch + run: | + git checkout master + git pull + cp -r scripts _scripts + git checkout ${{ matrix.base-branch }} + + + - name: Configure node + uses: actions/setup-node@v3 + + - name: Install dependencies + run: | + yarn + git checkout yarn.lock + + - name: Install branch-diff + run: | + npm i -g @bengl/branch-diff + + - name: Configure branch-diff + run: | + mkdir -p ~/.config/changelog-maker + echo "{\"token\":\"${{secrets.GITHUB_TOKEN}}\",\"user\":\"${{github.actor}}\"}" > ~/.config/changelog-maker/config.json + + - name: Commit branch diffs + id: commit_branch_diffs + run: | + node _scripts/prepare-release-proposal.js commit-branch-diffs ${{ matrix.base-branch }} > branch-diffs.txt + + - name: Calculate release type + id: calc-release-type + run: | + release_type=`grep -q "(SEMVER-MINOR)" branch-diffs.txt && echo "minor" || echo "patch"` + echo "release-type=$release_type" >> $GITHUB_OUTPUT + + - name: Create proposal branch + id: create_branch + run: | + branch_name=`node _scripts/prepare-release-proposal.js create-branch ${{ steps.calc-release-type.outputs.release-type }}` + echo "branch_name=$branch_name" >> $GITHUB_OUTPUT + + - name: Push proposal branch + run: | + git push origin ${{steps.create_branch.outputs.branch_name}} + + - name: Update package.json + id: pkg + run: | + content=`node _scripts/prepare-release-proposal.js update-package-json ${{ steps.calc-release-type.outputs.release-type }}` + echo "version=$content" >> $GITHUB_OUTPUT + + - name: Create PR + run: | + gh pr create --draft --base ${{ matrix.base-branch }} --title "v${{ steps.pkg.outputs.version }}" -F branch-diffs.txt + rm branch-diffs.txt + env: + GH_TOKEN: ${{ github.token }} + + # Commit package.json and push to proposal branch after the PR is created to force CI execution + - name: Commit package.json + run: | + git add package.json + git commit -m "v${{ steps.pkg.outputs.version }}" + + - name: Push package.json update + run: | + git push origin ${{steps.create_branch.outputs.branch_name}} + + - name: Clean _scripts + run: | + rm -rf _scripts diff --git a/.github/workflows/profiling.yml b/.github/workflows/profiling.yml index 05e9696cc48..7477e38dade 100644 --- a/.github/workflows/profiling.yml +++ b/.github/workflows/profiling.yml @@ -15,33 +15,38 @@ jobs: macos: runs-on: macos-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - - run: yarn install + - uses: ./.github/actions/install - run: yarn test:profiler:ci - - uses: codecov/codecov-action@v2 + - run: yarn test:integration:profiler + - uses: codecov/codecov-action@v3 ubuntu: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/16 - - run: yarn test:profiler:ci + - uses: ./.github/actions/install - uses: ./.github/actions/node/18 - run: yarn test:profiler:ci + - run: yarn test:integration:profiler - uses: ./.github/actions/node/20 - run: yarn test:profiler:ci + - run: yarn test:integration:profiler - uses: ./.github/actions/node/latest - run: yarn test:profiler:ci - - uses: codecov/codecov-action@v2 + - run: yarn test:integration:profiler + - uses: codecov/codecov-action@v3 windows: runs-on: windows-latest steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/node/setup - - run: yarn install + - uses: actions/checkout@v4 + - uses: actions/setup-node@v3 + with: + node-version: '18' + - uses: ./.github/actions/install - run: yarn test:profiler:ci - - uses: codecov/codecov-action@v2 + - run: yarn test:integration:profiler + - uses: codecov/codecov-action@v3 diff --git a/.github/workflows/project.yml b/.github/workflows/project.yml index 05079d33112..588e148fdeb 100644 --- a/.github/workflows/project.yml +++ b/.github/workflows/project.yml @@ -18,34 +18,67 @@ jobs: # setting fail-fast to false in an attempt to prevent this from happening fail-fast: false matrix: - version: [16, 18, latest] + version: [18, 20, latest] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-node@v3 with: node-version: ${{ matrix.version }} # Disable core dumps since some integration tests intentionally abort and core dump generation takes around 5-10s - - run: yarn install + - uses: ./.github/actions/install - run: sudo sysctl -w kernel.core_pattern='|/bin/false' - run: yarn test:integration + # We'll run these separately for earlier (i.e. unsupported) versions + integration-guardrails: + strategy: + matrix: + version: [12, 14, 16] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.version }} + - uses: ./.github/actions/install + - run: node node_modules/.bin/mocha --colors --timeout 30000 integration-tests/init.spec.js + integration-ci: strategy: matrix: - version: [16, latest] - framework: [cucumber, playwright] + version: [18, latest] + framework: [cucumber, playwright, selenium, jest, mocha] runs-on: ubuntu-latest env: DD_SERVICE: dd-trace-js-integration-tests DD_CIVISIBILITY_AGENTLESS_ENABLED: 1 DD_API_KEY: ${{ secrets.DD_API_KEY_CI_APP }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-node@v3 with: node-version: ${{ matrix.version }} - - run: yarn install + - name: Install Google Chrome + run: | + sudo sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list' + wget -q -O - https://dl.google.com/linux/linux_signing_key.pub | sudo apt-key add - + if [ $? -ne 0 ]; then echo "Failed to add Google key"; exit 1; fi + sudo apt-get update + sudo apt-get install -y google-chrome-stable + if [ $? -ne 0 ]; then echo "Failed to install Google Chrome"; exit 1; fi + if: ${{ matrix.framework == 'selenium' }} + - name: Install ChromeDriver + run: | + export CHROME_VERSION=$(google-chrome --version) + CHROME_DRIVER_DOWNLOAD_URL=$(node --experimental-fetch scripts/get-chrome-driver-download-url.js) + wget -q "$CHROME_DRIVER_DOWNLOAD_URL" + if [ $? -ne 0 ]; then echo "Failed to download ChromeDriver"; exit 1; fi + unzip chromedriver-linux64.zip + sudo mv chromedriver-linux64/chromedriver /usr/bin/chromedriver + sudo chmod +x /usr/bin/chromedriver + if: ${{ matrix.framework == 'selenium' }} + - uses: ./.github/actions/install - run: yarn test:integration:${{ matrix.framework }} env: NODE_OPTIONS: '-r ./ci/init' @@ -53,39 +86,64 @@ jobs: integration-cypress: strategy: matrix: + # Important: This is outside the minimum supported version of dd-trace-js + # Node > 16 does not work with Cypress@6.7.0 (not even without our plugin) + # TODO: figure out what to do with this: we might have to deprecate support for cypress@6.7.0 version: [16, latest] # 6.7.0 is the minimum version we support cypress-version: [6.7.0, latest] + module-type: ['commonJS', 'esm'] runs-on: ubuntu-latest env: DD_SERVICE: dd-trace-js-integration-tests DD_CIVISIBILITY_AGENTLESS_ENABLED: 1 DD_API_KEY: ${{ secrets.DD_API_KEY_CI_APP }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - - run: yarn install + - uses: ./.github/actions/install - uses: actions/setup-node@v3 with: node-version: ${{ matrix.version }} - - run: yarn test:integration:cypress + - run: yarn config set ignore-engines true + - run: yarn test:integration:cypress --ignore-engines env: CYPRESS_VERSION: ${{ matrix.cypress-version }} NODE_OPTIONS: '-r ./ci/init' + CYPRESS_MODULE_TYPE: ${{ matrix.module-type }} + + + integration-vitest: + runs-on: ubuntu-latest + env: + DD_SERVICE: dd-trace-js-integration-tests + DD_CIVISIBILITY_AGENTLESS_ENABLED: 1 + DD_API_KEY: ${{ secrets.DD_API_KEY_CI_APP }} + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/node/setup + - uses: ./.github/actions/install + - uses: actions/setup-node@v3 + with: + node-version: 20 + - run: yarn test:integration:vitest + env: + NODE_OPTIONS: '-r ./ci/init' lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - - run: yarn install + - uses: ./.github/actions/install - run: yarn lint typescript: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - - run: yarn install + - uses: ./.github/actions/install - run: yarn type:test - run: yarn type:doc + diff --git a/.github/workflows/rebase-release-proposal.yml b/.github/workflows/rebase-release-proposal.yml new file mode 100644 index 00000000000..3ec2f1022a8 --- /dev/null +++ b/.github/workflows/rebase-release-proposal.yml @@ -0,0 +1,89 @@ +name: Rebase release proposal + +on: + workflow_dispatch: + inputs: + base-branch: + description: 'Branch to rebase onto' + required: true + type: choice + options: + - v4.x + - v5.x + +jobs: + check: + runs-on: ubuntu-latest + permissions: + id-token: write + contents: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Git + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + - name: Get PR details + id: get_pr + run: | + pr_number=$(gh pr list --head ${{ github.ref_name }} --json number --jq '.[0].number') + echo "PR_NUMBER=$pr_number" >> $GITHUB_ENV + env: + GH_TOKEN: ${{ github.token }} + + - name: Check PR approval + id: check_approval + run: | + approvals=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + "https://api.github.com/repos/${{ github.repository }}/pulls/$PR_NUMBER/reviews" \ + | jq '[.[] | select(.state == "APPROVED")] | length') + if [ "$approvals" -eq 0 ]; then + exit 1 + fi + + - name: Check CI status + id: check_ci_status + run: | + status=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + "https://api.github.com/repos/${{ github.repository }}/commits/${{ github.sha }}/status" \ + | jq -r '.state') + if [ "$status" != "success" ]; then + exit 1 + fi + + release: + needs: check + + runs-on: ubuntu-latest + + permissions: write-all + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.GH_ACCESS_TOKEN_RELEASE }} + + - name: Set up Git + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + - name: Checkout base branch + run: | + git checkout ${{ github.event.inputs.base-branch }} + + - name: Rebase branch + run: | + git rebase ${{ github.ref_name }} + + - name: Push rebased branch + run: | + git push diff --git a/.github/workflows/release-3.yml b/.github/workflows/release-3.yml index ec25371051a..107d333a7d6 100644 --- a/.github/workflows/release-3.yml +++ b/.github/workflows/release-3.yml @@ -19,7 +19,7 @@ jobs: env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: actions/setup-node@v3 with: registry-url: 'https://registry.npmjs.org' @@ -27,30 +27,7 @@ jobs: - id: pkg run: | content=`cat ./package.json | tr '\n' ' '` - echo "::set-output name=json::$content" + echo "json=$content" >> $GITHUB_OUTPUT - run: | git tag v${{ fromJson(steps.pkg.outputs.json).version }} git push origin v${{ fromJson(steps.pkg.outputs.json).version }} - - injection-image-publish: - runs-on: ubuntu-latest - needs: ['publish'] - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - - name: Log in to the Container registry - uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - id: pkg - run: | - content=`cat ./package.json | tr '\n' ' '` - echo "::set-output name=json::$content" - - name: npm pack for injection image - run: | - npm pack dd-trace@${{ fromJson(steps.pkg.outputs.json).version }} - - uses: ./.github/actions/injection - with: - init-image-version: v${{ fromJson(steps.pkg.outputs.json).version }} diff --git a/.github/workflows/release-4.yml b/.github/workflows/release-4.yml new file mode 100644 index 00000000000..169450d6cf2 --- /dev/null +++ b/.github/workflows/release-4.yml @@ -0,0 +1,33 @@ +name: Release (4.x) + +on: + push: + branches: + - v4.x + +concurrency: + group: ${{ github.workflow }}-${{ github.ref || github.run_id }} + cancel-in-progress: true + +jobs: + publish: + runs-on: ubuntu-latest + environment: npm + permissions: + id-token: write + contents: write + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v3 + with: + registry-url: 'https://registry.npmjs.org' + - run: npm publish --tag latest-node16 --provenance + - id: pkg + run: | + content=`cat ./package.json | tr '\n' ' '` + echo "json=$content" >> $GITHUB_OUTPUT + - run: | + git tag v${{ fromJson(steps.pkg.outputs.json).version }} + git push origin v${{ fromJson(steps.pkg.outputs.json).version }} diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml index 936c0ee0737..173b921267f 100644 --- a/.github/workflows/release-dev.yml +++ b/.github/workflows/release-dev.yml @@ -1,9 +1,6 @@ name: Release dev release line -on: - push: - branches: - - master +on: workflow_dispatch jobs: dev_release: @@ -15,37 +12,18 @@ jobs: env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-node@v3 with: registry-url: 'https://registry.npmjs.org' - - run: yarn install + - uses: ./.github/actions/install - id: pkg run: | content=`cat ./package.json | tr '\n' ' '` - echo "::set-output name=json::$content" + echo "json=$content" >> $GITHUB_OUTPUT - run: npm version --no-git-tag-version ${{ fromJson(steps.pkg.outputs.json).version }}-$(git rev-parse --short HEAD)+${{ github.run_id }}.${{ github.run_attempt }} - run: npm publish --tag dev --provenance - run: | git tag --force dev git push origin :refs/tags/dev git push origin --tags - - injection-image-publish: - runs-on: ubuntu-latest - needs: ['dev_release'] - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - - name: Log in to the Container registry - uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: npm pack for injection image - run: | - npm pack dd-trace@dev - - uses: ./.github/actions/injection - with: - init-image-version: dev diff --git a/.github/workflows/release-latest.yml b/.github/workflows/release-latest.yml index a45ed3c87a7..6fa92f3ee23 100644 --- a/.github/workflows/release-latest.yml +++ b/.github/workflows/release-latest.yml @@ -3,7 +3,7 @@ name: Release (latest) on: push: branches: - - v4.x + - v5.x concurrency: group: ${{ github.workflow }}-${{ github.ref || github.run_id }} @@ -21,7 +21,7 @@ jobs: outputs: pkgjson: ${{ steps.pkg.outputs.json }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: actions/setup-node@v3 with: registry-url: 'https://registry.npmjs.org' @@ -29,44 +29,24 @@ jobs: - id: pkg run: | content=`cat ./package.json | tr '\n' ' '` - echo "::set-output name=json::$content" + echo "json=$content" >> $GITHUB_OUTPUT - run: | git tag v${{ fromJson(steps.pkg.outputs.json).version }} git push origin v${{ fromJson(steps.pkg.outputs.json).version }} - injection-image-publish: - runs-on: ubuntu-latest - needs: ['publish'] - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - - name: Log in to the Container registry - uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - id: pkg - run: | - content=`cat ./package.json | tr '\n' ' '` - echo "::set-output name=json::$content" - - name: npm pack for injection image - run: | - npm pack dd-trace@${{ fromJson(steps.pkg.outputs.json).version }} - - uses: ./.github/actions/injection - with: - init-image-version: v${{ fromJson(steps.pkg.outputs.json).version }} - docs: runs-on: ubuntu-latest + permissions: + id-token: write + contents: write needs: ['publish'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-node@v3 - id: pkg run: | content=`cat ./package.json | tr '\n' ' '` - echo "::set-output name=json::$content" + echo "json=$content" >> $GITHUB_OUTPUT - run: yarn - name: Build working-directory: docs @@ -74,7 +54,7 @@ jobs: yarn yarn build mv out /tmp/out - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: gh-pages - name: Deploy diff --git a/.github/workflows/release-proposal.yml b/.github/workflows/release-proposal.yml index 4935f78c232..5faf193d3ef 100644 --- a/.github/workflows/release-proposal.yml +++ b/.github/workflows/release-proposal.yml @@ -8,7 +8,7 @@ jobs: check_labels: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: fetch-depth: 0 - uses: actions/setup-node@v3 diff --git a/.github/workflows/serverless-integration-test.yml b/.github/workflows/serverless-integration-test.yml index 1687b18fc22..b2750f11d45 100644 --- a/.github/workflows/serverless-integration-test.yml +++ b/.github/workflows/serverless-integration-test.yml @@ -13,12 +13,12 @@ jobs: id-token: 'write' strategy: matrix: - version: [16, latest] + version: [18, latest] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - - run: yarn install + - uses: ./.github/actions/install - uses: actions/setup-node@v3 with: node-version: ${{ matrix.version }} diff --git a/.github/workflows/serverless-performance.yml b/.github/workflows/serverless-performance.yml deleted file mode 100644 index 47c330ddc4f..00000000000 --- a/.github/workflows/serverless-performance.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Serverless performance test - -on: - pull_request: - -jobs: - performance-test: - runs-on: ubuntu-latest - strategy: - max-parallel: 4 - matrix: - include: - - node-version: 16.14 - aws-runtime-name: "nodejs16.x" - - node-version: 18.12 - aws-runtime-name: "nodejs18.x" - steps: - - name: Checkout dd-trace-js - uses: actions/checkout@v3 - - name: Checkout datadog-lambda-js - uses: actions/checkout@v3 - with: - repository: DataDog/datadog-lambda-js - path: datadog-lambda-js - - name: Install node-gyp - run: | - yarn global add node-gyp - - name: Update package.json to the current ref - run: | - cd datadog-lambda-js - yarn add --dev https://github.com/DataDog/dd-trace-js#refs/heads/${GITHUB_HEAD_REF} - - name: Build the layer - env: - NODE_VERSION: ${{ matrix.node-version }} - run: | - cd datadog-lambda-js - ./scripts/build_layers.sh - - name: Performance tests - uses: DataDog/serverless-performance-test-action@main - with: - runtime_id: '${{ matrix.aws-runtime-name }}' - layer_path: 'datadog-lambda-js/.layers/datadog_lambda_node${{ matrix.node-version }}.zip' - layer_name: 'performance-tester-nodejs-${{ matrix.node-version }}' - role: arn:aws:iam::425362996713:role/serverless-integration-test-lambda-role - pr_number: ${{ github.event.pull_request.number }} - env: - AWS_ACCESS_KEY_ID: ${{ secrets.SERVERLESS_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.SERVERLESS_AWS_SECRET_ACCESS_KEY }} - AWS_REGION: sa-east-1 - DD_API_KEY: ${{ secrets.SERVERLESS_DD_API_KEY }} diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index b37230370c2..0a7d4094b8b 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -11,76 +11,86 @@ on: - cron: '00 04 * * 2-6' jobs: - system-tests: + build-artifacts: runs-on: ubuntu-latest + steps: + - name: Checkout dd-trace-js + uses: actions/checkout@v4 + with: + path: dd-trace-js + - name: Pack dd-trace-js + run: mkdir -p ./binaries && echo /binaries/$(npm pack --pack-destination ./binaries ./dd-trace-js) > ./binaries/nodejs-load-from-npm + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: system_tests_binaries + path: ./binaries/**/* + + get-essential-scenarios: + name: Get parameters + uses: DataDog/system-tests/.github/workflows/compute-workflow-parameters.yml@main + with: + library: nodejs + scenarios_groups: essentials + + system-tests: + runs-on: ${{ contains(fromJSON('["CROSSED_TRACING_LIBRARIES", "INTEGRATIONS"]'), matrix.scenario) && 'ubuntu-latest-16-cores' || 'ubuntu-latest' }} + needs: + - get-essential-scenarios strategy: matrix: - include: - - weblog-variant: express4 - - weblog-variant: express4-typescript - - weblog-variant: nextjs + weblog-variant: ${{fromJson(needs.get-essential-scenarios.outputs.endtoend_weblogs)}} + scenario: ${{fromJson(needs.get-essential-scenarios.outputs.endtoend_scenarios)}} + env: TEST_LIBRARY: nodejs WEBLOG_VARIANT: ${{ matrix.weblog-variant }} DD_API_KEY: ${{ secrets.DD_API_KEY }} + SYSTEM_TESTS_AWS_ACCESS_KEY_ID: ${{ secrets.IDM_AWS_ACCESS_KEY_ID }} + SYSTEM_TESTS_AWS_SECRET_ACCESS_KEY: ${{ secrets.IDM_AWS_SECRET_ACCESS_KEY }} steps: - name: Checkout system tests - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: 'DataDog/system-tests' - - name: Checkout dd-trace-js - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: path: 'binaries/dd-trace-js' - - - name: Build weblog - run: ./build.sh -i weblog - - name: Build runner uses: ./.github/actions/install_runner - + - name: Pull images + uses: ./.github/actions/pull_images + with: + library: nodejs + weblog: ${{ matrix.weblog-variant }} + scenarios: '["${{ matrix.scenario }}"]' + cleanup: false + - name: Build weblog + run: ./build.sh -i weblog - name: Build agent + id: build-agent run: ./build.sh -i agent - - - name: Run - run: ./run.sh TRACER_ESSENTIAL_SCENARIOS - + - name: Run scenario ${{ matrix.scenario }} + run: ./run.sh ${{ matrix.scenario }} - name: Compress artifact if: ${{ always() }} run: tar -czvf artifact.tar.gz $(ls | grep logs) - - name: Upload artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 if: ${{ always() }} with: - name: logs_express-poc + name: logs_${{ matrix.weblog-variant }}-${{ matrix.scenario }} path: artifact.tar.gz parametric: - runs-on: ubuntu-latest - env: - TEST_LIBRARY: nodejs - NODEJS_DDTRACE_MODULE: datadog/dd-trace-js#${{ github.sha }} - steps: - - name: Checkout system tests - uses: actions/checkout@v3 - with: - repository: 'DataDog/system-tests' - - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - name: Build - run: ./build.sh -i runner - - name: Run - run: ./run.sh PARAMETRIC - - name: Compress artifact - if: ${{ always() }} - run: tar -czvf artifact.tar.gz $(ls | grep logs) - - name: Upload artifact - uses: actions/upload-artifact@v3 - if: ${{ always() }} - with: - name: logs_parametric - path: artifact.tar.gz + needs: + - build-artifacts + uses: DataDog/system-tests/.github/workflows/run-parametric.yml@main + secrets: inherit + with: + library: nodejs + binaries_artifact: system_tests_binaries + _experimental_job_count: 8 + _experimental_job_matrix: '[1,2,3,4,5,6,7,8]' diff --git a/.github/workflows/test-k8s-lib-injection.yaml b/.github/workflows/test-k8s-lib-injection.yaml deleted file mode 100644 index d489708c06a..00000000000 --- a/.github/workflows/test-k8s-lib-injection.yaml +++ /dev/null @@ -1,74 +0,0 @@ -name: "Lib Injection Test" - -on: - pull_request: - push: - branches: [master] - schedule: - - cron: '0 4 * * *' - -jobs: - - build-and-publish-init-image: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f # 2.3.4 - - - name: Log in to the Container registry - uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set Docker Image Tag - id: set_names - run: | - echo "Docker image tag: $(echo ${GITHUB_HEAD_REF-${GITHUB_REF#refs/heads/}} | tr / -)" - echo "::set-output name=image_name::$(echo ${GITHUB_HEAD_REF-${GITHUB_REF#refs/heads/}} | tr / -)" - - - name: Npm pack for injection image - run: | - npm pack - - - uses: ./.github/actions/injection - with: - init-image-version: ${GITHUB_SHA} - - - name: Push snapshot image - run: | - docker buildx build --platform=linux/amd64,linux/arm/v7,linux/arm64/v8 -t ghcr.io/datadog/dd-trace-js/dd-lib-js-init:latest_snapshot --push lib-injection - if: ${{ steps.set_names.outputs.image_name }} == 'master' - - lib-injection-tests: - needs: - - build-and-publish-init-image - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - strategy: - matrix: - lib-injection-connection: [ 'network','uds'] - lib-injection-use-admission-controller: ['', 'use-admission-controller'] - weblog-variant: [ 'sample-app'] - fail-fast: false - env: - TEST_LIBRARY: nodejs - WEBLOG_VARIANT: ${{ matrix.weblog-variant }} - LIBRARY_INJECTION_CONNECTION: ${{ matrix.lib-injection-connection }} - LIBRARY_INJECTION_ADMISSION_CONTROLLER: ${{ matrix.lib-injection-use-admission-controller }} - DOCKER_REGISTRY_IMAGES_PATH: ghcr.io/datadog - DOCKER_IMAGE_TAG: ${{ github.sha }} - BUILDX_PLATFORMS: linux/amd64 - MODE: manual - steps: - - name: lib-injection test runner - id: lib-injection-test-runner - uses: DataDog/system-tests/lib-injection/runner@main - with: - docker-registry: ghcr.io - docker-registry-username: ${{ github.repository_owner }} - docker-registry-password: ${{ secrets.GITHUB_TOKEN }} - test-script: ./lib-injection/run-manual-lib-injection.sh diff --git a/.github/workflows/tracing.yml b/.github/workflows/tracing.yml index 1b580a24aa3..7ffcbe59dea 100644 --- a/.github/workflows/tracing.yml +++ b/.github/workflows/tracing.yml @@ -15,33 +15,33 @@ jobs: macos: runs-on: macos-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - - run: yarn install + - uses: ./.github/actions/install - run: yarn test:trace:core:ci - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 ubuntu: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - - run: yarn install - - uses: ./.github/actions/node/16 - - run: yarn test:trace:core:ci + - uses: ./.github/actions/install - uses: ./.github/actions/node/18 - run: yarn test:trace:core:ci - uses: ./.github/actions/node/20 - run: yarn test:trace:core:ci - uses: ./.github/actions/node/latest - run: yarn test:trace:core:ci - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 windows: runs-on: windows-latest steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/node/setup - - run: yarn install + - uses: actions/checkout@v4 + - uses: actions/setup-node@v3 + with: + node-version: '18' + - uses: ./.github/actions/install - run: yarn test:trace:core:ci - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 diff --git a/.gitignore b/.gitignore index ff2cfaa8e23..a8dcafe063b 100644 --- a/.gitignore +++ b/.gitignore @@ -125,3 +125,4 @@ packages/dd-trace/test/appsec/next/*/package.json packages/dd-trace/test/appsec/next/*/node_modules packages/dd-trace/test/appsec/next/*/yarn.lock !packages/dd-trace/**/telemetry/logs +packages/datadog-plugin-azure-functions/test/integration-test/fixtures/node_modules diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0fd16dae1cb..87d896df458 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,93 +1,33 @@ stages: - - package - - deploy + - shared-pipeline - benchmarks - benchmarks-pr-comment + - single-step-instrumentation-tests + - macrobenchmarks include: - - remote: https://gitlab-templates.ddbuild.io/apm/packaging.yml + - remote: https://gitlab-templates.ddbuild.io/libdatadog/include/one-pipeline.yml - local: ".gitlab/benchmarks.yml" + - local: ".gitlab/macrobenchmarks.yml" variables: - JS_PACKAGE_VERSION: - description: "Version to build for .deb and .rpm. Must be already published in NPM" + # dd-trace-js has some exceptions to the default names + AGENT_REPO_PRODUCT_NAME: auto_inject-node + SYSTEM_TESTS_LIBRARY: nodejs -.common: &common - tags: [ "runner:main", "size:large" ] +onboarding_tests_installer: + parallel: + matrix: + - ONBOARDING_FILTER_WEBLOG: [test-app-nodejs,test-app-nodejs-container] + SCENARIO: [ INSTALLER_AUTO_INJECTION, SIMPLE_AUTO_INJECTION_PROFILING ] -package: - extends: .package - rules: - - if: $JS_PACKAGE_VERSION - when: on_success - - if: '$CI_COMMIT_TAG =~ /^v.*/' - when: on_success - script: - - ../.gitlab/build-deb-rpm.sh - -package-arm: - extends: .package-arm - rules: - - if: $JS_PACKAGE_VERSION - when: on_success - - if: '$CI_COMMIT_TAG =~ /^v.*/' - when: on_success - script: - - ../.gitlab/build-deb-rpm.sh - -.release-package: - stage: deploy - variables: - PRODUCT_NAME: auto_inject-node - PACKAGE_FILTER: js # product name is "node" but package name ends "js" - -deploy_to_reliability_env: - stage: deploy - rules: - - if: $CI_PIPELINE_SOURCE == "schedule" - when: on_success - - when: manual - allow_failure: true - trigger: - project: DataDog/apm-reliability/datadog-reliability-env - variables: - UPSTREAM_BRANCH: $CI_COMMIT_REF_NAME - UPSTREAM_PROJECT_ID: $CI_PROJECT_ID - UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME - UPSTREAM_COMMIT_SHA: $CI_COMMIT_SHA - -deploy_to_docker_registries: - stage: deploy - rules: - - if: '$CI_COMMIT_TAG =~ /^v.*/ || $CI_COMMIT_TAG == "dev"' - when: on_success - - when: manual - allow_failure: true - trigger: - project: DataDog/public-images - branch: main - strategy: depend +onboarding_tests_k8s_injection: variables: - IMG_SOURCES: ghcr.io/datadog/dd-trace-js/dd-lib-js-init:$CI_COMMIT_TAG - IMG_DESTINATIONS: dd-lib-js-init:$CI_COMMIT_TAG - IMG_SIGNING: "false" - RETRY_COUNT: 5 - RETRY_DELAY: 300 + WEBLOG_VARIANT: sample-app -deploy_latest_to_docker_registries: - stage: deploy +requirements_json_test: rules: - - if: '$CI_COMMIT_TAG =~ /^v.*/' - when: on_success - - when: manual - allow_failure: true - trigger: - project: DataDog/public-images - branch: main - strategy: depend + - when: on_success variables: - IMG_SOURCES: ghcr.io/datadog/dd-trace-js/dd-lib-js-init:$CI_COMMIT_TAG - IMG_DESTINATIONS: dd-lib-js-init:latest - IMG_SIGNING: "false" - RETRY_COUNT: 5 - RETRY_DELAY: 300 + REQUIREMENTS_BLOCK_JSON_PATH: ".gitlab/requirements_block.json" + REQUIREMENTS_ALLOW_JSON_PATH: ".gitlab/requirements_allow.json" diff --git a/.gitlab/benchmarks.yml b/.gitlab/benchmarks.yml index 864d76cb877..57eba976441 100644 --- a/.gitlab/benchmarks.yml +++ b/.gitlab/benchmarks.yml @@ -1,5 +1,6 @@ variables: - BASE_CI_IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/benchmarking-platform:dd-trace-js + MICROBENCHMARKS_CI_IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/benchmarking-platform:dd-trace-js + SLS_CI_IMAGE: registry.ddbuild.io/ci/serverless-tools:1 # Benchmark's env variables. Modify to tweak benchmark parameters. UNCONFIDENCE_THRESHOLD: "5.0" @@ -7,9 +8,10 @@ variables: .benchmarks: stage: benchmarks + needs: [ ] when: on_success tags: ["runner:apm-k8s-tweaked-metal"] - image: $BASE_CI_IMAGE + image: $MICROBENCHMARKS_CI_IMAGE interruptible: true timeout: 15m script: @@ -28,9 +30,10 @@ variables: benchmarks-pr-comment: stage: benchmarks-pr-comment + needs: [ benchmark, benchmark-serverless ] when: on_success tags: ["arch:amd64"] - image: $BASE_CI_IMAGE + image: $MICROBENCHMARKS_CI_IMAGE script: - cd platform && (git init && git remote add origin https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform && git pull origin dd-trace-js) - bp-runner bp-runner.pr-comment.yml --debug @@ -41,9 +44,10 @@ benchmarks-pr-comment: check-big-regressions: stage: benchmarks-pr-comment + needs: [ benchmark, benchmark-serverless ] when: on_success tags: ["arch:amd64"] - image: $BASE_CI_IMAGE + image: $MICROBENCHMARKS_CI_IMAGE script: - cd platform && (git init && git remote add origin https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform && git pull origin dd-trace-js) - bp-runner bp-runner.fail-on-regression.yml --debug @@ -55,12 +59,6 @@ benchmark: extends: .benchmarks parallel: matrix: - - MAJOR_VERSION: 16 - GROUP: 1 - - MAJOR_VERSION: 16 - GROUP: 2 - - MAJOR_VERSION: 16 - GROUP: 3 - MAJOR_VERSION: 18 GROUP: 1 - MAJOR_VERSION: 18 @@ -69,3 +67,32 @@ benchmark: GROUP: 3 variables: SPLITS: 3 + +benchmark-serverless: + stage: benchmarks + image: $SLS_CI_IMAGE + tags: ["arch:amd64"] + when: on_success + needs: + - benchmark-serverless-trigger + script: + - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/serverless-tools.git ./serverless-tools && cd ./serverless-tools + - ./ci/check_trigger_status.sh + +benchmark-serverless-trigger: + stage: benchmarks + needs: [] + trigger: + project: DataDog/serverless-tools + strategy: depend + allow_failure: true + variables: + UPSTREAM_PIPELINE_ID: $CI_PIPELINE_ID + UPSTREAM_PROJECT_URL: $CI_PROJECT_URL + UPSTREAM_COMMIT_BRANCH: $CI_COMMIT_BRANCH + UPSTREAM_COMMIT_AUTHOR: $CI_COMMIT_AUTHOR + UPSTREAM_COMMIT_TITLE: $CI_COMMIT_TITLE + UPSTREAM_COMMIT_TAG: $CI_COMMIT_TAG + UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME + UPSTREAM_GITLAB_USER_LOGIN: $GITLAB_USER_LOGIN + UPSTREAM_GITLAB_USER_EMAIL: $GITLAB_USER_EMAIL diff --git a/.gitlab/build-deb-rpm.sh b/.gitlab/build-deb-rpm.sh deleted file mode 100755 index a2e30755814..00000000000 --- a/.gitlab/build-deb-rpm.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -if [ -n "$CI_COMMIT_TAG" ] && [ -z "$JS_PACKAGE_VERSION" ]; then - JS_PACKAGE_VERSION=${CI_COMMIT_TAG##v} -fi - -echo -n $JS_PACKAGE_VERSION > auto_inject-node.version - -source common_build_functions.sh - -# Extract package to a dir to make changes -fpm --input-type npm \ - --npm-package-name-prefix "" \ - --output-type dir --prefix "" \ - --version "$JS_PACKAGE_VERSION" \ - --verbose \ - --name dd-trace dd-trace - -cp auto_inject-node.version dd-trace.dir/lib/version - -# Build packages -fpm_wrapper "datadog-apm-library-js" "$JS_PACKAGE_VERSION" \ - --input-type dir \ - --url "https://github.com/DataDog/dd-trace-js" \ - --description "Datadog APM client library for Javascript" \ - --license "BSD-3-Clause" \ - --chdir=dd-trace.dir/lib \ - --prefix "$LIBRARIES_INSTALL_BASE/nodejs" \ - .=. diff --git a/.gitlab/macrobenchmarks.yml b/.gitlab/macrobenchmarks.yml new file mode 100644 index 00000000000..4392babe28b --- /dev/null +++ b/.gitlab/macrobenchmarks.yml @@ -0,0 +1,67 @@ +.macrobenchmarks: + stage: macrobenchmarks + rules: + - if: ($NIGHTLY_BENCHMARKS || $CI_PIPELINE_SOURCE != "schedule") && $CI_COMMIT_REF_NAME == "master" + when: always + - when: manual + tags: ["runner:apm-k8s-same-cpu"] + needs: [] + interruptible: true + timeout: 1h + image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/benchmarking-platform:js-hapi + script: + - git clone --branch js/hapi https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform platform && cd platform + - bp-runner bp-runner.yml --debug -t + artifacts: + name: "artifacts" + when: always + paths: + - platform/artifacts/ + expire_in: 3 months + variables: + FF_USE_LEGACY_KUBERNETES_EXECUTION_STRATEGY: "true" + + K6_OPTIONS_WARMUP_RATE: 500 + K6_OPTIONS_WARMUP_DURATION: 1m + K6_OPTIONS_WARMUP_GRACEFUL_STOP: 10s + K6_OPTIONS_WARMUP_PRE_ALLOCATED_VUS: 4 + K6_OPTIONS_WARMUP_MAX_VUS: 4 + + K6_OPTIONS_NORMAL_OPERATION_RATE: 300 + K6_OPTIONS_NORMAL_OPERATION_DURATION: 10m + K6_OPTIONS_NORMAL_OPERATION_GRACEFUL_STOP: 10s + K6_OPTIONS_NORMAL_OPERATION_PRE_ALLOCATED_VUS: 4 + K6_OPTIONS_NORMAL_OPERATION_MAX_VUS: 4 + + K6_OPTIONS_HIGH_LOAD_RATE: 700 + K6_OPTIONS_HIGH_LOAD_DURATION: 3m + K6_OPTIONS_HIGH_LOAD_GRACEFUL_STOP: 10s + K6_OPTIONS_HIGH_LOAD_PRE_ALLOCATED_VUS: 4 + K6_OPTIONS_HIGH_LOAD_MAX_VUS: 4 + + DDTRACE_INSTALL_VERSION: "git://github.com/Datadog/dd-trace-js.git#${CI_COMMIT_SHA}" + + # Workaround: Currently we're not running the benchmarks on every PR, but GitHub still shows them as pending. + # By marking the benchmarks as allow_failure, this should go away. (This workaround should be removed once the + # benchmarks get changed to run on every PR) + allow_failure: true + + # Retry on Gitlab internal system failures + retry: + max: 2 + when: + - unknown_failure + - data_integrity_failure + - runner_system_failure + - scheduler_failure + - api_failure + +baseline: + extends: .macrobenchmarks + variables: + DD_BENCHMARKS_CONFIGURATION: baseline + +only-tracing: + extends: .macrobenchmarks + variables: + DD_BENCHMARKS_CONFIGURATION: only-tracing diff --git a/.gitlab/prepare-oci-package.sh b/.gitlab/prepare-oci-package.sh new file mode 100755 index 00000000000..af579f04355 --- /dev/null +++ b/.gitlab/prepare-oci-package.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +set -e + +cd .. + +npm pack + +mkdir -p packaging/sources + +npm install --prefix ./packaging/sources/ dd-trace-*.tgz + +rm packaging/sources/*.json # package.json and package-lock.json are unneeded + +if [ -n "$CI_COMMIT_TAG" ] && [ -z "$JS_PACKAGE_VERSION" ]; then + JS_PACKAGE_VERSION=${CI_COMMIT_TAG##v} +elif [ -z "$CI_COMMIT_TAG" ] && [ -z "$JS_PACKAGE_VERSION" ]; then + JS_PACKAGE_VERSION="$(jq --raw-output '.version' package.json)${CI_VERSION_SUFFIX}" +fi + +echo -n $JS_PACKAGE_VERSION > packaging/sources/version + +cd packaging + +cp ../requirements.json sources/requirements.json diff --git a/.gitlab/requirements_allow.json b/.gitlab/requirements_allow.json new file mode 100644 index 00000000000..e832f6e7132 --- /dev/null +++ b/.gitlab/requirements_allow.json @@ -0,0 +1,19 @@ +[ + {"name": "min glibc x64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:2.17"}}, + {"name": "ok glibc x64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:2.23"}}, + {"name": "high glibc x64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:3.0"}}, + {"name": "musl x64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "musl:1.2.2"}}, + {"name": "min glibc arm64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "arm64", "libc": "glibc:2.17"}}, + {"name": "ok glibc arm64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "arm64", "libc": "glibc:2.27"}}, + {"name": "glibc x86","filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x86", "libc": "glibc:2.19"}}, + {"name": "musl arm","filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "arm", "libc": "musl:1.2.2"}}, + {"name": "musl arm64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "arm64", "libc": "musl:1.2.2"}}, + {"name": "musl x64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "musl:1.2.2"}}, + {"name": "musl x86", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x86", "libc": "musl:1.2.2"}}, + {"name": "windows x64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "windows", "arch": "x64"}}, + {"name": "windows x86", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "windows", "arch": "x86"}}, + {"name": "macos x64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "darwin", "arch": "x64"}}, + {"name": "macos arm64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "darwin", "arch": "arm64"}}, + {"name": "node app", "filepath": "/pathto/node", "args": ["/pathto/node", "./app.js"], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:2.40"}}, + {"name": "ts-node app", "filepath": "/pathto/ts-node", "args": ["/pathto/ts-node", "./app.js"], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:2.40"}} +] diff --git a/.gitlab/requirements_block.json b/.gitlab/requirements_block.json new file mode 100644 index 00000000000..e728f802915 --- /dev/null +++ b/.gitlab/requirements_block.json @@ -0,0 +1,11 @@ +[ + {"name": "unsupported 2.x glibc x64","filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:2.16"}}, + {"name": "unsupported 1.x glibc x64","filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:1.22"}}, + {"name": "unsupported 2.x.x glibc x64","filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:2.16.9"}}, + {"name": "unsupported 2.x glibc arm64","filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "arm64", "libc": "glibc:2.16"}}, + {"name": "unsupported 2.x.x glibc x64","filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "arm64", "libc": "glibc:2.16.9"}}, + {"name": "unsupported 2.x.x glibc x86","filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x86", "libc": "glibc:2.17"}}, + {"name": "npm","filepath": "/pathto/node", "args": ["/pathto/node", "/pathto/npm-cli.js"], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:2.40"}}, + {"name": "yarn","filepath": "/pathto/node", "args": ["/pathto/node", "/pathto/yarn.js"], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:2.40"}}, + {"name": "pnpm","filepath": "/pathto/node", "args": ["/pathto/node", "/pathto/pnpm.cjs"], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:2.40"}} +] diff --git a/.mocharc.yml b/.mocharc.yml new file mode 100644 index 00000000000..7646acebf54 --- /dev/null +++ b/.mocharc.yml @@ -0,0 +1,3 @@ +color: true +exit: true # TODO: Fix tests so that this is not needed. +timeout: '5000' diff --git a/.npmignore b/.npmignore index af16b4ac73a..6955a334148 100644 --- a/.npmignore +++ b/.npmignore @@ -3,18 +3,18 @@ !packages/*/lib/**/* !packages/*/src/**/* !packages/*/index.js -!scripts/**/* +!scripts/preinstall.js !vendor/**/* -!CONTRIBUTING.md !LICENSE !LICENSE-3rdparty.csv -!MIGRATING.md !README.md !index.d.ts !index.js !esbuild.js !init.js +!initialize.mjs !loader-hook.mjs +!register.js !package.json !cypress/**/* !ci/**/* diff --git a/.nvmrc b/.nvmrc index b6a7d89c68e..3c032078a4a 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -16 +18 diff --git a/.vscode/launch.json b/.vscode/launch.json index a5d0c61d976..3df35f8cbc1 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -8,15 +8,11 @@ "type": "node", "request": "launch", "name": "Test Current File", - "runtimeExecutable": "yarn", - "runtimeArgs": [ - "tdd", - "${file}", - "--inspect-brk=9229" + "skipFiles": [ + "/**" ], - "port": 9229, - "console": "integratedTerminal", - "internalConsoleOptions": "neverOpen" - }, + "program": "${file}", + "console": "integratedTerminal" + } ] } diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 2ca4a2c7b59..00000000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "debug.node.autoAttach": "on" -} diff --git a/CODEOWNERS b/CODEOWNERS index 6287f0489aa..da66c3557b0 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -3,10 +3,19 @@ /packages/dd-trace/src/appsec/ @DataDog/asm-js /packages/dd-trace/test/appsec/ @DataDog/asm-js -/packages/datadog-plugin-*/ @Datadog/dd-trace-js @Datadog/apm-framework-integrations-reviewers-js -/packages/datadog-instrumentations/ @Datadog/dd-trace-js @Datadog/apm-framework-integrations-reviewers-js -/packages/ddtrace/src/plugins/ @DataDog/dd-trace-js @Datadog/apm-framework-integrations-reviewers-js -/packages/ddtrace/test/plugins/ @DataDog/dd-trace-js @Datadog/apm-framework-integrations-reviewers-js +/integration-tests/debugger/ @DataDog/dd-trace-js @DataDog/debugger +/packages/datadog-code-origin/ @DataDog/dd-trace-js @DataDog/debugger +/packages/datadog-plugin-*/**/code_origin.* @DataDog/dd-trace-js @DataDog/debugger +/packages/dd-trace/src/debugger/ @DataDog/dd-trace-js @DataDog/debugger +/packages/dd-trace/test/debugger/ @DataDog/dd-trace-js @DataDog/debugger + +/packages/dd-trace/src/lambda/ @DataDog/serverless-aws +/packages/dd-trace/test/lambda/ @DataDog/serverless-aws + +/packages/datadog-plugin-*/ @Datadog/dd-trace-js @Datadog/apm-idm-js +/packages/datadog-instrumentations/ @Datadog/dd-trace-js @Datadog/apm-idm-js +/packages/ddtrace/src/plugins/ @DataDog/dd-trace-js @Datadog/apm-idm-js +/packages/ddtrace/test/plugins/ @DataDog/dd-trace-js @Datadog/apm-idm-js /packages/dd-trace/src/ci-visibility/ @DataDog/ci-app-libraries /packages/datadog-plugin-jest/ @DataDog/ci-app-libraries @@ -14,29 +23,42 @@ /packages/datadog-plugin-cucumber/ @DataDog/ci-app-libraries /packages/datadog-plugin-cypress/ @DataDog/ci-app-libraries /packages/datadog-plugin-playwright/ @DataDog/ci-app-libraries +/packages/datadog-plugin-vitest/ @DataDog/ci-app-libraries +/packages/dd-trace/src/plugins/util/git.js @DataDog/ci-app-libraries +/packages/dd-trace/src/plugins/ci_plugin.js @DataDog/ci-app-libraries +/packages/dd-trace/test/ci-visibility/ @DataDog/ci-app-libraries +/packages/dd-trace/test/plugins/util/git.spec.js @DataDog/ci-app-libraries +/packages/dd-trace/test/plugins/util/ci-env/ @DataDog/ci-app-libraries /packages/datadog-instrumentations/src/jest.js @DataDog/ci-app-libraries -/packages/datadog-instrumentations/src/mocha.js @DataDog/ci-app-libraries +/packages/datadog-instrumentations/src/mocha/ @DataDog/ci-app-libraries /packages/datadog-instrumentations/src/cucumber.js @DataDog/ci-app-libraries /packages/datadog-instrumentations/src/cypress.js @DataDog/ci-app-libraries /packages/datadog-instrumentations/src/playwright.js @DataDog/ci-app-libraries +/packages/datadog-instrumentations/src/vitest.js @DataDog/ci-app-libraries /integration-tests/ci-visibility/ @DataDog/ci-app-libraries /integration-tests/cucumber/ @DataDog/ci-app-libraries /integration-tests/cypress/ @DataDog/ci-app-libraries /integration-tests/playwright/ @DataDog/ci-app-libraries -/integration-tests/ci-visibility-spec.js @DataDog/ci-app-libraries +/integration-tests/jest/jest.spec.js @DataDog/ci-app-libraries +/integration-tests/mocha/mocha.spec.js @DataDog/ci-app-libraries +/integration-tests/playwright/playwright.spec.js @DataDog/ci-app-libraries +/integration-tests/cucumber/cucumber.spec.js @DataDog/ci-app-libraries +/integration-tests/cypress/cypress.spec.js @DataDog/ci-app-libraries +/integration-tests/vitest/vitest.spec.js @DataDog/ci-app-libraries +/integration-tests/vitest.config.mjs @DataDog/ci-app-libraries /integration-tests/test-api-manual.spec.js @DataDog/ci-app-libraries -/packages/dd-trace/src/service-naming/ @Datadog/apm-framework-integrations-reviewers-js -/packages/dd-trace/test/service-naming/ @Datadog/apm-framework-integrations-reviewers-js +/packages/dd-trace/src/service-naming/ @Datadog/apm-idm-js +/packages/dd-trace/test/service-naming/ @Datadog/apm-idm-js # CI /.github/workflows/appsec.yml @DataDog/asm-js /.github/workflows/ci-visibility-performance.yml @DataDog/ci-app-libraries /.github/workflows/codeql-analysis.yml @DataDog/dd-trace-js /.github/workflows/core.yml @DataDog/dd-trace-js -/.github/workflows/lambda.yml @DataDog/serverless-apm +/.github/workflows/lambda.yml @DataDog/serverless-aws /.github/workflows/package-size.yml @DataDog/dd-trace-js /.github/workflows/plugins.yml @DataDog/dd-trace-js /.github/workflows/pr-labels.yml @DataDog/dd-trace-js @@ -46,9 +68,8 @@ /.github/workflows/release-dev.yml @DataDog/dd-trace-js /.github/workflows/release-latest.yml @DataDog/dd-trace-js /.github/workflows/release-proposal.yml @DataDog/dd-trace-js -/.github/workflows/serverless-integration-test.yml @DataDog/serverless -/.github/workflows/serverless-performance.yml @DataDog/serverless-apm @DataDog/serverless -/.github/workflows/system-tests.yml @DataDog/asm-js +/.github/workflows/serverless-integration-test.yml @DataDog/serverless-aws @DataDog/serverless +/.github/workflows/system-tests.yml @DataDog/dd-trace-js @DataDog/asm-js /.github/workflows/test-k8s-lib-injection.yaml @DataDog/dd-trace-js /.github/workflows/tracing.yml @DataDog/dd-trace-js /.gitlab/benchmarks.yml @DataDog/dd-trace-js diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fc76e7a8fb3..ea960983105 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -70,4 +70,96 @@ We follow an all-green policy which means that for any PR to be merged _all_ tes Eventually we plan to look into putting these permission-required tests behind a label which team members can add to their PRs at creation to run the full CI and can add to outside contributor PRs to trigger the CI from their own user credentials. If the label is not present there will be another action which checks the label is present. Rather than showing a bunch of confusing failures to new contributors it would just show a single job failure which indicates an additional label is required, and we can name it in a way that makes it clear that it's not the responsibility of the outside contributor to add it. Something like `approve-full-ci` is one possible choice there. +## Development Requirements + +Since this project supports multiple Node versions, using a version +manager such as [nvm](https://github.com/creationix/nvm) is recommended. + +We use [yarn](https://yarnpkg.com/) for its workspace functionality, so make sure to install that as well. + +To install dependencies once you have Node and yarn installed, run: + +```sh +$ yarn +``` + + +## Testing + +### Prerequisites + +The `pg-native` package requires `pg_config` to be in your `$PATH` to be able to install. +Please refer to [the "Install" section](https://github.com/brianc/node-postgres/tree/master/packages/pg-native#install) of the `pg-native` documentation for how to ensure your environment is configured correctly. + +### Setup + +Before running _plugin_ tests, the data stores need to be running. +The easiest way to start all of them is to use the provided +docker-compose configuration: + +```sh +$ docker-compose up -d -V --remove-orphans --force-recreate +$ yarn services +``` + +> **Note** +> The `aerospike`, `couchbase`, `grpc` and `oracledb` instrumentations rely on +> native modules that do not compile on ARM64 devices (for example M1/M2 Mac) +> - their tests cannot be run locally on these devices. + +### Unit Tests + +There are several types of unit tests, for various types of components. The +following commands may be useful: + +```sh +# Tracer core tests (i.e. testing `packages/dd-trace`) +$ yarn test:trace:core +# "Core" library tests (i.e. testing `packages/datadog-core` +$ yarn test:core +# Instrumentations tests (i.e. testing `packages/datadog-instrumentations` +$ yarn test:instrumentations +``` + +Several other components have test commands as well. See `package.json` for +details. + +To test _plugins_ (i.e. components in `packages/datadog-plugin-XXXX` +directories, set the `PLUGINS` environment variable to the plugin you're +interested in, and use `yarn test:plugins`. If you need to test multiple +plugins you may separate then with a pipe (`|`) delimiter. Here's an +example testing the `express` and `bluebird` plugins: + +```sh +PLUGINS="express|bluebird" yarn test:plugins +``` + + +### Linting + +We use [ESLint](https://eslint.org) to make sure that new code +conforms to our coding standards. + +To run the linter, use: + +```sh +$ yarn lint +``` + + +### Benchmarks + +Our microbenchmarks live in `benchmark/sirun`. Each directory in there +corresponds to a specific benchmark test and its variants, which are used to +track regressions and improvements over time. + +In addition to those, when two or more approaches must be compared, please write +a benchmark in the `benchmark/index.js` module so that we can keep track of the +most efficient algorithm. To run your benchmark, use: + +```sh +$ yarn bench +``` + + [1]: https://docs.datadoghq.com/help diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 15404050720..0ce2aba174a 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -12,26 +12,24 @@ require,dc-polyfill,MIT,Copyright 2023 Datadog Inc. require,ignore,MIT,Copyright 2013 Kael Zhang and contributors require,import-in-the-middle,Apache license 2.0,Copyright 2021 Datadog Inc. require,int64-buffer,MIT,Copyright 2015-2016 Yusuke Kawasaki -require,ipaddr.js,MIT,Copyright 2011-2017 whitequark require,istanbul-lib-coverage,BSD-3-Clause,Copyright 2012-2015 Yahoo! Inc. require,jest-docblock,MIT,Copyright Meta Platforms, Inc. and affiliates. require,koalas,MIT,Copyright 2013-2017 Brian Woodward require,limiter,MIT,Copyright 2011 John Hurliman -require,lodash.kebabcase,MIT,Copyright JS Foundation and other contributors -require,lodash.pick,MIT,Copyright JS Foundation and other contributors require,lodash.sortby,MIT,Copyright JS Foundation and other contributors -require,lodash.uniq,MIT,Copyright JS Foundation and other contributors require,lru-cache,ISC,Copyright (c) 2010-2022 Isaac Z. Schlueter and Contributors -require,methods,MIT,Copyright 2013-2014 TJ Holowaychuk require,module-details-from-path,MIT,Copyright 2016 Thomas Watson Steen require,msgpack-lite,MIT,Copyright 2015 Yusuke Kawasaki -require,node-abort-controller,MIT,Copyright (c) 2019 Steve Faulkner require,opentracing,MIT,Copyright 2016 Resonance Labs Inc require,path-to-regexp,MIT,Copyright 2014 Blake Embrey require,pprof-format,MIT,Copyright 2022 Stephen Belanger require,protobufjs,BSD-3-Clause,Copyright 2016 Daniel Wirtz +require,tlhunter-sorted-set,MIT,Copyright (c) 2023 Datadog Inc. require,retry,MIT,Copyright 2011 Tim Koschützki Felix Geisendörfer +require,rfdc,MIT,Copyright 2019 David Mark Clements require,semver,ISC,Copyright Isaac Z. Schlueter and Contributors +require,shell-quote,mit,Copyright (c) 2013 James Halliday +dev,@apollo/server,MIT,Copyright (c) 2016-2020 Apollo Graph, Inc. (Formerly Meteor Development Group, Inc.) dev,@types/node,MIT,Copyright Authors dev,autocannon,MIT,Copyright 2016 Matteo Collina dev,aws-sdk,Apache 2.0,Copyright 2012-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. @@ -49,9 +47,7 @@ dev,eslint-config-standard,MIT,Copyright Feross Aboukhadijeh dev,eslint-plugin-import,MIT,Copyright 2015 Ben Mosher dev,eslint-plugin-mocha,MIT,Copyright 2014 Mathias Schreck dev,eslint-plugin-n,MIT,Copyright 2015 Toru Nagashima -dev,eslint-plugin-node,MIT,Copyright 2015 Toru Nagashima dev,eslint-plugin-promise,ISC,jden and other contributors -dev,eslint-plugin-standard,MIT,Copyright 2015 Jamund Ferguson dev,express,MIT,Copyright 2009-2014 TJ Holowaychuk 2013-2014 Roman Shtylman 2014-2015 Douglas Christopher Wilson dev,get-port,MIT,Copyright Sindre Sorhus dev,glob,ISC,Copyright Isaac Z. Schlueter and Contributors @@ -68,7 +64,7 @@ dev,rimraf,ISC,Copyright Isaac Z. Schlueter and Contributors dev,sinon,BSD-3-Clause,Copyright 2010-2017 Christian Johansen dev,sinon-chai,WTFPL and BSD-2-Clause,Copyright 2004 Sam Hocevar 2012–2017 Domenic Denicola dev,tap,ISC,Copyright 2011-2022 Isaac Z. Schlueter and Contributors -dev,tape,MIT,Copyright James Halliday +dev,tiktoken,MIT,Copyright (c) 2022 OpenAI, Shantanu Jain file,aws-lambda-nodejs-runtime-interface-client,Apache 2.0,Copyright 2019 Amazon.com Inc. or its affiliates. All Rights Reserved. file,profile.proto,Apache license 2.0,Copyright 2016 Google Inc. file,is-git-url,MIT,Copyright (c) 2017 Jon Schlinkert. diff --git a/MIGRATING.md b/MIGRATING.md index 649037d39fa..39160fce6c0 100644 --- a/MIGRATING.md +++ b/MIGRATING.md @@ -4,6 +4,21 @@ This guide describes the steps to upgrade dd-trace from a major version to the next. If you are having any issues related to migrating, please feel free to open an issue or contact our [support](https://www.datadoghq.com/support/) team. +## 4.0 to 5.0 + +### Node 16 is no longer supported + +Node.js 16 has reached EOL in September 2023 and is no longer supported. Generally +speaking, we highly recommend always keeping Node.js up to date regardless of +our support policy. + +### Update `trace` TypeScript declaration + +The TypeScript declaration for `trace` has been updated to enforce +that calls to `tracer.trace(name, fn)` must receive a function which takes at least +the span object. Previously the span was technically optional when it should not have +been as the span must be handled. + ## 3.0 to 4.0 ### Node 14 is no longer supported diff --git a/README.md b/README.md index 8a3d9372125..3a7224b8d44 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # `dd-trace`: Node.js APM Tracer Library -[![npm v4](https://img.shields.io/npm/v/dd-trace/latest?color=blue&label=dd-trace%40v4&logo=npm)](https://www.npmjs.com/package/dd-trace) -[![npm v3](https://img.shields.io/npm/v/dd-trace/latest-node14?color=blue&label=dd-trace%40v3&logo=npm)](https://www.npmjs.com/package/dd-trace/v/latest-node12) +[![npm v5](https://img.shields.io/npm/v/dd-trace/latest?color=blue&label=dd-trace%40v5&logo=npm)](https://www.npmjs.com/package/dd-trace) +[![npm v4](https://img.shields.io/npm/v/dd-trace/latest-node16?color=blue&label=dd-trace%40v4&logo=npm)](https://www.npmjs.com/package/dd-trace/v/latest-node16) [![codecov](https://codecov.io/gh/DataDog/dd-trace-js/branch/master/graph/badge.svg)](https://codecov.io/gh/DataDog/dd-trace-js) Bits the dog JavaScript @@ -27,25 +27,26 @@ Most of the documentation for `dd-trace` is available on these webpages: | :---: | :---: | :---: | :---: | :---: | :---: | | [`v1`](https://github.com/DataDog/dd-trace-js/tree/v1.x) | ![npm v1](https://img.shields.io/npm/v/dd-trace/legacy-v1?color=white&label=%20&style=flat-square) | `>= v12` | **End of Life** | 2021-07-13 | 2022-02-25 | | [`v2`](https://github.com/DataDog/dd-trace-js/tree/v2.x) | ![npm v2](https://img.shields.io/npm/v/dd-trace/latest-node12?color=white&label=%20&style=flat-square) | `>= v12` | **End of Life** | 2022-01-28 | 2023-08-15 | -| [`v3`](https://github.com/DataDog/dd-trace-js/tree/v3.x) | ![npm v3](https://img.shields.io/npm/v/dd-trace/latest-node14?color=white&label=%20&style=flat-square) | `>= v14` | **Maintenance** | 2022-08-15 | 2024-05-15 | -| [`v4`](https://github.com/DataDog/dd-trace-js/tree/v4.x) | ![npm v4](https://img.shields.io/npm/v/dd-trace/latest?color=white&label=%20&style=flat-square) | `>= v16` | **Current** | 2023-05-12 | Unknown | +| [`v3`](https://github.com/DataDog/dd-trace-js/tree/v3.x) | ![npm v3](https://img.shields.io/npm/v/dd-trace/latest-node14?color=white&label=%20&style=flat-square) | `>= v14` | **End of Life** | 2022-08-15 | 2024-05-15 | +| [`v4`](https://github.com/DataDog/dd-trace-js/tree/v4.x) | ![npm v4](https://img.shields.io/npm/v/dd-trace/latest-node16?color=white&label=%20&style=flat-square) | `>= v16` | **Maintenance** | 2023-05-12 | 2025-01-11 | +| [`v5`](https://github.com/DataDog/dd-trace-js/tree/v5.x) | ![npm v5](https://img.shields.io/npm/v/dd-trace/latest?color=white&label=%20&style=flat-square) | `>= v18` | **Current** | 2024-01-11 | Unknown | -We currently maintain two release lines, namely `v3` and `v4`. -Features and bug fixes that are merged are released to the `v4` line and, if appropriate, also the `v3` line. +We currently maintain two release lines, namely `v5`, and `v4`. +Features and bug fixes that are merged are released to the `v5` line and, if appropriate, also `v4`. -For any new projects it is recommended to use the `v4` release line: +For any new projects it is recommended to use the `v5` release line: ```sh $ npm install dd-trace $ yarn add dd-trace ``` -However, existing projects that already use the `v3` release line, or projects that need to support EOL versions of Node.js, may continue to use these release lines. +However, existing projects that already use the `v4` release line, or projects that need to support EOL versions of Node.js, may continue to use these release lines. This is done by specifying the version when installing the package. ```sh -$ npm install dd-trace@3 -$ yarn add dd-trace@3 +$ npm install dd-trace@4 +$ yarn add dd-trace@4 ``` Any backwards-breaking functionality that is introduced into the library will result in an increase of the major version of the library and therefore a new release line. @@ -60,127 +61,27 @@ For more information about library versioning and compatibility, see the [NodeJS Changes associated with each individual release are documented on the [GitHub Releases](https://github.com/DataDog/dd-trace-js/releases) screen. -## Development +## Development and Contribution -Before contributing to this open source project, read our [CONTRIBUTING.md](https://github.com/DataDog/dd-trace-js/blob/master/CONTRIBUTING.md). +Please read the [CONTRIBUTING.md](https://github.com/DataDog/dd-trace-js/blob/master/CONTRIBUTING.md) document before contributing to this open source project. -## Requirements +## EcmaScript Modules (ESM) Support -Since this project supports multiple Node versions, using a version -manager such as [nvm](https://github.com/creationix/nvm) is recommended. +ESM support requires an additional command-line argument. Use the following to enable experimental ESM support with your application: -We use [yarn](https://yarnpkg.com/) for its workspace functionality, so make sure to install that as well. - -To install dependencies once you have Node and yarn installed, run: - -```sh -$ yarn -``` - - -## Testing - -Before running _plugin_ tests, the data stores need to be running. -The easiest way to start all of them is to use the provided -docker-compose configuration: - -```sh -$ docker-compose up -d -V --remove-orphans --force-recreate -$ yarn services -``` - -> **Note** -> The `couchbase`, `grpc` and `oracledb` instrumentations rely on native modules -> that do not compile on ARM64 devices (for example M1/M2 Mac) - their tests -> cannot be run locally on these devices. - -### Unit Tests - -There are several types of unit tests, for various types of components. The -following commands may be useful: - -```sh -# Tracer core tests (i.e. testing `packages/dd-trace`) -$ yarn test:trace:core -# "Core" library tests (i.e. testing `packages/datadog-core` -$ yarn test:core -# Instrumentations tests (i.e. testing `packages/datadog-instrumentations` -$ yarn test:instrumentations -``` - -Several other components have test commands as well. See `package.json` for -details. - -To test _plugins_ (i.e. components in `packages/datadog-plugin-XXXX` -directories, set the `PLUGINS` environment variable to the plugin you're -interested in, and use `yarn test:plugins`. If you need to test multiple -plugins you may separate then with a pipe (`|`) delimiter. Here's an -example testing the `express` and `bluebird` plugins: - -```sh -PLUGINS="express|bluebird" yarn test:plugins -``` - - -### Memory Leaks - -To run the memory leak tests, use: - -```sh -$ yarn leak:core - -# or - -$ yarn leak:plugins -``` - - -### Linting - -We use [ESLint](https://eslint.org) to make sure that new code -conforms to our coding standards. - -To run the linter, use: - -```sh -$ yarn lint -``` - - -### Experimental ESM Support - -> **Warning** -> -> ESM support has been temporarily disabled starting from Node 20 as significant -> changes are in progress. - -ESM support is currently in the experimental stages, while CJS has been supported -since inception. This means that code loaded using `require()` should work fine -but code loaded using `import` might not always work. - -Use the following command to enable experimental ESM support with your application: +Node.js < v20.6 ```sh node --loader dd-trace/loader-hook.mjs entrypoint.js ``` - -### Benchmarks - -Our microbenchmarks live in `benchmark/sirun`. Each directory in there -corresponds to a specific benchmark test and its variants, which are used to -track regressions and improvements over time. - -In addition to those, when two or more approaches must be compared, please write -a benchmark in the `benchmark/index.js` module so that we can keep track of the -most efficient algorithm. To run your benchmark, use: +Node.js >= v20.6 ```sh -$ yarn bench +node --import dd-trace/register.js entrypoint.js ``` - ## Serverless / Lambda Note that there is a separate Lambda project, [datadog-lambda-js](https://github.com/DataDog/datadog-lambda-js), that is responsible for enabling metrics and distributed tracing when your application runs on Lambda. @@ -192,40 +93,16 @@ Regardless of where you open the issue, someone at Datadog will try to help. ## Bundling -Generally, `dd-trace` works by intercepting `require()` calls that a Node.js application makes when loading modules. This includes modules that are built-in to Node.js, like the `fs` module for accessing the filesystem, as well as modules installed from the npm registry, like the `pg` database module. - -Also generally, bundlers work by crawling all of the `require()` calls that an application makes to files on disk, replacing the `require()` calls with custom code, and then concatenating all of the resulting JavaScript into one "bundled" file. When a built-in module is loaded, like `require('fs')`, that call can then remain the same in the resulting bundle. - -Fundamentally APM tools like `dd-trace` stop working at this point. Perhaps they continue to intercept the calls for built-in modules but don't intercept calls to third party libraries. This means that by default when you bundle a `dd-trace` app with a bundler it is likely to capture information about disk access (via `fs`) and outbound HTTP requests (via `http`), but will otherwise omit calls to third party libraries (like extracting incoming request route information for the `express` framework or showing which query is run for the `mysql` database client). +If you would like to trace your bundled application then please read this page on [bundling and dd-trace](https://docs.datadoghq.com/tracing/trace_collection/automatic_instrumentation/dd_libraries/nodejs/#bundling). It includes information on how to use our ESBuild plugin and includes caveats for other bundlers. -To get around this, one can treat all third party modules, or at least third party modules that the APM needs to instrument, as being "external" to the bundler. With this setting the instrumented modules remain on disk and continue to be loaded via `require()` while the non-instrumented modules are bundled. Sadly this results in a build with many extraneous files and starts to defeat the purpose of bundling. -For these reasons it's necessary to have custom-built bundler plugins. Such plugins are able to instruct the bundler on how to behave, injecting intermediary code and otherwise intercepting the "translated" `require()` calls. The result is that many more packages are then included in the bundled JavaScript file. Some applications can have 100% of modules bundled, however native modules still need to remain external to the bundle. - -### ESBuild Support - -This library provides experimental ESBuild support in the form of an ESBuild plugin. Require the `dd-trace/esbuild` module when building your bundle to enable the plugin. - -Here's an example of how one might use `dd-trace` with ESBuild: +## Security Vulnerabilities -```javascript -const ddPlugin = require('dd-trace/esbuild') -const esbuild = require('esbuild') +Please refer to the [SECURITY.md](https://github.com/DataDog/dd-trace-js/blob/master/SECURITY.md) document if you have found a security issue. -esbuild.build({ - entryPoints: ['app.js'], - bundle: true, - outfile: 'out.js', - plugins: [ddPlugin], - platform: 'node', // allows built-in modules to be required - target: ['node16'] -}).catch((err) => { - console.error(err) - process.exit(1) -}) -``` +## Datadog With OpenTelemetery -## Security Vulnerabilities +Please refer to the [Node.js Custom Instrumentation using OpenTelemetry API](https://docs.datadoghq.com/tracing/trace_collection/custom_instrumentation/nodejs/otel/) document. It includes information on how to use the OpenTelemetry API with dd-trace-js. -If you have found a security issue, please contact the security team directly at [security@datadoghq.com](mailto:security@datadoghq.com). +Note that our internal implementation of the OpenTelemetry API is currently set within the version range `>=1.0.0 <1.9.0`. This range will be updated at a regular cadence therefore, we recommend updating your tracer to the latest release to ensure up to date support. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000000..2061bbe0d09 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,17 @@ +# Security Policy + +This document outlines the security policy for the Datadog Node.js Tracer (aka `dd-trace-js`) and what to do if you discover a security vulnerability in the project. +Most notably, please do not share the details in a public forum (such as in a discussion, issue, or pull request) but instead reach out to us with the details. +This gives us an opportunity to release a fix for others to benefit from by the time details are made public. + + +## Supported Versions + +We accept vulnerability submissions for any [currently maintained release lines](https://github.com/DataDog/dd-trace-js#version-release-lines-and-maintenance). + + +## Reporting a Vulnerability + +If you discover a vulnerability in the Datadog Node.js Tracer (or any Datadog product for that matter) please submit details to the following email address: + +* [security@datadoghq.com](mailto:security@datadoghq.com) diff --git a/benchmark/core.js b/benchmark/core.js index 23912ae2fe0..1617f290e70 100644 --- a/benchmark/core.js +++ b/benchmark/core.js @@ -11,7 +11,10 @@ const Writer = proxyquire('../packages/dd-trace/src/exporters/agent/writer', { './request': () => Promise.resolve(), '../../encode/0.4': { AgentEncoder: function () { - return { encode () {} } + return { + encode () {}, + count () {} + } } } }) diff --git a/benchmark/dd-trace.js b/benchmark/dd-trace.js index 41b98f6574e..2254f0d121f 100644 --- a/benchmark/dd-trace.js +++ b/benchmark/dd-trace.js @@ -25,9 +25,9 @@ suite operation = () => { const span = tracer.startSpan('bench') span.addTags({ - 'tag1': str + generateString(10), - 'tag2': str + str + generateString(10), - 'tag3': str + str + str + generateString(10) + tag1: str + generateString(10), + tag2: str + str + generateString(10), + tag3: str + str + str + generateString(10) }) span.finish() } @@ -41,23 +41,23 @@ suite operation = () => { const rootSpan = tracer.startSpan('root') rootSpan.addTags({ - 'tag1': generateString(20), - 'tag2': generateString(20), - 'tag3': generateString(20) + tag1: generateString(20), + tag2: generateString(20), + tag3: generateString(20) }) const parentSpan = tracer.startSpan('parent', { childOf: rootSpan }) parentSpan.addTags({ - 'tag1': generateString(20), - 'tag2': generateString(20), - 'tag3': generateString(20) + tag1: generateString(20), + tag2: generateString(20), + tag3: generateString(20) }) const childSpan = tracer.startSpan('child', { childOf: parentSpan }) childSpan.addTags({ - 'tag1': generateString(20), - 'tag2': generateString(20), - 'tag3': generateString(20) + tag1: generateString(20), + tag2: generateString(20), + tag3: generateString(20) }) childSpan.finish() diff --git a/benchmark/e2e-ci/benchmark-run.js b/benchmark/e2e-ci/benchmark-run.js index c50f0534bd1..8194a1526ef 100644 --- a/benchmark/e2e-ci/benchmark-run.js +++ b/benchmark/e2e-ci/benchmark-run.js @@ -20,8 +20,8 @@ function getBranchUnderTest () { const getCommonHeaders = () => { return { 'Content-Type': 'application/json', - 'authorization': `Bearer ${process.env.ROBOT_CI_GITHUB_PERSONAL_ACCESS_TOKEN}`, - 'Accept': 'application/vnd.github.v3+json', + authorization: `Bearer ${process.env.ROBOT_CI_GITHUB_PERSONAL_ACCESS_TOKEN}`, + Accept: 'application/vnd.github.v3+json', 'user-agent': 'dd-trace benchmark tests' } } diff --git a/benchmark/e2e/benchmark-run.js b/benchmark/e2e/benchmark-run.js index 2bd04e6e6f9..6ff18678171 100644 --- a/benchmark/e2e/benchmark-run.js +++ b/benchmark/e2e/benchmark-run.js @@ -158,7 +158,7 @@ function pad (str, num) { function logResult (results, type, testAsyncHooks) { console.log(`\n${type.toUpperCase()}:`) if (testAsyncHooks) { - console.log(` without tracer with async_hooks with tracer`) + console.log(' without tracer with async_hooks with tracer') for (const name in results.withoutTracer[type]) { console.log( pad(name, 7), @@ -168,7 +168,7 @@ function logResult (results, type, testAsyncHooks) { ) } } else { - console.log(` without tracer with tracer`) + console.log(' without tracer with tracer') for (const name in results.withoutTracer[type]) { console.log( pad(name, 7), diff --git a/benchmark/e2e/express-helloworld-manyroutes/app.js b/benchmark/e2e/express-helloworld-manyroutes/app.js index f37144b5068..61729670bb6 100644 --- a/benchmark/e2e/express-helloworld-manyroutes/app.js +++ b/benchmark/e2e/express-helloworld-manyroutes/app.js @@ -1,4 +1,3 @@ - const crypto = require('crypto') const app = require('express')() diff --git a/benchmark/profiler/index.js b/benchmark/profiler/index.js index 556ada232ce..20f1455d05d 100644 --- a/benchmark/profiler/index.js +++ b/benchmark/profiler/index.js @@ -60,7 +60,7 @@ function benchmark (url, maxConnectionRequests) { } function compare (result1, result2) { - title(`Comparison (disabled VS enabled)`) + title('Comparison (disabled VS enabled)') compareNet(result1.net, result2.net) compareCpu(result1.cpu, result2.cpu) diff --git a/benchmark/sirun/Dockerfile b/benchmark/sirun/Dockerfile index f212bfd662c..6ce6d8557fe 100644 --- a/benchmark/sirun/Dockerfile +++ b/benchmark/sirun/Dockerfile @@ -30,7 +30,6 @@ RUN wget -O sirun.tar.gz https://github.com/DataDog/sirun/releases/download/v0.1 RUN mkdir -p /usr/local/nvm \ && wget -q -O - https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | bash \ && . $NVM_DIR/nvm.sh \ - && nvm install --no-progress 12.22.12 \ && nvm install --no-progress 14.21.3 \ && nvm install --no-progress 16.20.1 \ && nvm install --no-progress 18.16.1 \ diff --git a/benchmark/sirun/get-results.js b/benchmark/sirun/get-results.js index 266cc77337a..c5c65cb72fd 100644 --- a/benchmark/sirun/get-results.js +++ b/benchmark/sirun/get-results.js @@ -7,13 +7,13 @@ const { execSync } = require('child_process') const { CIRCLE_TOKEN, GITHUB_STATUS_TOKEN } = process.env -const circleHeaders = CIRCLE_TOKEN ? { - 'circle-token': CIRCLE_TOKEN -} : {} +const circleHeaders = CIRCLE_TOKEN + ? { 'circle-token': CIRCLE_TOKEN } + : {} -const githubHeaders = GITHUB_STATUS_TOKEN ? { - Authorization: `token ${GITHUB_STATUS_TOKEN}` -} : {} +const githubHeaders = GITHUB_STATUS_TOKEN + ? { Authorization: `token ${GITHUB_STATUS_TOKEN}` } + : {} const statusUrl = (ref, page) => `https://api.github.com/repos/DataDog/dd-trace-js/commits/${ref}/statuses?per_page=100&page=${page}` diff --git a/benchmark/sirun/plugin-graphql/index.js b/benchmark/sirun/plugin-graphql/index.js index 29ff7f64fd8..cc31b6828a3 100644 --- a/benchmark/sirun/plugin-graphql/index.js +++ b/benchmark/sirun/plugin-graphql/index.js @@ -23,7 +23,7 @@ if (Number(process.env.WITH_ASYNC_HOOKS)) { require('async_hooks').createHook(hook).enable() } -const graphql = require(`../../../versions/graphql`).get() +const graphql = require('../../../versions/graphql').get() const schema = require('./schema') const source = ` diff --git a/benchmark/sirun/plugin-graphql/schema.js b/benchmark/sirun/plugin-graphql/schema.js index 0c330b5bf71..d7da47739b9 100644 --- a/benchmark/sirun/plugin-graphql/schema.js +++ b/benchmark/sirun/plugin-graphql/schema.js @@ -1,6 +1,6 @@ 'use strict' -const graphql = require(`../../../versions/graphql`).get() +const graphql = require('../../../versions/graphql').get() const Human = new graphql.GraphQLObjectType({ name: 'Human', diff --git a/benchmark/sirun/results-diff.js b/benchmark/sirun/results-diff.js index 8540d7258fa..58e42955bfb 100644 --- a/benchmark/sirun/results-diff.js +++ b/benchmark/sirun/results-diff.js @@ -16,7 +16,7 @@ function walk (tree, oldTree, path = []) { } } - if (typeof tree === 'object') { + if (tree !== null && typeof tree === 'object') { const result = {} for (const name in tree) { if (name in oldTree) { @@ -26,7 +26,7 @@ function walk (tree, oldTree, path = []) { return result } - throw new Error(tree.toString()) + throw new Error(String(tree)) } module.exports = walk diff --git a/benchmark/sirun/scope/index.js b/benchmark/sirun/scope/index.js index a15c30336b7..20c2f6da2f6 100644 --- a/benchmark/sirun/scope/index.js +++ b/benchmark/sirun/scope/index.js @@ -1,4 +1,3 @@ - const { DD_TRACE_SCOPE, COUNT diff --git a/benchmark/sirun/squash-affinity.js b/benchmark/sirun/squash-affinity.js index c4b8854d073..96e96ab785b 100644 --- a/benchmark/sirun/squash-affinity.js +++ b/benchmark/sirun/squash-affinity.js @@ -32,4 +32,4 @@ function squashAffinity (obj) { } } -fs.writeFileSync(path.join(process.cwd(), `meta-temp.json`), JSON.stringify(metaJson, null, 2)) +fs.writeFileSync(path.join(process.cwd(), 'meta-temp.json'), JSON.stringify(metaJson, null, 2)) diff --git a/ci/cypress/after-run.js b/ci/cypress/after-run.js new file mode 100644 index 00000000000..8fec98e3d1f --- /dev/null +++ b/ci/cypress/after-run.js @@ -0,0 +1 @@ +module.exports = require('../../packages/datadog-plugin-cypress/src/after-run') diff --git a/ci/cypress/after-spec.js b/ci/cypress/after-spec.js new file mode 100644 index 00000000000..9c3ae9da74d --- /dev/null +++ b/ci/cypress/after-spec.js @@ -0,0 +1 @@ +module.exports = require('../../packages/datadog-plugin-cypress/src/after-spec') diff --git a/ci/init.js b/ci/init.js index b6f0ba9961b..b54e29abd4d 100644 --- a/ci/init.js +++ b/ci/init.js @@ -1,15 +1,13 @@ /* eslint-disable no-console */ const tracer = require('../packages/dd-trace') -const { ORIGIN_KEY } = require('../packages/dd-trace/src/constants') const { isTrue } = require('../packages/dd-trace/src/util') const isJestWorker = !!process.env.JEST_WORKER_ID +const isCucumberWorker = !!process.env.CUCUMBER_WORKER_ID +const isMochaWorker = !!process.env.MOCHA_WORKER_ID const options = { startupLogs: false, - tags: { - [ORIGIN_KEY]: 'ciapp-test' - }, isCiVisibility: true, flushInterval: isJestWorker ? 0 : 5000 } @@ -24,9 +22,9 @@ if (isAgentlessEnabled) { exporter: 'datadog' } } else { - console.error(`DD_CIVISIBILITY_AGENTLESS_ENABLED is set, \ -but neither DD_API_KEY nor DATADOG_API_KEY are set in your environment, \ -so dd-trace will not be initialized.`) + console.error('DD_CIVISIBILITY_AGENTLESS_ENABLED is set, but neither ' + + 'DD_API_KEY nor DATADOG_API_KEY are set in your environment, so ' + + 'dd-trace will not be initialized.') shouldInit = false } } else { @@ -41,9 +39,22 @@ if (isJestWorker) { } } +if (isCucumberWorker) { + options.experimental = { + exporter: 'cucumber_worker' + } +} + +if (isMochaWorker) { + options.experimental = { + exporter: 'mocha_worker' + } +} + if (shouldInit) { tracer.init(options) tracer.use('fs', false) + tracer.use('child_process', false) } module.exports = tracer diff --git a/docker-compose.yml b/docker-compose.yml index 2ff0e15120a..81bdd3c2032 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: "2" services: aerospike: image: aerospike:ce-6.4.0.3 - ports: + ports: - "127.0.0.1:3000-3002:3000-3002" couchbase: image: ghcr.io/datadog/couchbase-server-sandbox:latest @@ -38,6 +38,7 @@ services: - "127.0.0.1:6379:6379" mongo: image: circleci/mongo:3.6 + platform: linux/amd64 ports: - "127.0.0.1:27017:27017" oracledb: @@ -46,7 +47,7 @@ services: - '127.0.0.1:1521:1521' - '127.0.0.1:5500:5500' elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.14.0 + image: elasticsearch:7.17.22 environment: - discovery.type=single-node - "ES_JAVA_OPTS=-Xms128m -Xmx128m" @@ -69,9 +70,7 @@ services: ports: - "11211:11211" cassandra: - image: spotify/cassandra - environment: - - CASSANDRA_TOKEN=-9223372036854775808 + image: cassandra:3-focal ports: - "127.0.0.1:9042:9042" limitd: @@ -87,11 +86,26 @@ services: ports: - "127.0.0.1:8081:8081" localstack: - # TODO: Figure out why SNS doesn't work in >=1.2 - # https://github.com/localstack/localstack/issues/7479 - image: localstack/localstack:1.1.0 + image: localstack/localstack:3.0.2 ports: - "127.0.0.1:4566:4566" # Edge + environment: + - LOCALSTACK_SERVICES=dynamodb,kinesis,s3,sqs,sns,redshift,route53,logs,serverless,lambda,stepfunctions,events + - EXTRA_CORS_ALLOWED_HEADERS=x-amz-request-id,x-amzn-requestid,x-amz-id-2 + - EXTRA_CORS_EXPOSE_HEADERS=x-amz-request-id,x-amzn-requestid,x-amz-id-2 + - AWS_DEFAULT_REGION=us-east-1 + - FORCE_NONINTERACTIVE=true + - START_WEB=0 + - DEBUG=${DEBUG-} + - DOCKER_HOST=unix:///var/run/docker.sock + volumes: + - "/var/run/docker.sock:/var/run/docker.sock" + localstack-legacy: + # we have two localstacks since upgrading localstack was causing lambda & S3 tests to fail + # To-Do: Debug localstack / lambda and localstack / S3 + image: localstack/localstack:1.1.0 + ports: + - "127.0.0.1:4567:4567" # Edge environment: - LOCALSTACK_SERVICES=dynamodb,kinesis,s3,sqs,sns,redshift,route53,logs,serverless - EXTRA_CORS_ALLOWED_HEADERS=x-amz-request-id,x-amzn-requestid,x-amz-id-2 @@ -99,18 +113,27 @@ services: - AWS_DEFAULT_REGION=us-east-1 - FORCE_NONINTERACTIVE=true - START_WEB=0 + - GATEWAY_LISTEN=127.0.0.1:4567 + - EDGE_PORT=4567 + - EDGE_PORT_HTTP=4567 - LAMBDA_EXECUTOR=local kafka: - image: debezium/kafka:1.7 + platform: linux/arm64 + image: apache/kafka-native:3.8.0-rc2 ports: - "127.0.0.1:9092:9092" - "127.0.0.1:9093:9093" environment: - - CLUSTER_ID=5Yr1SIgYQz-b-dgRabWx4g - - NODE_ID=1 - - CREATE_TOPICS="test-topic:1:1" - - KAFKA_CONTROLLER_QUORUM_VOTERS=1@kafka:9093 + - KAFKA_PROCESS_ROLES=broker,controller + - KAFKA_NODE_ID=1 + - KAFKA_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093 + - KAFKA_CONTROLLER_QUORUM_VOTERS=1@127.0.0.1:9093 + - KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER + - CLUSTER_ID=5L6g3nShT-eMCtK--X86sw - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://127.0.0.1:9092 + - KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT + - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT + - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 opensearch: image: opensearchproject/opensearch:2 @@ -132,11 +155,11 @@ services: - LDAP_PASSWORDS=password1,password2 testagent: - image: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.13.1 + image: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.16.0 ports: - "127.0.0.1:9126:9126" environment: - LOG_LEVEL=DEBUG - TRACE_LANGUAGE=javascript - - DISABLED_CHECKS=trace_content_length + - ENABLED_CHECKS=trace_stall,meta_tracer_version_header,trace_count_header,trace_peer_service - PORT=9126 diff --git a/docs/API.md b/docs/API.md index 5902b284e60..19827e5977d 100644 --- a/docs/API.md +++ b/docs/API.md @@ -8,7 +8,7 @@ The module exported by this library is an instance of the [Tracer](./interfaces/

Automatic Instrumentation

-APM provides out-of-the-box instrumentation for many popular frameworks and libraries by using a plugin system. By default all built-in plugins are enabled. Disabling plugins can cause unexpected side effects, so it is highly recommended to leave them enabled. +APM provides out-of-the-box instrumentation for many popular frameworks and libraries by using a plugin system. By default, all built-in plugins are enabled. Disabling plugins can cause unexpected side effects, so it is highly recommended to leave them enabled. Built-in plugins can be configured individually: @@ -24,9 +24,11 @@ tracer.use('pg', {
+
+
@@ -87,6 +89,7 @@ tracer.use('pg', {
+
@@ -94,60 +97,65 @@ tracer.use('pg', {
+

Available Plugins

-* [amqp10](./interfaces/plugins.amqp10.html) -* [amqplib](./interfaces/plugins.amqplib.html) -* [aws-sdk](./interfaces/plugins.aws_sdk.html) -* [bluebird](./interfaces/plugins.bluebird.html) -* [couchbase](./interfaces/plugins.couchbase.html) -* [cucumber](./interfaces/plugins.cucumber.html) -* [bunyan](./interfaces/plugins.bunyan.html) -* [cassandra-driver](./interfaces/plugins.cassandra_driver.html) -* [connect](./interfaces/plugins.connect.html) -* [dns](./interfaces/plugins.dns.html) -* [elasticsearch](./interfaces/plugins.elasticsearch.html) -* [express](./interfaces/plugins.express.html) -* [fastify](./interfaces/plugins.fastify.html) -* [fetch](./interfaces/plugins.fetch.html) -* [generic-pool](./interfaces/plugins.generic_pool.html) -* [google-cloud-pubsub](./interfaces/plugins.google_cloud_pubsub.html) -* [graphql](./interfaces/plugins.graphql.html) -* [grpc](./interfaces/plugins.grpc.html) -* [hapi](./interfaces/plugins.hapi.html) -* [http](./interfaces/plugins.http.html) -* [http2](./interfaces/plugins.http2.html) -* [ioredis](./interfaces/plugins.ioredis.html) -* [jest](./interfaces/plugins.jest.html) -* [kafkajs](./interfaces/plugins.kafkajs.html) -* [knex](./interfaces/plugins.knex.html) -* [koa](./interfaces/plugins.koa.html) -* [ldapjs](./interfaces/plugins.ldapjs.html) -* [mariadb](./interfaces/plugins.mariadb.html) -* [microgateway--core](./interfaces/plugins.microgateway_core.html) -* [mocha](./interfaces/plugins.mocha.html) -* [mongodb-core](./interfaces/plugins.mongodb_core.html) -* [mysql](./interfaces/plugins.mysql.html) -* [mysql2](./interfaces/plugins.mysql2.html) -* [net](./interfaces/plugins.net.html) -* [next](./interfaces/plugins.next.html) -* [opensearch](./interfaces/plugins.opensearch.html) -* [openai](./interfaces/plugins.openai.html) -* [oracledb](./interfaces/plugins.oracledb.html) -* [paperplane](./interfaces/plugins.paperplane.html) -* [pino](./interfaces/plugins.pino.html) -* [pg](./interfaces/plugins.pg.html) -* [promise](./interfaces/plugins.promise.html) -* [promise-js](./interfaces/plugins.promise_js.html) -* [q](./interfaces/plugins.q.html) -* [redis](./interfaces/plugins.redis.html) -* [restify](./interfaces/plugins.restify.html) -* [router](./interfaces/plugins.router.html) -* [tedious](./interfaces/plugins.tedious.html) -* [when](./interfaces/plugins.when.html) -* [winston](./interfaces/plugins.winston.html) +* [amqp10](./interfaces/export_.plugins.amqp10.html) +* [amqplib](./interfaces/export_.plugins.amqplib.html) +* [avsc](./interfaces/export_.plugins.avsc.html) +* [aws-sdk](./interfaces/export_.plugins.aws_sdk.html) +* [azure-functions](./interfaces/export_.plugins.azure_functions.html) +* [bluebird](./interfaces/export_.plugins.bluebird.html) +* [couchbase](./interfaces/export_.plugins.couchbase.html) +* [cucumber](./interfaces/export_.plugins.cucumber.html) +* [bunyan](./interfaces/export_.plugins.bunyan.html) +* [cassandra-driver](./interfaces/export_.plugins.cassandra_driver.html) +* [connect](./interfaces/export_.plugins.connect.html) +* [dns](./interfaces/export_.plugins.dns.html) +* [elasticsearch](./interfaces/export_.plugins.elasticsearch.html) +* [express](./interfaces/export_.plugins.express.html) +* [fastify](./interfaces/export_.plugins.fastify.html) +* [fetch](./interfaces/export_.plugins.fetch.html) +* [generic-pool](./interfaces/export_.plugins.generic_pool.html) +* [google-cloud-pubsub](./interfaces/export_.plugins.google_cloud_pubsub.html) +* [graphql](./interfaces/export_.plugins.graphql.html) +* [grpc](./interfaces/export_.plugins.grpc.html) +* [hapi](./interfaces/export_.plugins.hapi.html) +* [http](./interfaces/export_.plugins.http.html) +* [http2](./interfaces/export_.plugins.http2.html) +* [ioredis](./interfaces/export_.plugins.ioredis.html) +* [jest](./interfaces/export_.plugins.jest.html) +* [kafkajs](./interfaces/export_.plugins.kafkajs.html) +* [knex](./interfaces/export_.plugins.knex.html) +* [koa](./interfaces/export_.plugins.koa.html) +* [ldapjs](./interfaces/export_.plugins.ldapjs.html) +* [mariadb](./interfaces/export_.plugins.mariadb.html) +* [microgateway--core](./interfaces/export_.plugins.microgateway_core.html) +* [mocha](./interfaces/export_.plugins.mocha.html) +* [mongodb-core](./interfaces/export_.plugins.mongodb_core.html) +* [mysql](./interfaces/export_.plugins.mysql.html) +* [mysql2](./interfaces/export_.plugins.mysql2.html) +* [net](./interfaces/export_.plugins.net.html) +* [next](./interfaces/export_.plugins.next.html) +* [opensearch](./interfaces/export_.plugins.opensearch.html) +* [openai](./interfaces/export_.plugins.openai.html) +* [oracledb](./interfaces/export_.plugins.oracledb.html) +* [paperplane](./interfaces/export_.plugins.paperplane.html) +* [pino](./interfaces/export_.plugins.pino.html) +* [pg](./interfaces/export_.plugins.pg.html) +* [promise](./interfaces/export_.plugins.promise.html) +* [promise-js](./interfaces/export_.plugins.promise_js.html) +* [protobufjs](./interfaces/export_.plugins.protobufjs.html) +* [q](./interfaces/export_.plugins.q.html) +* [redis](./interfaces/export_.plugins.redis.html) +* [restify](./interfaces/export_.plugins.restify.html) +* [router](./interfaces/export_.plugins.router.html) +* [tedious](./interfaces/export_.plugins.tedious.html) +* [undici](./interfaces/export_.plugins.undici.html) +* [when](./interfaces/export_.plugins.when.html) +* [winston](./interfaces/export_.plugins.winston.html)

Manual Instrumentation

@@ -192,7 +200,7 @@ Errors passed to the callback will automatically be added to the span.

Promise

-For promises, the span will be finished afer the promise has been either resolved or rejected. +For promises, the span will be finished after the promise has been either resolved or rejected. ```javascript function handle () { @@ -260,7 +268,7 @@ This method returns the active span from the current scope.

scope.activate(span, fn)

This method activates the provided span in a new scope available in the -provided function. Any asynchronous context created from whithin that function +provided function. Any asynchronous context created from within that function will also have the same scope. ```javascript @@ -342,7 +350,7 @@ const opentracing = require('opentracing') opentracing.initGlobalTracer(tracer) ``` -The following tags are available to override Datadog specific options: +The following tags are available to override Datadog-specific options: * `service.name`: The service name to be used for this span. The service name from the tracer will be used if this is not provided. * `resource.name`: The resource name to be used for this span. The operation name will be used if this is not provided. @@ -359,7 +367,7 @@ const tracerProvider = new tracer.TracerProvider() tracerProvider.register() ``` -The following attributes are available to override Datadog specific options: +The following attributes are available to override Datadog-specific options: * `service.name`: The service name to be used for this span. The service name from the tracer will be used if this is not provided. * `resource.name`: The resource name to be used for this span. The operation name will be used if this is not provided. @@ -373,7 +381,7 @@ Options can be configured as a parameter to the [init()](./interfaces/tracer.htm

Custom Logging

-By default, logging from this library is disabled. In order to get debugging information and errors sent to logs, the `debug` options should be set to `true` in the [init()](./interfaces/tracer.html#init) method. +By default, logging from this library is disabled. In order to get debugging information and errors sent to logs, the `DD_TRACE_DEBUG` env var should be set to `true`. The tracer will then log debug information to `console.log()` and errors to `console.error()`. This behavior can be changed by passing a custom logger to the tracer. The logger should contain a `debug()` and `error()` methods that can handle messages and errors, respectively. @@ -386,14 +394,15 @@ const logger = bunyan.createLogger({ level: 'trace' }) +process.env.DD_TRACE_DEBUG = 'true' + const tracer = require('dd-trace').init({ logger: { error: err => logger.error(err), warn: message => logger.warn(message), info: message => logger.info(message), debug: message => logger.trace(message), - }, - debug: true + } }) ``` diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000000..f35f562bf71 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,6 @@ +Note: If you're viewing this `docs/` directory on GitHub +you'll find that none of the links work and that most of +the content is missing. This directory simply contains +base files used to generate docs for the API docs site: + +[API Documentation](https://datadoghq.dev/dd-trace-js/) diff --git a/docs/add-redirects.sh b/docs/add-redirects.sh new file mode 100755 index 00000000000..92d58ba3263 --- /dev/null +++ b/docs/add-redirects.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash + +# Previously, URLs to plugin pages looked like this: +# interfaces/plugins.amqp10.html +# +# Now, with an updated typedoc and updated types, they look like this: +# interfaces/export_.plugins.connect.html +# +# This script automatically generates basic HTML files to redirect users who +# visit the old URLs to the new URL. + +echo "writing redirects..." + +declare -a plugins=( + "amqp10" + "amqplib" + "avsc" + "aws_sdk" + "bluebird" + "couchbase" + "cucumber" + "bunyan" + "cassandra_driver" + "connect" + "dns" + "elasticsearch" + "express" + "fastify" + "fetch" + "generic_pool" + "google_cloud_pubsub" + "graphql" + "grpc" + "hapi" + "http" + "http2" + "ioredis" + "jest" + "kafkajs" + "knex" + "koa" + "ldapjs" + "mariadb" + "microgateway_core" + "mocha" + "mongodb_core" + "mysql" + "mysql2" + "net" + "next" + "opensearch" + "openai" + "oracledb" + "paperplane" + "pino" + "pg" + "promise" + "promise_js" + "protobufjs" + "q" + "redis" + "restify" + "router" + "tedious" + "undici" + "when" + "winston" +) + +for i in "${plugins[@]}" +do + echo "" > out/interfaces/plugins.$i.html +done + +echo "done." diff --git a/docs/package.json b/docs/package.json index 3f58f83cbda..30cb5dd848a 100644 --- a/docs/package.json +++ b/docs/package.json @@ -3,14 +3,14 @@ "version": "1.0.0", "main": "typedoc.js", "scripts": { - "build": "typedoc", + "build": "typedoc ../index.d.ts && ./add-redirects.sh", "pretest": "tsc -p . && tsc test", "test": "node test" }, "license": "BSD-3-Clause", "private": true, "devDependencies": { - "typedoc": "^0.17.3", - "typescript": "^3.8.3" + "typedoc": "^0.25.8", + "typescript": "^4.6" } } diff --git a/docs/test.ts b/docs/test.ts index 3c9342a0bba..9c6c7df6211 100644 --- a/docs/test.ts +++ b/docs/test.ts @@ -1,5 +1,6 @@ import { performance } from 'perf_hooks' import ddTrace, { tracer, Tracer, TracerOptions, Span, SpanContext, SpanOptions, Scope, User } from '..'; +import type { plugins } from '..'; import { opentelemetry } from '..'; import { formats, kinds, priority, tags, types } from '../ext'; import { BINARY, HTTP_HEADERS, LOG, TEXT_MAP } from '../ext/formats'; @@ -108,13 +109,34 @@ tracer.init({ obfuscatorValueRegex: '.*', blockedTemplateHtml: './blocked.html', blockedTemplateJson: './blocked.json', + blockedTemplateGraphql: './blockedgraphql.json', eventTracking: { mode: 'safe' }, apiSecurity: { enabled: true, requestSampling: 1.0 + }, + rasp: { + enabled: true + }, + stackTrace: { + enabled: true, + maxStackTraces: 5, + maxDepth: 42 } + }, + iast: { + enabled: true, + cookieFilterPattern: '.*', + requestSampling: 50, + maxConcurrentRequests: 4, + maxContextOperations: 30, + deduplicationEnabled: true, + redactionEnabled: true, + redactionNamePattern: 'password', + redactionValuePattern: 'bearer', + telemetryVerbosity: 'OFF' } }); @@ -122,13 +144,20 @@ tracer.init({ experimental: { iast: { enabled: true, + cookieFilterPattern: '.*', requestSampling: 50, maxConcurrentRequests: 4, maxContextOperations: 30, deduplicationEnabled: true, redactionEnabled: true, redactionNamePattern: 'password', - redactionValuePattern: 'bearer' + redactionValuePattern: 'bearer', + telemetryVerbosity: 'OFF' + }, + appsec: { + standalone: { + enabled: true + } } } }) @@ -156,39 +185,39 @@ const httpOptions = { middleware: true }; -const httpServerOptions = { +const httpServerOptions: plugins.HttpServer = { ...httpOptions, hooks: { - request: (span: Span, req, res) => {} + request: (span?: Span, req?, res?) => {} } }; -const httpClientOptions = { +const httpClientOptions: plugins.HttpClient = { ...httpOptions, splitByDomain: true, propagationBlocklist: ['url', /url/, url => true], hooks: { - request: (span: Span, req, res) => {} + request: (span?: Span, req?, res?) => { } } }; -const http2ServerOptions = { +const http2ServerOptions: plugins.Http2Server = { ...httpOptions }; -const http2ClientOptions = { +const http2ClientOptions: plugins.Http2Client = { ...httpOptions, splitByDomain: true }; -const nextOptions = { +const nextOptions: plugins.next = { service: 'test', hooks: { - request: (span: Span, params) => { }, + request: (span?: Span, params?) => { }, }, }; -const graphqlOptions = { +const graphqlOptions: plugins.graphql = { service: 'test', depth: 2, source: true, @@ -196,24 +225,25 @@ const graphqlOptions = { collapse: false, signature: false, hooks: { - execute: (span: Span, args, res) => {}, - validate: (span: Span, document, errors) => {}, - parse: (span: Span, source, document) => {} + execute: (span?: Span, args?, res?) => {}, + validate: (span?: Span, document?, errors?) => {}, + parse: (span?: Span, source?, document?) => {} } }; -const elasticsearchOptions = { +const elasticsearchOptions: plugins.elasticsearch = { service: 'test', hooks: { - query: (span: Span, params) => {}, + query: (span?: Span, params?) => {}, }, }; -const awsSdkOptions = { +const awsSdkOptions: plugins.aws_sdk = { service: 'test', splitByAwsService: false, + batchPropagationEnabled: false, hooks: { - request: (span: Span, response) => {}, + request: (span?: Span, response?) => {}, }, s3: false, sqs: { @@ -222,43 +252,45 @@ const awsSdkOptions = { } }; -const redisOptions = { +const redisOptions: plugins.redis = { service: 'test', allowlist: ['info', /auth/i, command => true], blocklist: ['info', /auth/i, command => true], }; -const sharedbOptions = { +const sharedbOptions: plugins.sharedb = { service: 'test', hooks: { - receive: (span: Span, request) => {}, - reply: (span: Span, request, reply) => {}, + receive: (span?: Span, request?) => {}, + reply: (span?: Span, request?, reply?) => {}, }, }; -const moleculerOptions = { +const moleculerOptions: plugins.moleculer = { service: 'test', client: false, - params: true, server: { meta: true } }; -const openSearchOptions = { +const openSearchOptions: plugins.opensearch = { service: 'test', hooks: { - query: (span: Span, params) => {}, + query: (span?: Span, params?) => {}, }, }; tracer.use('amqp10'); tracer.use('amqplib'); +tracer.use('avsc'); tracer.use('aws-sdk'); tracer.use('aws-sdk', awsSdkOptions); +tracer.use('azure-functions'); tracer.use('bunyan'); tracer.use('couchbase'); tracer.use('cassandra-driver'); +tracer.use('child_process'); tracer.use('connect'); tracer.use('connect', httpServerOptions); tracer.use('cypress'); @@ -292,9 +324,6 @@ tracer.use('http', { tracer.use('http', { client: httpClientOptions }); -tracer.use('http', { - enablePropagationWithAmazonHeaders: true -}); tracer.use('http2'); tracer.use('http2', { server: http2ServerOptions @@ -337,15 +366,20 @@ tracer.use('playwright'); tracer.use('pg'); tracer.use('pg', { service: params => `${params.host}-${params.database}` }); tracer.use('pino'); +tracer.use('protobufjs'); tracer.use('redis'); tracer.use('redis', redisOptions); tracer.use('restify'); tracer.use('restify', httpServerOptions); tracer.use('rhea'); tracer.use('router'); +tracer.use('selenium'); tracer.use('sharedb'); tracer.use('sharedb', sharedbOptions); tracer.use('tedious'); +tracer.use('undici'); +tracer.use('vitest'); +tracer.use('vitest', { service: 'vitest-service' }); tracer.use('winston'); tracer.use('express', false) @@ -355,8 +389,9 @@ tracer.use('express', { measured: true }); span = tracer.startSpan('test'); span = tracer.startSpan('test', {}); +span = tracer.startSpan('test', { childOf: span }); span = tracer.startSpan('test', { - childOf: span || span.context(), + childOf: span.context(), references: [], startTime: 123456789.1234, tags: { @@ -370,7 +405,7 @@ tracer.trace('test', { service: 'foo', resource: 'bar', type: 'baz' }, () => {}) tracer.trace('test', { measured: true }, () => {}) tracer.trace('test', (span: Span) => {}) tracer.trace('test', (span: Span, fn: () => void) => {}) -tracer.trace('test', (span: Span, fn: (err: Error) => string) => {}) +tracer.trace('test', (span: Span, fn: (err: Error) => void) => {}) promise = tracer.trace('test', () => Promise.resolve()) @@ -381,8 +416,9 @@ promise = tracer.wrap('test', () => Promise.resolve())() const carrier = {} -tracer.inject(span || span.context(), HTTP_HEADERS, carrier); -context = tracer.extract(HTTP_HEADERS, carrier); +tracer.inject(span, HTTP_HEADERS, carrier); +tracer.inject(span.context(), HTTP_HEADERS, carrier); +context = tracer.extract(HTTP_HEADERS, carrier)!; traceId = context.toTraceId(); spanId = context.toSpanId(); @@ -390,7 +426,7 @@ traceparent = context.toTraceparent(); const scope = tracer.scope() -span = scope.active(); +span = scope.active()!; const activateStringType: string = scope.activate(span, () => 'test'); const activateVoidType: void = scope.activate(span, () => {}); diff --git a/docs/typedoc.js b/docs/typedoc.js index bce72b793e2..ccb5c55c04c 100644 --- a/docs/typedoc.js +++ b/docs/typedoc.js @@ -4,8 +4,8 @@ module.exports = { excludeExternals: true, excludePrivate: true, excludeProtected: true, - includeDeclarations: true, - mode: 'file', + // includeDeclarations: true, + // mode: 'file', name: 'dd-trace', out: 'out', readme: 'API.md' diff --git a/docs/yarn.lock b/docs/yarn.lock index 1f1ffeef806..4b011ed3db2 100644 --- a/docs/yarn.lock +++ b/docs/yarn.lock @@ -2,243 +2,76 @@ # yarn lockfile v1 +ansi-sequence-parser@^1.1.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/ansi-sequence-parser/-/ansi-sequence-parser-1.1.1.tgz#e0aa1cdcbc8f8bb0b5bca625aac41f5f056973cf" + integrity sha512-vJXt3yiaUL4UU546s3rPXlsry/RnM730G1+HkpKE012AN0sx1eOrxSu95oKDIonskeLTijMgqWZ3uDEe3NFvyg== + balanced-match@^1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee" integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== -brace-expansion@^1.1.7: - version "1.1.11" - resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd" - integrity sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA== +brace-expansion@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-2.0.1.tgz#1edc459e0f0c548486ecf9fc99f2221364b9a0ae" + integrity sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA== dependencies: balanced-match "^1.0.0" - concat-map "0.0.1" - -concat-map@0.0.1: - version "0.0.1" - resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" - integrity sha1-2Klr13/Wjfd5OnMDajug1UBdR3s= - -fs-extra@^8.1.0: - version "8.1.0" - resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-8.1.0.tgz#49d43c45a88cd9677668cb7be1b46efdb8d2e1c0" - integrity sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g== - dependencies: - graceful-fs "^4.2.0" - jsonfile "^4.0.0" - universalify "^0.1.0" - -fs.realpath@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" - integrity sha1-FQStJSMVjKpA20onh8sBQRmU6k8= - -function-bind@^1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d" - integrity sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A== - -glob@^7.0.0: - version "7.2.0" - resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.0.tgz#d15535af7732e02e948f4c41628bd910293f6023" - integrity sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q== - dependencies: - fs.realpath "^1.0.0" - inflight "^1.0.4" - inherits "2" - minimatch "^3.0.4" - once "^1.3.0" - path-is-absolute "^1.0.0" - -graceful-fs@^4.1.6, graceful-fs@^4.2.0: - version "4.2.8" - resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.8.tgz#e412b8d33f5e006593cbd3cee6df9f2cebbe802a" - integrity sha512-qkIilPUYcNhJpd33n0GBXTB1MMPp14TxEsEs0pTrsSVucApsYzW5V+Q8Qxhik6KU3evy+qkAAowTByymK0avdg== - -handlebars@^4.7.6: - version "4.7.7" - resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.7.7.tgz#9ce33416aad02dbd6c8fafa8240d5d98004945a1" - integrity sha512-aAcXm5OAfE/8IXkcZvCepKU3VzW1/39Fb5ZuqMtgI/hT8X2YgoMvBY5dLhq/cpOvw7Lk1nK/UF71aLG/ZnVYRA== - dependencies: - minimist "^1.2.5" - neo-async "^2.6.0" - source-map "^0.6.1" - wordwrap "^1.0.0" - optionalDependencies: - uglify-js "^3.1.4" - -has@^1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/has/-/has-1.0.3.tgz#722d7cbfc1f6aa8241f16dd814e011e1f41e8796" - integrity sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw== - dependencies: - function-bind "^1.1.1" - -highlight.js@^10.0.0: - version "10.7.3" - resolved "https://registry.yarnpkg.com/highlight.js/-/highlight.js-10.7.3.tgz#697272e3991356e40c3cac566a74eef681756531" - integrity sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A== - -inflight@^1.0.4: - version "1.0.6" - resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9" - integrity sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk= - dependencies: - once "^1.3.0" - wrappy "1" - -inherits@2: - version "2.0.4" - resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" - integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== - -interpret@^1.0.0: - version "1.4.0" - resolved "https://registry.yarnpkg.com/interpret/-/interpret-1.4.0.tgz#665ab8bc4da27a774a40584e812e3e0fa45b1a1e" - integrity sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA== - -is-core-module@^2.2.0: - version "2.8.0" - resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.8.0.tgz#0321336c3d0925e497fd97f5d95cb114a5ccd548" - integrity sha512-vd15qHsaqrRL7dtH6QNuy0ndJmRDrS9HAM1CAiSifNUFv4x1a0CCVsj18hJ1mShxIG6T2i1sO78MkP56r0nYRw== - dependencies: - has "^1.0.3" -jsonfile@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-4.0.0.tgz#8771aae0799b64076b76640fca058f9c10e33ecb" - integrity sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss= - optionalDependencies: - graceful-fs "^4.1.6" +jsonc-parser@^3.2.0: + version "3.2.1" + resolved "https://registry.yarnpkg.com/jsonc-parser/-/jsonc-parser-3.2.1.tgz#031904571ccf929d7670ee8c547545081cb37f1a" + integrity sha512-AilxAyFOAcK5wA1+LeaySVBrHsGQvUFCDWXKpZjzaL0PqW+xfBOttn8GNtWKFWqneyMZj41MWF9Kl6iPWLwgOA== -lodash@^4.17.15: - version "4.17.21" - resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" - integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== - -lunr@^2.3.8: +lunr@^2.3.9: version "2.3.9" resolved "https://registry.yarnpkg.com/lunr/-/lunr-2.3.9.tgz#18b123142832337dd6e964df1a5a7707b25d35e1" integrity sha512-zTU3DaZaF3Rt9rhN3uBMGQD3dD2/vFQqnvZCDv4dl5iOzq2IZQqTxu90r4E5J+nP70J3ilqVCrbho2eWaeW8Ow== -marked@1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/marked/-/marked-1.0.0.tgz#d35784245a04871e5988a491e28867362e941693" - integrity sha512-Wo+L1pWTVibfrSr+TTtMuiMfNzmZWiOPeO7rZsQUY5bgsxpHesBEcIWJloWVTFnrMXnf/TL30eTFSGJddmQAng== - -minimatch@^3.0.0, minimatch@^3.0.4: - version "3.0.4" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" - integrity sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA== - dependencies: - brace-expansion "^1.1.7" - -minimist@^1.2.5: - version "1.2.5" - resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602" - integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw== - -neo-async@^2.6.0: - version "2.6.2" - resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.2.tgz#b4aafb93e3aeb2d8174ca53cf163ab7d7308305f" - integrity sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw== - -once@^1.3.0: - version "1.4.0" - resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" - integrity sha1-WDsap3WWHUsROsF9nFC6753Xa9E= - dependencies: - wrappy "1" - -path-is-absolute@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" - integrity sha1-F0uSaHNVNP+8es5r9TpanhtcX18= - -path-parse@^1.0.6: - version "1.0.7" - resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.7.tgz#fbc114b60ca42b30d9daf5858e4bd68bbedb6735" - integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw== - -progress@^2.0.3: - version "2.0.3" - resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8" - integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA== - -rechoir@^0.6.2: - version "0.6.2" - resolved "https://registry.yarnpkg.com/rechoir/-/rechoir-0.6.2.tgz#85204b54dba82d5742e28c96756ef43af50e3384" - integrity sha1-hSBLVNuoLVdC4oyWdW70OvUOM4Q= - dependencies: - resolve "^1.1.6" - -resolve@^1.1.6: - version "1.20.0" - resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.20.0.tgz#629a013fb3f70755d6f0b7935cc1c2c5378b1975" - integrity sha512-wENBPt4ySzg4ybFQW2TT1zMQucPK95HSh/nq2CFTZVOGut2+pQvSsgtda4d26YrYcr067wjbmzOG8byDPBX63A== - dependencies: - is-core-module "^2.2.0" - path-parse "^1.0.6" +marked@^4.3.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/marked/-/marked-4.3.0.tgz#796362821b019f734054582038b116481b456cf3" + integrity sha512-PRsaiG84bK+AMvxziE/lCFss8juXjNaWzVbN5tXAm4XjeaS9NAHhop+PjQxz2A9h8Q4M/xGmzP8vqNwy6JeK0A== -shelljs@^0.8.4: - version "0.8.4" - resolved "https://registry.yarnpkg.com/shelljs/-/shelljs-0.8.4.tgz#de7684feeb767f8716b326078a8a00875890e3c2" - integrity sha512-7gk3UZ9kOfPLIAbslLzyWeGiEqx9e3rxwZM0KE6EL8GlGwjym9Mrlx5/p33bWTu9YG6vcS4MBxYZDHYr5lr8BQ== +minimatch@^9.0.3: + version "9.0.3" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-9.0.3.tgz#a6e00c3de44c3a542bfaae70abfc22420a6da825" + integrity sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg== dependencies: - glob "^7.0.0" - interpret "^1.0.0" - rechoir "^0.6.2" + brace-expansion "^2.0.1" -source-map@^0.6.1: - version "0.6.1" - resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263" - integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== - -typedoc-default-themes@^0.10.2: - version "0.10.2" - resolved "https://registry.yarnpkg.com/typedoc-default-themes/-/typedoc-default-themes-0.10.2.tgz#743380a80afe62c5ef92ca1bd4abe2ac596be4d2" - integrity sha512-zo09yRj+xwLFE3hyhJeVHWRSPuKEIAsFK5r2u47KL/HBKqpwdUSanoaz5L34IKiSATFrjG5ywmIu98hPVMfxZg== +shiki@^0.14.7: + version "0.14.7" + resolved "https://registry.yarnpkg.com/shiki/-/shiki-0.14.7.tgz#c3c9e1853e9737845f1d2ef81b31bcfb07056d4e" + integrity sha512-dNPAPrxSc87ua2sKJ3H5dQ/6ZaY8RNnaAqK+t0eG7p0Soi2ydiqbGOTaZCqaYvA/uZYfS1LJnemt3Q+mSfcPCg== dependencies: - lunr "^2.3.8" - -typedoc@^0.17.3: - version "0.17.8" - resolved "https://registry.yarnpkg.com/typedoc/-/typedoc-0.17.8.tgz#96b67e9454aa7853bfc4dc9a55c8a07adfd5478e" - integrity sha512-/OyrHCJ8jtzu+QZ+771YaxQ9s4g5Z3XsQE3Ma7q+BL392xxBn4UMvvCdVnqKC2T/dz03/VXSLVKOP3lHmDdc/w== + ansi-sequence-parser "^1.1.0" + jsonc-parser "^3.2.0" + vscode-oniguruma "^1.7.0" + vscode-textmate "^8.0.0" + +typedoc@^0.25.8: + version "0.25.8" + resolved "https://registry.yarnpkg.com/typedoc/-/typedoc-0.25.8.tgz#7d0e1bf12d23bf1c459fd4893c82cb855911ff12" + integrity sha512-mh8oLW66nwmeB9uTa0Bdcjfis+48bAjSH3uqdzSuSawfduROQLlXw//WSNZLYDdhmMVB7YcYZicq6e8T0d271A== dependencies: - fs-extra "^8.1.0" - handlebars "^4.7.6" - highlight.js "^10.0.0" - lodash "^4.17.15" - lunr "^2.3.8" - marked "1.0.0" - minimatch "^3.0.0" - progress "^2.0.3" - shelljs "^0.8.4" - typedoc-default-themes "^0.10.2" - -typescript@^3.8.3: - version "3.9.10" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.9.10.tgz#70f3910ac7a51ed6bef79da7800690b19bf778b8" - integrity sha512-w6fIxVE/H1PkLKcCPsFqKE7Kv7QUwhU8qQY2MueZXWx5cPZdwFupLgKK3vntcK98BtNHZtAF4LA/yl2a7k8R6Q== - -uglify-js@^3.1.4: - version "3.14.5" - resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.14.5.tgz#cdabb7d4954231d80cb4a927654c4655e51f4859" - integrity sha512-qZukoSxOG0urUTvjc2ERMTcAy+BiFh3weWAkeurLwjrCba73poHmG3E36XEjd/JGukMzwTL7uCxZiAexj8ppvQ== - -universalify@^0.1.0: - version "0.1.2" - resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66" - integrity sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg== - -wordwrap@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-1.0.0.tgz#27584810891456a4171c8d0226441ade90cbcaeb" - integrity sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus= - -wrappy@1: - version "1.0.2" - resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" - integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8= + lunr "^2.3.9" + marked "^4.3.0" + minimatch "^9.0.3" + shiki "^0.14.7" + +typescript@^4.6: + version "4.9.5" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.9.5.tgz#095979f9bcc0d09da324d58d03ce8f8374cbe65a" + integrity sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g== + +vscode-oniguruma@^1.7.0: + version "1.7.0" + resolved "https://registry.yarnpkg.com/vscode-oniguruma/-/vscode-oniguruma-1.7.0.tgz#439bfad8fe71abd7798338d1cd3dc53a8beea94b" + integrity sha512-L9WMGRfrjOhgHSdOYgCt/yRMsXzLDJSL7BPrOZt73gU0iWO4mpqzqQzOz5srxqTvMBaR0XZTSrVWo4j55Rc6cA== + +vscode-textmate@^8.0.0: + version "8.0.0" + resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-8.0.0.tgz#2c7a3b1163ef0441097e0b5d6389cd5504b59e5d" + integrity sha512-AFbieoL7a5LMqcnOF04ji+rpXadgOXnZsxQr//r83kLPr7biP7am3g9zbaZIaBGwBRWeSvoMD4mgPdX3e4NWBg== diff --git a/ext/exporters.d.ts b/ext/exporters.d.ts index 2fed24f39ea..2f462dd93e7 100644 --- a/ext/exporters.d.ts +++ b/ext/exporters.d.ts @@ -3,7 +3,9 @@ declare const exporters: { AGENT: 'agent', DATADOG: 'datadog', AGENT_PROXY: 'agent_proxy', - JEST_WORKER: 'jest_worker' + JEST_WORKER: 'jest_worker', + CUCUMBER_WORKER: 'cucumber_worker', + MOCHA_WORKER: 'mocha_worker' } export = exporters diff --git a/ext/exporters.js b/ext/exporters.js index 82425b2a550..770116c3152 100644 --- a/ext/exporters.js +++ b/ext/exporters.js @@ -4,5 +4,7 @@ module.exports = { AGENT: 'agent', DATADOG: 'datadog', AGENT_PROXY: 'agent_proxy', - JEST_WORKER: 'jest_worker' + JEST_WORKER: 'jest_worker', + CUCUMBER_WORKER: 'cucumber_worker', + MOCHA_WORKER: 'mocha_worker' } diff --git a/ext/formats.d.ts b/ext/formats.d.ts index be4f361230a..fe1f3852185 100644 --- a/ext/formats.d.ts +++ b/ext/formats.d.ts @@ -5,6 +5,7 @@ declare const formats: { HTTP_HEADERS: typeof opentracing.FORMAT_HTTP_HEADERS BINARY: typeof opentracing.FORMAT_BINARY LOG: 'log' + TEXT_MAP_DSM: 'text_map_dsm' } export = formats diff --git a/ext/formats.js b/ext/formats.js index 3c755fc88cc..5dd7f5506c7 100644 --- a/ext/formats.js +++ b/ext/formats.js @@ -4,5 +4,6 @@ module.exports = { TEXT_MAP: 'text_map', HTTP_HEADERS: 'http_headers', BINARY: 'binary', - LOG: 'log' + LOG: 'log', + TEXT_MAP_DSM: 'text_map_dsm' } diff --git a/ext/tags.d.ts b/ext/tags.d.ts index 0aafd03138f..1acf4f4f38e 100644 --- a/ext/tags.d.ts +++ b/ext/tags.d.ts @@ -10,6 +10,7 @@ declare const tags: { MANUAL_DROP: 'manual.drop' MEASURED: '_dd.measured' BASE_SERVICE: '_dd.base_service' + DD_PARENT_ID: '_dd.parent_id' HTTP_URL: 'http.url' HTTP_METHOD: 'http.method' HTTP_STATUS_CODE: 'http.status_code' diff --git a/ext/tags.js b/ext/tags.js index e270a6bde3a..c12aa1a57dc 100644 --- a/ext/tags.js +++ b/ext/tags.js @@ -13,6 +13,7 @@ const tags = { MANUAL_DROP: 'manual.drop', MEASURED: '_dd.measured', BASE_SERVICE: '_dd.base_service', + DD_PARENT_ID: '_dd.parent_id', // HTTP HTTP_URL: 'http.url', diff --git a/ext/types.d.ts b/ext/types.d.ts index 703d88f794b..549f4d58ec1 100644 --- a/ext/types.d.ts +++ b/ext/types.d.ts @@ -1,5 +1,6 @@ declare const types: { HTTP: 'http' + SERVERLESS: 'serverless' WEB: 'web' } diff --git a/ext/types.js b/ext/types.js index d8863f04bb2..884b6a495e5 100644 --- a/ext/types.js +++ b/ext/types.js @@ -2,5 +2,6 @@ module.exports = { HTTP: 'http', + SERVERLESS: 'serverless', WEB: 'web' } diff --git a/index.d.ts b/index.d.ts index f84328ea25c..2e5aa4c57a8 100644 --- a/index.d.ts +++ b/index.d.ts @@ -1,22 +1,29 @@ import { ClientRequest, IncomingMessage, OutgoingMessage, ServerResponse } from "http"; import { LookupFunction } from 'net'; import * as opentracing from "opentracing"; -import { SpanOptions } from "opentracing/lib/tracer"; import * as otel from "@opentelemetry/api"; -export { SpanOptions }; - /** * Tracer is the entry-point of the Datadog tracing implementation. */ -export declare interface Tracer extends opentracing.Tracer { +interface Tracer extends opentracing.Tracer { + /** + * Add tracer as a named export + */ + tracer: Tracer; + + /** + * For compatibility with NodeNext + esModuleInterop: false + */ + default: Tracer; + /** * Starts and returns a new Span representing a logical unit of work. * @param {string} name The name of the operation. - * @param {SpanOptions} [options] Options for the newly created span. + * @param {tracer.SpanOptions} [options] Options for the newly created span. * @returns {Span} A new Span object. */ - startSpan (name: string, options?: SpanOptions): Span; + startSpan (name: string, options?: tracer.SpanOptions): tracer.Span; /** * Injects the given SpanContext instance for cross-process propagation @@ -28,7 +35,7 @@ export declare interface Tracer extends opentracing.Tracer { * @param {string} format The format of the carrier. * @param {any} carrier The carrier object. */ - inject (spanContext: SpanContext | Span, format: string, carrier: any): void; + inject (spanContext: tracer.SpanContext | tracer.Span, format: string, carrier: any): void; /** * Returns a SpanContext instance extracted from `carrier` in the given @@ -39,12 +46,12 @@ export declare interface Tracer extends opentracing.Tracer { * The extracted SpanContext, or null if no such SpanContext could * be found in `carrier` */ - extract (format: string, carrier: any): SpanContext | null; + extract (format: string, carrier: any): tracer.SpanContext | null; /** * Initializes the tracer. This should be called before importing other libraries. */ - init (options?: TracerOptions): this; + init (options?: tracer.TracerOptions): this; /** * Sets the URL for the trace agent. This should only be called _after_ @@ -63,7 +70,7 @@ export declare interface Tracer extends opentracing.Tracer { /** * Returns a reference to the current scope. */ - scope (): Scope; + scope (): tracer.Scope; /** * Instruments a function by automatically creating a span activated on its @@ -83,8 +90,9 @@ export declare interface Tracer extends opentracing.Tracer { * unless there is already an active span or `childOf` option. Note that this * option is deprecated and has been removed in version 4.0. */ - trace (name: string, fn: (span?: Span, fn?: (error?: Error) => any) => T): T; - trace (name: string, options: TraceOptions & SpanOptions, fn: (span?: Span, done?: (error?: Error) => string) => T): T; + trace (name: string, fn: (span: tracer.Span) => T): T; + trace (name: string, fn: (span: tracer.Span, done: (error?: Error) => void) => T): T; + trace (name: string, options: tracer.TraceOptions & tracer.SpanOptions, fn: (span?: tracer.Span, done?: (error?: Error) => void) => T): T; /** * Wrap a function to automatically create a span activated on its @@ -101,13 +109,19 @@ export declare interface Tracer extends opentracing.Tracer { * which case the span will finish at the end of the function execution. */ wrap any> (name: string, fn: T): T; - wrap any> (name: string, options: TraceOptions & SpanOptions, fn: T): T; - wrap any> (name: string, options: (...args: any[]) => TraceOptions & SpanOptions, fn: T): T; + wrap any> (name: string, options: tracer.TraceOptions & tracer.SpanOptions, fn: T): T; + wrap any> (name: string, options: (...args: any[]) => tracer.TraceOptions & tracer.SpanOptions, fn: T): T; /** - * Create and return a string that can be included in the of a - * document to enable RUM tracing to include it. The resulting string - * should not be cached. + * Returns an HTML string containing tags that should be included in + * the of a document to enable correlating the current trace with the + * RUM view. Otherwise, it is not possible to associate the trace used to + * generate the initial HTML document with a given RUM view. The resulting + * HTML document should not be cached as the meta tags are time-sensitive + * and are associated with a specific user. + * + * Note that this feature is currently not supported by the backend and + * using it will have no effect. */ getRumData (): string; @@ -116,1851 +130,2074 @@ export declare interface Tracer extends opentracing.Tracer { * @param {User} user Properties of the authenticated user. Accepts custom fields. * @returns {Tracer} The Tracer instance for chaining. */ - setUser (user: User): Tracer; + setUser (user: tracer.User): Tracer; - appsec: Appsec; + appsec: tracer.Appsec; - TracerProvider: opentelemetry.TracerProvider; + TracerProvider: tracer.opentelemetry.TracerProvider; - dogstatsd: DogStatsD; + dogstatsd: tracer.DogStatsD; } -export declare interface TraceOptions extends Analyzable { - /** - * The resource you are tracing. The resource name must not be longer than - * 5000 characters. - */ - resource?: string, - - /** - * The service you are tracing. The service name must not be longer than - * 100 characters. - */ - service?: string, - - /** - * The type of request. - */ - type?: string +// left out of the namespace, so it +// is doesn't need to be exported for Tracer +/** @hidden */ +interface Plugins { + "aerospike": tracer.plugins.aerospike; + "amqp10": tracer.plugins.amqp10; + "amqplib": tracer.plugins.amqplib; + "apollo": tracer.plugins.apollo; + "avsc": tracer.plugins.avsc; + "aws-sdk": tracer.plugins.aws_sdk; + "azure-functions": tracer.plugins.azure_functions; + "bunyan": tracer.plugins.bunyan; + "cassandra-driver": tracer.plugins.cassandra_driver; + "child_process": tracer.plugins.child_process; + "connect": tracer.plugins.connect; + "couchbase": tracer.plugins.couchbase; + "cucumber": tracer.plugins.cucumber; + "cypress": tracer.plugins.cypress; + "dns": tracer.plugins.dns; + "elasticsearch": tracer.plugins.elasticsearch; + "express": tracer.plugins.express; + "fastify": tracer.plugins.fastify; + "fetch": tracer.plugins.fetch; + "generic-pool": tracer.plugins.generic_pool; + "google-cloud-pubsub": tracer.plugins.google_cloud_pubsub; + "graphql": tracer.plugins.graphql; + "grpc": tracer.plugins.grpc; + "hapi": tracer.plugins.hapi; + "http": tracer.plugins.http; + "http2": tracer.plugins.http2; + "ioredis": tracer.plugins.ioredis; + "jest": tracer.plugins.jest; + "kafkajs": tracer.plugins.kafkajs + "knex": tracer.plugins.knex; + "koa": tracer.plugins.koa; + "mariadb": tracer.plugins.mariadb; + "memcached": tracer.plugins.memcached; + "microgateway-core": tracer.plugins.microgateway_core; + "mocha": tracer.plugins.mocha; + "moleculer": tracer.plugins.moleculer; + "mongodb-core": tracer.plugins.mongodb_core; + "mongoose": tracer.plugins.mongoose; + "mysql": tracer.plugins.mysql; + "mysql2": tracer.plugins.mysql2; + "net": tracer.plugins.net; + "next": tracer.plugins.next; + "openai": tracer.plugins.openai; + "opensearch": tracer.plugins.opensearch; + "oracledb": tracer.plugins.oracledb; + "paperplane": tracer.plugins.paperplane; + "playwright": tracer.plugins.playwright; + "pg": tracer.plugins.pg; + "pino": tracer.plugins.pino; + "protobufjs": tracer.plugins.protobufjs; + "redis": tracer.plugins.redis; + "restify": tracer.plugins.restify; + "rhea": tracer.plugins.rhea; + "router": tracer.plugins.router; + "selenium": tracer.plugins.selenium; + "sharedb": tracer.plugins.sharedb; + "tedious": tracer.plugins.tedious; + "undici": tracer.plugins.undici; + "vitest": tracer.plugins.vitest; + "winston": tracer.plugins.winston; } -/** - * Span represents a logical unit of work as part of a broader Trace. - * Examples of span might include remote procedure calls or a in-process - * function calls to sub-components. A Trace has a single, top-level "root" - * Span that in turn may have zero or more child Spans, which in turn may - * have children. - */ -export declare interface Span extends opentracing.Span { - context (): SpanContext; -} +declare namespace tracer { + export type SpanOptions = opentracing.SpanOptions; + export { Tracer }; -/** - * SpanContext represents Span state that must propagate to descendant Spans - * and across process boundaries. - * - * SpanContext is logically divided into two pieces: the user-level "Baggage" - * (see setBaggageItem and getBaggageItem) that propagates across Span - * boundaries and any Tracer-implementation-specific fields that are needed to - * identify or otherwise contextualize the associated Span instance (e.g., a - * tuple). - */ -export declare interface SpanContext extends opentracing.SpanContext { - /** - * Returns the string representation of the internal trace ID. - */ - toTraceId (): string; + export interface TraceOptions extends Analyzable { + /** + * The resource you are tracing. The resource name must not be longer than + * 5000 characters. + */ + resource?: string, - /** - * Returns the string representation of the internal span ID. - */ - toSpanId (): string; + /** + * The service you are tracing. The service name must not be longer than + * 100 characters. + */ + service?: string, - /** - * Returns the string representation used for DBM integration. - */ - toTraceparent (): string; -} + /** + * The type of request. + */ + type?: string -/** - * Sampling rule to configure on the priority sampler. - */ -export declare interface SamplingRule { - /** - * Sampling rate for this rule. - */ - sampleRate: number + /** + * An array of span links + */ + links?: Array<{ context: SpanContext, attributes?: Object }> + } /** - * Service on which to apply this rule. The rule will apply to all services if not provided. + * Span represents a logical unit of work as part of a broader Trace. + * Examples of span might include remote procedure calls or a in-process + * function calls to sub-components. A Trace has a single, top-level "root" + * Span that in turn may have zero or more child Spans, which in turn may + * have children. */ - service?: string | RegExp + export interface Span extends opentracing.Span { + context (): SpanContext; - /** - * Operation name on which to apply this rule. The rule will apply to all operation names if not provided. - */ - name?: string | RegExp -} + /** + * Causally links another span to the current span + * @param {SpanContext} context The context of the span to link to. + * @param {Object} attributes An optional key value pair of arbitrary values. + * @returns {void} + */ + addLink (context: SpanContext, attributes?: Object): void; + } -/** - * Span sampling rules to ingest single spans where the enclosing trace is dropped - */ -export declare interface SpanSamplingRule { /** - * Sampling rate for this rule. Will default to 1.0 (always) if not provided. + * SpanContext represents Span state that must propagate to descendant Spans + * and across process boundaries. + * + * SpanContext is logically divided into two pieces: the user-level "Baggage" + * (see setBaggageItem and getBaggageItem) that propagates across Span + * boundaries and any Tracer-implementation-specific fields that are needed to + * identify or otherwise contextualize the associated Span instance (e.g., a + * tuple). */ - sampleRate?: number + export interface SpanContext extends opentracing.SpanContext { + /** + * Returns the string representation of the internal trace ID. + */ + toTraceId (): string; - /** - * Maximum number of spans matching a span sampling rule to be allowed per second. - */ - maxPerSecond?: number + /** + * Returns the string representation of the internal span ID. + */ + toSpanId (): string; - /** - * Service name or pattern on which to apply this rule. The rule will apply to all services if not provided. - */ - service?: string + /** + * Returns the string representation used for DBM integration. + */ + toTraceparent (): string; + } /** - * Operation name or pattern on which to apply this rule. The rule will apply to all operation names if not provided. + * Sampling rule to configure on the priority sampler. */ - name?: string -} + export interface SamplingRule { + /** + * Sampling rate for this rule. + */ + sampleRate: number -/** - * Selection and priority order of context propagation injection and extraction mechanisms. - */ -export declare interface PropagationStyle { - /** - * Selection of context propagation injection mechanisms. - */ - inject: string[], + /** + * Service on which to apply this rule. The rule will apply to all services if not provided. + */ + service?: string | RegExp - /** - * Selection and priority order of context propagation extraction mechanisms. - */ - extract: string[] -} + /** + * Operation name on which to apply this rule. The rule will apply to all operation names if not provided. + */ + name?: string | RegExp + } -/** - * List of options available to the tracer. - */ -export declare interface TracerOptions { /** - * Whether to enable trace ID injection in log records to be able to correlate - * traces with logs. - * @default false + * Span sampling rules to ingest single spans where the enclosing trace is dropped */ - logInjection?: boolean, + export interface SpanSamplingRule { + /** + * Sampling rate for this rule. Will default to 1.0 (always) if not provided. + */ + sampleRate?: number - /** - * Whether to enable startup logs. - * @default true - */ - startupLogs?: boolean, + /** + * Maximum number of spans matching a span sampling rule to be allowed per second. + */ + maxPerSecond?: number - /** - * The service name to be used for this program. If not set, the service name - * will attempted to be inferred from package.json - */ - service?: string; + /** + * Service name or pattern on which to apply this rule. The rule will apply to all services if not provided. + */ + service?: string - /** - * Provide service name mappings for each plugin. - */ - serviceMapping?: { [key: string]: string }; + /** + * Operation name or pattern on which to apply this rule. The rule will apply to all operation names if not provided. + */ + name?: string + } /** - * The url of the trace agent that the tracer will submit to. - * Takes priority over hostname and port, if set. + * Selection and priority order of context propagation injection and extraction mechanisms. */ - url?: string; + export interface PropagationStyle { + /** + * Selection of context propagation injection mechanisms. + */ + inject: string[], - /** - * The address of the trace agent that the tracer will submit to. - * @default 'localhost' - */ - hostname?: string; + /** + * Selection and priority order of context propagation extraction mechanisms. + */ + extract: string[] + } /** - * The port of the trace agent that the tracer will submit to. - * @default 8126 + * List of options available to the tracer. */ - port?: number | string; + export interface TracerOptions { + /** + * Whether to enable trace ID injection in log records to be able to correlate + * traces with logs. + * @default false + */ + logInjection?: boolean, - /** - * Whether to enable profiling. - */ - profiling?: boolean + /** + * Whether to enable startup logs. + * @default true + */ + startupLogs?: boolean, - /** - * Options specific for the Dogstatsd agent. - */ - dogstatsd?: { /** - * The hostname of the Dogstatsd agent that the metrics will submitted to. + * The service name to be used for this program. If not set, the service name + * will attempted to be inferred from package.json */ - hostname?: string + service?: string; /** - * The port of the Dogstatsd agent that the metrics will submitted to. - * @default 8125 + * Provide service name mappings for each plugin. */ - port?: number - }; + serviceMapping?: { [key: string]: string }; - /** - * Set an application’s environment e.g. prod, pre-prod, stage. - */ - env?: string; + /** + * The url of the trace agent that the tracer will submit to. + * Takes priority over hostname and port, if set. + */ + url?: string; - /** - * The version number of the application. If not set, the version - * will attempted to be inferred from package.json. - */ - version?: string; + /** + * The address of the trace agent that the tracer will submit to. + * @default 'localhost' + */ + hostname?: string; - /** - * Controls the ingestion sample rate (between 0 and 1) between the agent and the backend. - */ - sampleRate?: number; + /** + * The port of the trace agent that the tracer will submit to. + * @default 8126 + */ + port?: number | string; - /** - * Global rate limit that is applied on the global sample rate and all rules, - * and controls the ingestion rate limit between the agent and the backend. - * Defaults to deferring the decision to the agent. - */ - rateLimit?: number, + /** + * Whether to enable profiling. + */ + profiling?: boolean - /** - * Sampling rules to apply to priority samplin. Each rule is a JSON, - * consisting of `service` and `name`, which are regexes to match against - * a trace's `service` and `name`, and a corresponding `sampleRate`. If not - * specified, will defer to global sampling rate for all spans. - * @default [] - */ - samplingRules?: SamplingRule[] + /** + * Options specific for the Dogstatsd agent. + */ + dogstatsd?: { + /** + * The hostname of the Dogstatsd agent that the metrics will submitted to. + */ + hostname?: string - /** - * Span sampling rules that take effect when the enclosing trace is dropped, to ingest single spans - * @default [] - */ - spanSamplingRules?: SpanSamplingRule[] + /** + * The port of the Dogstatsd agent that the metrics will submitted to. + * @default 8125 + */ + port?: number + }; - /** - * Interval in milliseconds at which the tracer will submit traces to the agent. - * @default 2000 - */ - flushInterval?: number; + /** + * Set an application’s environment e.g. prod, pre-prod, stage. + */ + env?: string; - /** - * Number of spans before partially exporting a trace. This prevents keeping all the spans in memory for very large traces. - * @default 1000 - */ - flushMinSpans?: number; + /** + * The version number of the application. If not set, the version + * will attempted to be inferred from package.json. + */ + version?: string; - /** - * Whether to enable runtime metrics. - * @default false - */ - runtimeMetrics?: boolean + /** + * Controls the ingestion sample rate (between 0 and 1) between the agent and the backend. + */ + sampleRate?: number; - /** - * Custom function for DNS lookups when sending requests to the agent. - * @default dns.lookup() - */ - lookup?: LookupFunction + /** + * Global rate limit that is applied on the global sample rate and all rules, + * and controls the ingestion rate limit between the agent and the backend. + * Defaults to deferring the decision to the agent. + */ + rateLimit?: number, - /** - * Protocol version to use for requests to the agent. The version configured must be supported by the agent version installed or all traces will be dropped. - * @default 0.4 - */ - protocolVersion?: string + /** + * Sampling rules to apply to priority samplin. Each rule is a JSON, + * consisting of `service` and `name`, which are regexes to match against + * a trace's `service` and `name`, and a corresponding `sampleRate`. If not + * specified, will defer to global sampling rate for all spans. + * @default [] + */ + samplingRules?: SamplingRule[] - /** - * Deprecated in favor of the global versions of the variables provided under this option - * - * @deprecated - * @hidden - */ - ingestion?: { /** - * Controls the ingestion sample rate (between 0 and 1) between the agent and the backend. + * Span sampling rules that take effect when the enclosing trace is dropped, to ingest single spans + * @default [] */ - sampleRate?: number + spanSamplingRules?: SpanSamplingRule[] /** - * Controls the ingestion rate limit between the agent and the backend. Defaults to deferring the decision to the agent. + * Interval in milliseconds at which the tracer will submit traces to the agent. + * @default 2000 */ - rateLimit?: number - }; + flushInterval?: number; - /** - * Experimental features can be enabled individually using key / value pairs. - * @default {} - */ - experimental?: { - b3?: boolean - traceparent?: boolean + /** + * Number of spans before partially exporting a trace. This prevents keeping all the spans in memory for very large traces. + * @default 1000 + */ + flushMinSpans?: number; /** - * Whether to add an auto-generated `runtime-id` tag to metrics. + * Whether to enable runtime metrics. * @default false */ - runtimeId?: boolean + runtimeMetrics?: boolean /** - * Whether to write traces to log output or agentless, rather than send to an agent - * @default false + * Custom function for DNS lookups when sending requests to the agent. + * @default dns.lookup() */ - exporter?: 'log' | 'agent' | 'datadog' + lookup?: LookupFunction /** - * Whether to enable the experimental `getRumData` method. - * @default false + * Protocol version to use for requests to the agent. The version configured must be supported by the agent version installed or all traces will be dropped. + * @default 0.4 */ - enableGetRumData?: boolean + protocolVersion?: string /** - * Configuration of the IAST. Can be a boolean as an alias to `iast.enabled`. + * Deprecated in favor of the global versions of the variables provided under this option + * + * @deprecated + * @hidden */ - iast?: boolean | { - /** - * Whether to enable IAST. - * @default false - */ - enabled?: boolean, - /** - * Controls the percentage of requests that iast will analyze - * @default 30 - */ - requestSampling?: number, + ingestion?: { /** - * Controls how many request can be analyzing code vulnerabilities at the same time - * @default 2 + * Controls the ingestion sample rate (between 0 and 1) between the agent and the backend. */ - maxConcurrentRequests?: number, + sampleRate?: number + /** - * Controls how many code vulnerabilities can be detected in the same request - * @default 2 + * Controls the ingestion rate limit between the agent and the backend. Defaults to deferring the decision to the agent. */ - maxContextOperations?: number, + rateLimit?: number + }; + + /** + * Experimental features can be enabled individually using key / value pairs. + * @default {} + */ + experimental?: { + b3?: boolean + traceparent?: boolean + /** - * Whether to enable vulnerability deduplication + * Whether to add an auto-generated `runtime-id` tag to metrics. + * @default false */ - deduplicationEnabled?: boolean + runtimeId?: boolean + /** - * Whether to enable vulnerability redaction - * @default true + * Whether to write traces to log output or agentless, rather than send to an agent + * @default false */ - redactionEnabled?: boolean, + exporter?: 'log' | 'agent' | 'datadog' + /** - * Specifies a regex that will redact sensitive source names in vulnerability reports. + * Whether to enable the experimental `getRumData` method. + * @default false */ - redactionNamePattern?: string, + enableGetRumData?: boolean + /** - * Specifies a regex that will redact sensitive source values in vulnerability reports. + * Configuration of the IAST. Can be a boolean as an alias to `iast.enabled`. */ - redactionValuePattern?: string - } - }; - - /** - * Whether to load all built-in plugins. - * @default true - */ - plugins?: boolean; - - /** - * Custom logger to be used by the tracer (if debug = true), - * should support error(), warn(), info(), and debug() methods - * see https://datadog.github.io/dd-trace-js/#custom-logging - */ - logger?: { - error: (err: Error | string) => void; - warn: (message: string) => void; - info: (message: string) => void; - debug: (message: string) => void; - }; - - /** - * Global tags that should be assigned to every span. - */ - tags?: { [key: string]: any }; - - /** - * Specifies which scope implementation to use. The default is to use the best - * implementation for the runtime. Only change this if you know what you are - * doing. - */ - scope?: 'async_hooks' | 'async_local_storage' | 'async_resource' | 'sync' | 'noop' - - /** - * Whether to report the hostname of the service host. This is used when the agent is deployed on a different host and cannot determine the hostname automatically. - * @default false - */ - reportHostname?: boolean - - /** - * A string representing the minimum tracer log level to use when debug logging is enabled - * @default 'debug' - */ - logLevel?: 'error' | 'debug' - - /** - * If false, require a parent in order to trace. - * @default true - * @deprecated since version 4.0 - */ - orphanable?: boolean - - /** - * Enables DBM to APM link using tag injection. - * @default 'disabled' - */ - dbmPropagationMode?: 'disabled' | 'service' | 'full' + iast?: boolean | IastOptions + + appsec?: { + /** + * Configuration of Standalone ASM mode + */ + standalone?: { + /** + * Whether to enable Standalone ASM. + * @default false + */ + enabled?: boolean + } + } + }; - /** - * Configuration of the AppSec protection. Can be a boolean as an alias to `appsec.enabled`. - */ - appsec?: boolean | { /** - * Whether to enable AppSec. - * @default false + * Whether to load all built-in plugins. + * @default true */ - enabled?: boolean, + plugins?: boolean; /** - * Specifies a path to a custom rules file. + * Custom logger to be used by the tracer (if debug = true), + * should support error(), warn(), info(), and debug() methods + * see https://datadog.github.io/dd-trace-js/#custom-logging */ - rules?: string, + logger?: { + error: (err: Error | string) => void; + warn: (message: string) => void; + info: (message: string) => void; + debug: (message: string) => void; + }; /** - * Controls the maximum amount of traces sampled by AppSec attacks, per second. - * @default 100 + * Global tags that should be assigned to every span. */ - rateLimit?: number, + tags?: { [key: string]: any }; /** - * Controls the maximum amount of time in microseconds the WAF is allowed to run synchronously for. - * @default 5000 + * Specifies which scope implementation to use. The default is to use the best + * implementation for the runtime. Only change this if you know what you are + * doing. */ - wafTimeout?: number, + scope?: 'async_hooks' | 'async_local_storage' | 'async_resource' | 'sync' | 'noop' /** - * Specifies a regex that will redact sensitive data by its key in attack reports. + * Whether to report the hostname of the service host. This is used when the agent is deployed on a different host and cannot determine the hostname automatically. + * @default false */ - obfuscatorKeyRegex?: string, + reportHostname?: boolean /** - * Specifies a regex that will redact sensitive data by its value in attack reports. + * A string representing the minimum tracer log level to use when debug logging is enabled + * @default 'debug' */ - obfuscatorValueRegex?: string, + logLevel?: 'error' | 'debug' /** - * Specifies a path to a custom blocking template html file. + * If false, require a parent in order to trace. + * @default true + * @deprecated since version 4.0 */ - blockedTemplateHtml?: string, + orphanable?: boolean /** - * Specifies a path to a custom blocking template json file. + * Enables DBM to APM link using tag injection. + * @default 'disabled' */ - blockedTemplateJson?: string, + dbmPropagationMode?: 'disabled' | 'service' | 'full' /** - * Controls the automated user event tracking configuration + * Configuration of the AppSec protection. Can be a boolean as an alias to `appsec.enabled`. */ - eventTracking?: { + appsec?: boolean | { /** - * Controls the automated user event tracking mode. Possible values are disabled, safe and extended. - * On safe mode, any detected Personally Identifiable Information (PII) about the user will be redacted from the event. - * On extended mode, no redaction will take place. - * @default 'safe' - */ - mode?: 'safe' | 'extended' | 'disabled' - }, - - /** - * Configuration for Api Security sampling - */ - apiSecurity?: { - /** Whether to enable Api Security. + * Whether to enable AppSec. * @default false */ enabled?: boolean, - /** Controls the request sampling rate (between 0 and 1) in which Api Security is triggered. - * The value will be coerced back if it's outside of the 0-1 range. - * @default 0.1 + /** + * Specifies a path to a custom rules file. */ - requestSampling?: number - } - }; + rules?: string, - /** - * Configuration of ASM Remote Configuration - */ - remoteConfig?: { - /** - * Specifies the remote configuration polling interval in seconds - * @default 5 - */ - pollInterval?: number, - } + /** + * Controls the maximum amount of traces sampled by AppSec attacks, per second. + * @default 100 + */ + rateLimit?: number, - /** - * Whether to enable client IP collection from relevant IP headers - * @default false - */ - clientIpEnabled?: boolean + /** + * Controls the maximum amount of time in microseconds the WAF is allowed to run synchronously for. + * @default 5000 + */ + wafTimeout?: number, - /** - * Custom header name to source the http.client_ip tag from. - */ - clientIpHeader?: string, - - /** - * The selection and priority order of context propagation injection and extraction mechanisms. - */ - propagationStyle?: string[] | PropagationStyle -} - -/** - * User object that can be passed to `tracer.setUser()`. - */ -export declare interface User { - /** - * Unique identifier of the user. - * Mandatory. - */ - id: string, - - /** - * Email of the user. - */ - email?: string, - - /** - * User-friendly name of the user. - */ - name?: string, - - /** - * Session ID of the user. - */ - session_id?: string, - - /** - * Role the user is making the request under. - */ - role?: string, + /** + * Specifies a regex that will redact sensitive data by its key in attack reports. + */ + obfuscatorKeyRegex?: string, - /** - * Scopes or granted authorizations the user currently possesses. - * The value could come from the scope associated with an OAuth2 - * Access Token or an attribute value in a SAML 2 Assertion. - */ - scope?: string, + /** + * Specifies a regex that will redact sensitive data by its value in attack reports. + */ + obfuscatorValueRegex?: string, - /** - * Custom fields to attach to the user (RBAC, Oauth, etc…). - */ - [key: string]: string | undefined -} + /** + * Specifies a path to a custom blocking template html file. + */ + blockedTemplateHtml?: string, -export declare interface DogStatsD { - /** - * Increments a metric by the specified value, optionally specifying tags. - * @param {string} stat The dot-separated metric name. - * @param {number} value The amount to increment the stat by. - * @param {[tag:string]:string|number} tags Tags to pass along, such as `{ foo: 'bar' }`. Values are combined with config.tags. - */ - increment(stat: string, value?: number, tags?: { [tag: string]: string|number }): void + /** + * Specifies a path to a custom blocking template json file. + */ + blockedTemplateJson?: string, - /** - * Decrements a metric by the specified value, optionally specifying tags. - * @param {string} stat The dot-separated metric name. - * @param {number} value The amount to decrement the stat by. - * @param {[tag:string]:string|number} tags Tags to pass along, such as `{ foo: 'bar' }`. Values are combined with config.tags. - */ - decrement(stat: string, value?: number, tags?: { [tag: string]: string|number }): void + /** + * Specifies a path to a custom blocking template json file for graphql requests + */ + blockedTemplateGraphql?: string, - /** - * Sets a distribution value, optionally specifying tags. - * @param {string} stat The dot-separated metric name. - * @param {number} value The amount to increment the stat by. - * @param {[tag:string]:string|number} tags Tags to pass along, such as `{ foo: 'bar' }`. Values are combined with config.tags. - */ - distribution(stat: string, value?: number, tags?: { [tag: string]: string|number }): void + /** + * Controls the automated user event tracking configuration + */ + eventTracking?: { + /** + * Controls the automated user event tracking mode. Possible values are disabled, safe and extended. + * On safe mode, any detected Personally Identifiable Information (PII) about the user will be redacted from the event. + * On extended mode, no redaction will take place. + * @default 'safe' + */ + mode?: 'safe' | 'extended' | 'disabled' + }, + /** + * Configuration for Api Security sampling + */ + apiSecurity?: { + /** Whether to enable Api Security. + * @default false + */ + enabled?: boolean, + + /** Controls the request sampling rate (between 0 and 1) in which Api Security is triggered. + * The value will be coerced back if it's outside of the 0-1 range. + * @default 0.1 + */ + requestSampling?: number + }, + /** + * Configuration for RASP + */ + rasp?: { + /** Whether to enable RASP. + * @default false + */ + enabled?: boolean + }, + /** + * Configuration for stack trace reporting + */ + stackTrace?: { + /** Whether to enable stack trace reporting. + * @default true + */ + enabled?: boolean, + + /** Specifies the maximum number of stack traces to be reported. + * @default 2 + */ + maxStackTraces?: number, + + /** Specifies the maximum depth of a stack trace to be reported. + * @default 32 + */ + maxDepth?: number, + } + }; - /** - * Sets a gauge value, optionally specifying tags. - * @param {string} stat The dot-separated metric name. - * @param {number} value The amount to increment the stat by. - * @param {[tag:string]:string|number} tags Tags to pass along, such as `{ foo: 'bar' }`. Values are combined with config.tags. - */ - gauge(stat: string, value?: number, tags?: { [tag: string]: string|number }): void + /** + * Configuration of the IAST. Can be a boolean as an alias to `iast.enabled`. + */ + iast?: boolean | IastOptions - /** - * Forces any unsent metrics to be sent - * - * @beta This method is experimental and could be removed in future versions. - */ - flush(): void -} + /** + * Configuration of ASM Remote Configuration + */ + remoteConfig?: { + /** + * Specifies the remote configuration polling interval in seconds + * @default 5 + */ + pollInterval?: number, + } -export declare interface Appsec { - /** - * Links a successful login event to the current trace. Will link the passed user to the current trace with Appsec.setUser() internally. - * @param {User} user Properties of the authenticated user. Accepts custom fields. - * @param {[key: string]: string} metadata Custom fields to link to the login success event. - * - * @beta This method is in beta and could change in future versions. - */ - trackUserLoginSuccessEvent(user: User, metadata?: { [key: string]: string }): void + /** + * Whether to enable client IP collection from relevant IP headers + * @default false + */ + clientIpEnabled?: boolean - /** - * Links a failed login event to the current trace. - * @param {string} userId The user id of the attemped login. - * @param {boolean} exists If the user id exists. - * @param {[key: string]: string} metadata Custom fields to link to the login failure event. - * - * @beta This method is in beta and could change in future versions. - */ - trackUserLoginFailureEvent(userId: string, exists: boolean, metadata?: { [key: string]: string }): void + /** + * Custom header name to source the http.client_ip tag from. + */ + clientIpHeader?: string, - /** - * Links a custom event to the current trace. - * @param {string} eventName The name of the event. - * @param {[key: string]: string} metadata Custom fields to link to the event. - * - * @beta This method is in beta and could change in future versions. - */ - trackCustomEvent(eventName: string, metadata?: { [key: string]: string }): void + /** + * The selection and priority order of context propagation injection and extraction mechanisms. + */ + propagationStyle?: string[] | PropagationStyle - /** - * Checks if the passed user should be blocked according to AppSec rules. - * If no user is linked to the current trace, will link the passed user to it. - * @param {User} user Properties of the authenticated user. Accepts custom fields. - * @return {boolean} Indicates whether the user should be blocked. - * - * @beta This method is in beta and could change in the future - */ - isUserBlocked(user: User): boolean + /** + * Cloud payload report as tags + */ + cloudPayloadTagging?: { + /** + * Additional JSONPath queries to replace with `redacted` in request payloads + * Undefined or invalid JSONPath queries disable the feature for requests. + */ + request?: string, + /** + * Additional JSONPath queries to replace with `redacted` in response payloads + * Undefined or invalid JSONPath queries disable the feature for responses. + */ + response?: string, + /** + * Maximum depth of payload traversal for tags + */ + maxDepth?: number + } + } /** - * Sends a "blocked" template response based on the request accept header and ends the response. - * **You should stop processing the request after calling this function!** - * @param {IncomingMessage} req Can be passed to force which request to act on. Optional. - * @param {OutgoingMessage} res Can be passed to force which response to act on. Optional. - * @return {boolean} Indicates if the action was successful. - * - * @beta This method is in beta and could change in the future + * User object that can be passed to `tracer.setUser()`. */ - blockRequest(req?: IncomingMessage, res?: OutgoingMessage): boolean + export interface User { + /** + * Unique identifier of the user. + * Mandatory. + */ + id: string, - /** - * Links an authenticated user to the current trace. - * @param {User} user Properties of the authenticated user. Accepts custom fields. - * - * @beta This method is in beta and could change in the future - */ - setUser(user: User): void -} + /** + * Email of the user. + */ + email?: string, -/** @hidden */ -declare type anyObject = { - [key: string]: any; -}; + /** + * User-friendly name of the user. + */ + name?: string, -/** @hidden */ -interface TransportRequestParams { - method: string; - path: string; - body?: anyObject; - bulkBody?: anyObject; - querystring?: anyObject; -} + /** + * Session ID of the user. + */ + session_id?: string, -/** - * The Datadog Scope Manager. This is used for context propagation. - */ -export declare interface Scope { - /** - * Get the current active span or null if there is none. - * - * @returns {Span} The active span. - */ - active (): Span | null; + /** + * Role the user is making the request under. + */ + role?: string, - /** - * Activate a span in the scope of a function. - * - * @param {Span} span The span to activate. - * @param {Function} fn Function that will have the span activated on its scope. - * @returns The return value of the provided function. - */ - activate (span: Span, fn: ((...args: any[]) => T)): T; + /** + * Scopes or granted authorizations the user currently possesses. + * The value could come from the scope associated with an OAuth2 + * Access Token or an attribute value in a SAML 2 Assertion. + */ + scope?: string, - /** - * Binds a target to the provided span, or the active span if omitted. - * - * @param {Function|Promise} target Target that will have the span activated on its scope. - * @param {Span} [span=scope.active()] The span to activate. - * @returns The bound target. - */ - bind void> (fn: T, span?: Span | null): T; - bind V> (fn: T, span?: Span | null): T; - bind (fn: Promise, span?: Span | null): Promise; -} + /** + * Custom fields to attach to the user (RBAC, Oauth, etc…). + */ + [key: string]: string | undefined + } -/** @hidden */ -interface Plugins { - "amqp10": plugins.amqp10; - "amqplib": plugins.amqplib; - "aws-sdk": plugins.aws_sdk; - "bunyan": plugins.bunyan; - "cassandra-driver": plugins.cassandra_driver; - "connect": plugins.connect; - "couchbase": plugins.couchbase; - "cucumber": plugins.cucumber; - "cypress": plugins.cypress; - "dns": plugins.dns; - "elasticsearch": plugins.elasticsearch; - "express": plugins.express; - "fastify": plugins.fastify; - "fetch": plugins.fetch; - "generic-pool": plugins.generic_pool; - "google-cloud-pubsub": plugins.google_cloud_pubsub; - "graphql": plugins.graphql; - "grpc": plugins.grpc; - "hapi": plugins.hapi; - "http": plugins.http; - "http2": plugins.http2; - "ioredis": plugins.ioredis; - "jest": plugins.jest; - "kafkajs": plugins.kafkajs - "knex": plugins.knex; - "koa": plugins.koa; - "mariadb": plugins.mariadb; - "memcached": plugins.memcached; - "microgateway-core": plugins.microgateway_core; - "mocha": plugins.mocha; - "moleculer": plugins.moleculer; - "mongodb-core": plugins.mongodb_core; - "mongoose": plugins.mongoose; - "mysql": plugins.mysql; - "mysql2": plugins.mysql2; - "net": plugins.net; - "next": plugins.next; - "openai": plugins.openai; - "opensearch": plugins.opensearch; - "oracledb": plugins.oracledb; - "paperplane": plugins.paperplane; - "playwright": plugins.playwright; - "pg": plugins.pg; - "pino": plugins.pino; - "redis": plugins.redis; - "restify": plugins.restify; - "rhea": plugins.rhea; - "router": plugins.router; - "sharedb": plugins.sharedb; - "tedious": plugins.tedious; - "winston": plugins.winston; -} + export interface DogStatsD { + /** + * Increments a metric by the specified value, optionally specifying tags. + * @param {string} stat The dot-separated metric name. + * @param {number} value The amount to increment the stat by. + * @param {[tag:string]:string|number} tags Tags to pass along, such as `{ foo: 'bar' }`. Values are combined with config.tags. + */ + increment(stat: string, value?: number, tags?: { [tag: string]: string|number }): void -/** @hidden */ -interface Analyzable { - /** - * Whether to measure the span. Can also be set to a key-value pair with span - * names as keys and booleans as values for more granular control. - */ - measured?: boolean | { [key: string]: boolean }; -} + /** + * Decrements a metric by the specified value, optionally specifying tags. + * @param {string} stat The dot-separated metric name. + * @param {number} value The amount to decrement the stat by. + * @param {[tag:string]:string|number} tags Tags to pass along, such as `{ foo: 'bar' }`. Values are combined with config.tags. + */ + decrement(stat: string, value?: number, tags?: { [tag: string]: string|number }): void -declare namespace plugins { - /** @hidden */ - interface Integration { /** - * The service name to be used for this plugin. + * Sets a distribution value, optionally specifying tags. + * @param {string} stat The dot-separated metric name. + * @param {number} value The amount to increment the stat by. + * @param {[tag:string]:string|number} tags Tags to pass along, such as `{ foo: 'bar' }`. Values are combined with config.tags. */ - service?: string | any; + distribution(stat: string, value?: number, tags?: { [tag: string]: string|number }): void - /** Whether to enable the plugin. - * @default true + /** + * Sets a gauge value, optionally specifying tags. + * @param {string} stat The dot-separated metric name. + * @param {number} value The amount to increment the stat by. + * @param {[tag:string]:string|number} tags Tags to pass along, such as `{ foo: 'bar' }`. Values are combined with config.tags. */ - enabled?: boolean; - } + gauge(stat: string, value?: number, tags?: { [tag: string]: string|number }): void - /** @hidden */ - interface Instrumentation extends Integration, Analyzable {} + /** + * Sets a histogram value, optionally specifying tags. + * @param {string} stat The dot-separated metric name. + * @param {number} value The amount to increment the stat by. + * @param {[tag:string]:string|number} tags Tags to pass along, such as `{ foo: 'bar' }`. Values are combined with config.tags. + */ + histogram(stat: string, value?: number, tags?: { [tag: string]: string|number }): void - /** @hidden */ - interface Http extends Instrumentation { /** - * List of URLs that should be instrumented. + * Forces any unsent metrics to be sent * - * @default /^.*$/ + * @beta This method is experimental and could be removed in future versions. */ - allowlist?: string | RegExp | ((url: string) => boolean) | (string | RegExp | ((url: string) => boolean))[]; + flush(): void + } + export interface Appsec { /** - * Deprecated in favor of `allowlist`. + * Links a successful login event to the current trace. Will link the passed user to the current trace with Appsec.setUser() internally. + * @param {User} user Properties of the authenticated user. Accepts custom fields. + * @param {[key: string]: string} metadata Custom fields to link to the login success event. * - * @deprecated - * @hidden + * @beta This method is in beta and could change in future versions. */ - whitelist?: string | RegExp | ((url: string) => boolean) | (string | RegExp | ((url: string) => boolean))[]; + trackUserLoginSuccessEvent(user: User, metadata?: { [key: string]: string }): void /** - * List of URLs that should not be instrumented. Takes precedence over - * allowlist if a URL matches an entry in both. + * Links a failed login event to the current trace. + * @param {string} userId The user id of the attemped login. + * @param {boolean} exists If the user id exists. + * @param {[key: string]: string} metadata Custom fields to link to the login failure event. * - * @default [] + * @beta This method is in beta and could change in future versions. */ - blocklist?: string | RegExp | ((url: string) => boolean) | (string | RegExp | ((url: string) => boolean))[]; + trackUserLoginFailureEvent(userId: string, exists: boolean, metadata?: { [key: string]: string }): void /** - * Deprecated in favor of `blocklist`. + * Links a custom event to the current trace. + * @param {string} eventName The name of the event. + * @param {[key: string]: string} metadata Custom fields to link to the event. * - * @deprecated - * @hidden + * @beta This method is in beta and could change in future versions. */ - blacklist?: string | RegExp | ((url: string) => boolean) | (string | RegExp | ((url: string) => boolean))[]; + trackCustomEvent(eventName: string, metadata?: { [key: string]: string }): void /** - * An array of headers to include in the span metadata. + * Checks if the passed user should be blocked according to AppSec rules. + * If no user is linked to the current trace, will link the passed user to it. + * @param {User} user Properties of the authenticated user. Accepts custom fields. + * @return {boolean} Indicates whether the user should be blocked. * - * @default [] + * @beta This method is in beta and could change in the future */ - headers?: string[]; + isUserBlocked(user: User): boolean /** - * Callback function to determine if there was an error. It should take a - * status code as its only parameter and return `true` for success or `false` - * for errors. + * Sends a "blocked" template response based on the request accept header and ends the response. + * **You should stop processing the request after calling this function!** + * @param {IncomingMessage} req Can be passed to force which request to act on. Optional. + * @param {OutgoingMessage} res Can be passed to force which response to act on. Optional. + * @return {boolean} Indicates if the action was successful. * - * @default code => code < 500 + * @beta This method is in beta and could change in the future */ - validateStatus?: (code: number) => boolean; + blockRequest(req?: IncomingMessage, res?: OutgoingMessage): boolean /** - * Enable injection of tracing headers into requests signed with AWS IAM headers. - * Disable this if you get AWS signature errors (HTTP 403). + * Links an authenticated user to the current trace. + * @param {User} user Properties of the authenticated user. Accepts custom fields. * - * @default false + * @beta This method is in beta and could change in the future */ - enablePropagationWithAmazonHeaders?: boolean; + setUser(user: User): void } /** @hidden */ - interface HttpServer extends Http { + type anyObject = { + [key: string]: any; + }; + + /** @hidden */ + interface TransportRequestParams { + method: string; + path: string; + body?: anyObject; + bulkBody?: anyObject; + querystring?: anyObject; + } + + /** + * The Datadog Scope Manager. This is used for context propagation. + */ + export interface Scope { /** - * Callback function to determine if there was an error. It should take a - * status code as its only parameter and return `true` for success or `false` - * for errors. + * Get the current active span or null if there is none. * - * @default code => code < 500 + * @returns {Span} The active span. */ - validateStatus?: (code: number) => boolean; + active (): Span | null; /** - * Hooks to run before spans are finished. + * Activate a span in the scope of a function. + * + * @param {Span} span The span to activate. + * @param {Function} fn Function that will have the span activated on its scope. + * @returns The return value of the provided function. */ - hooks?: { - /** - * Hook to execute just before the request span finishes. - */ - request?: (span?: Span, req?: IncomingMessage, res?: ServerResponse) => any; - }; + activate (span: Span, fn: ((...args: any[]) => T)): T; /** - * Whether to enable instrumentation of .middleware spans + * Binds a target to the provided span, or the active span if omitted. * - * @default true + * @param {Function|Promise} fn Target that will have the span activated on its scope. + * @param {Span} [span=scope.active()] The span to activate. + * @returns The bound target. */ - middleware?: boolean; + bind void> (fn: T, span?: Span | null): T; + bind V> (fn: T, span?: Span | null): T; + bind (fn: Promise, span?: Span | null): Promise; } /** @hidden */ - interface HttpClient extends Http { + interface Analyzable { /** - * Use the remote endpoint host as the service name instead of the default. - * - * @default false + * Whether to measure the span. Can also be set to a key-value pair with span + * names as keys and booleans as values for more granular control. */ - splitByDomain?: boolean; + measured?: boolean | { [key: string]: boolean }; + } + + export namespace plugins { + /** @hidden */ + interface Integration { + /** + * The service name to be used for this plugin. + */ + service?: string | any; + + /** Whether to enable the plugin. + * @default true + */ + enabled?: boolean; + } + + /** @hidden */ + interface Instrumentation extends Integration, Analyzable {} + + /** @hidden */ + interface Http extends Instrumentation { + /** + * List of URLs/paths that should be instrumented. + * + * Note that when used for an http client the entry represents a full + * outbound URL (`https://example.org/api/foo`) but when used as a + * server the entry represents an inbound path (`/api/foo`). + * + * @default /^.*$/ + */ + allowlist?: string | RegExp | ((urlOrPath: string) => boolean) | (string | RegExp | ((urlOrPath: string) => boolean))[]; + + /** + * Deprecated in favor of `allowlist`. + * + * @deprecated + * @hidden + */ + whitelist?: string | RegExp | ((urlOrPath: string) => boolean) | (string | RegExp | ((urlOrPath: string) => boolean))[]; + + /** + * List of URLs/paths that should not be instrumented. Takes precedence over + * allowlist if a URL matches an entry in both. + * + * Note that when used for an http client the entry represents a full + * outbound URL (`https://example.org/api/foo`) but when used as a + * server the entry represents an inbound path (`/api/foo`). + * + * @default [] + */ + blocklist?: string | RegExp | ((urlOrPath: string) => boolean) | (string | RegExp | ((urlOrPath: string) => boolean))[]; + + /** + * Deprecated in favor of `blocklist`. + * + * @deprecated + * @hidden + */ + blacklist?: string | RegExp | ((urlOrPath: string) => boolean) | (string | RegExp | ((urlOrPath: string) => boolean))[]; + + /** + * An array of headers to include in the span metadata. + * + * @default [] + */ + headers?: string[]; + + /** + * Callback function to determine if there was an error. It should take a + * status code as its only parameter and return `true` for success or `false` + * for errors. + * + * @default code => code < 500 + */ + validateStatus?: (code: number) => boolean; + } + + /** @hidden */ + interface HttpServer extends Http { + /** + * Callback function to determine if there was an error. It should take a + * status code as its only parameter and return `true` for success or `false` + * for errors. + * + * @default code => code < 500 + */ + validateStatus?: (code: number) => boolean; + + /** + * Hooks to run before spans are finished. + */ + hooks?: { + /** + * Hook to execute just before the request span finishes. + */ + request?: (span?: Span, req?: IncomingMessage, res?: ServerResponse) => any; + }; + + /** + * Whether to enable instrumentation of .middleware spans + * + * @default true + */ + middleware?: boolean; + } + + /** @hidden */ + interface HttpClient extends Http { + /** + * Use the remote endpoint host as the service name instead of the default. + * + * @default false + */ + splitByDomain?: boolean; + + /** + * Callback function to determine if there was an error. It should take a + * status code as its only parameter and return `true` for success or `false` + * for errors. + * + * @default code => code < 400 || code >= 500 + */ + validateStatus?: (code: number) => boolean; + + /** + * Hooks to run before spans are finished. + */ + hooks?: { + /** + * Hook to execute just before the request span finishes. + */ + request?: (span?: Span, req?: ClientRequest, res?: IncomingMessage) => any; + }; + + /** + * List of urls to which propagation headers should not be injected + */ + propagationBlocklist?: string | RegExp | ((url: string) => boolean) | (string | RegExp | ((url: string) => boolean))[]; + } + + /** @hidden */ + interface Http2Client extends Http { + /** + * Use the remote endpoint host as the service name instead of the default. + * + * @default false + */ + splitByDomain?: boolean; + + /** + * Callback function to determine if there was an error. It should take a + * status code as its only parameter and return `true` for success or `false` + * for errors. + * + * @default code => code < 400 || code >= 500 + */ + validateStatus?: (code: number) => boolean; + } + + /** @hidden */ + interface Http2Server extends Http { + /** + * Callback function to determine if there was an error. It should take a + * status code as its only parameter and return `true` for success or `false` + * for errors. + * + * @default code => code < 500 + */ + validateStatus?: (code: number) => boolean; + } + + /** @hidden */ + interface Grpc extends Instrumentation { + /** + * An array of metadata entries to record. Can also be a callback that returns + * the key/value pairs to record. For example, using + * `variables => variables` would record all variables. + */ + metadata?: string[] | ((variables: { [key: string]: any }) => { [key: string]: any }); + } + + /** @hidden */ + interface Moleculer extends Instrumentation { + /** + * Whether to include context meta as tags. + * + * @default false + */ + meta?: boolean; + } /** - * Callback function to determine if there was an error. It should take a - * status code as its only parameter and return `true` for success or `false` - * for errors. - * - * @default code => code < 400 || code >= 500 + * This plugin automatically instruments the + * [aerospike](https://github.com/aerospike/aerospike-client-nodejs) for module versions >= v3.16.2. + */ + interface aerospike extends Instrumentation {} + + /** + * This plugin automatically instruments the + * [amqp10](https://github.com/noodlefrenzy/node-amqp10) module. + */ + interface amqp10 extends Instrumentation {} + + /** + * This plugin automatically instruments the + * [amqplib](https://github.com/squaremo/amqp.node) module. */ - validateStatus?: (code: number) => boolean; + interface amqplib extends Instrumentation {} /** - * Hooks to run before spans are finished. + * Currently this plugin automatically instruments + * [@apollo/gateway](https://github.com/apollographql/federation) for module versions >= v2.3.0. + * This module uses graphql operations to service requests & thus generates graphql spans. + * We recommend disabling the graphql plugin if you only want to trace @apollo/gateway */ - hooks?: { + interface apollo extends Instrumentation { /** - * Hook to execute just before the request span finishes. + * Whether to include the source of the operation within the query as a tag + * on every span. This may contain sensitive information and should only be + * enabled if sensitive data is always sent as variables and not in the + * query text. + * + * @default false */ - request?: (span?: Span, req?: ClientRequest, res?: IncomingMessage) => any; - }; + source?: boolean; + + /** + * Whether to enable signature calculation for the resource name. This can + * be disabled if your apollo/gateway operations always have a name. Note that when + * disabled all queries will need to be named for this to work properly. + * + * @default true + */ + signature?: boolean; + } /** - * List of urls to which propagation headers should not be injected + * This plugin automatically patches the [avsc](https://github.com/mtth/avsc) module + * to collect avro message schemas when Datastreams Monitoring is enabled. */ - propagationBlocklist?: string | RegExp | ((url: string) => boolean) | (string | RegExp | ((url: string) => boolean))[]; - } + interface avsc extends Integration {} - /** @hidden */ - interface Http2Client extends Http { /** - * Use the remote endpoint host as the service name instead of the default. - * - * @default false + * This plugin automatically instruments the + * [aws-sdk](https://github.com/aws/aws-sdk-js) module. */ - splitByDomain?: boolean; + interface aws_sdk extends Instrumentation { + /** + * Whether to add a suffix to the service name so that each AWS service has its own service name. + * @default true + */ + splitByAwsService?: boolean; + + /** + * Whether to inject all messages during batch AWS SQS, Kinesis, and SNS send operations. Normal + * behavior is to inject the first message in batch send operations. + * @default false + */ + batchPropagationEnabled?: boolean; + + /** + * Hooks to run before spans are finished. + */ + hooks?: { + /** + * Hook to execute just before the aws span finishes. + */ + request?: (span?: Span, response?: anyObject) => any; + }; + + /** + * Configuration for individual services to enable/disable them. Message + * queue services can also configure the producer and consumer individually + * by passing an object with a `producer` and `consumer` properties. The + * list of valid service keys is in the service-specific section of + * https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Config.html + */ + [key: string]: boolean | Object | undefined; + } /** - * Callback function to determine if there was an error. It should take a - * status code as its only parameter and return `true` for success or `false` - * for errors. - * - * @default code => code < 400 || code >= 500 + * This plugin automatically instruments the + * @azure/functions module. + */ + interface azure_functions extends Instrumentation {} + + /** + * This plugin patches the [bunyan](https://github.com/trentm/node-bunyan) + * to automatically inject trace identifiers in log records when the + * [logInjection](interfaces/traceroptions.html#logInjection) option is enabled + * on the tracer. */ - validateStatus?: (code: number) => boolean; - } + interface bunyan extends Integration {} - /** @hidden */ - interface Http2Server extends Http { /** - * Callback function to determine if there was an error. It should take a - * status code as its only parameter and return `true` for success or `false` - * for errors. - * - * @default code => code < 500 + * This plugin automatically instruments the + * [cassandra-driver](https://github.com/datastax/nodejs-driver) module. */ - validateStatus?: (code: number) => boolean; - } + interface cassandra_driver extends Instrumentation {} - /** @hidden */ - interface Grpc extends Instrumentation { /** - * An array of metadata entries to record. Can also be a callback that returns - * the key/value pairs to record. For example, using - * `variables => variables` would record all variables. + * This plugin automatically instruments the + * [child_process](https://nodejs.org/api/child_process.html) module. */ - metadata?: string[] | ((variables: { [key: string]: any }) => { [key: string]: any }); - } + interface child_process extends Instrumentation {} - /** @hidden */ - interface Moleculer extends Instrumentation { /** - * Whether to include context meta as tags. - * - * @default false + * This plugin automatically instruments the + * [connect](https://github.com/senchalabs/connect) module. */ - meta?: boolean; - } + interface connect extends HttpServer {} - /** - * This plugin automatically instruments the - * [amqp10](https://github.com/noodlefrenzy/node-amqp10) module. - */ - interface amqp10 extends Instrumentation {} + /** + * This plugin automatically instruments the + * [couchbase](https://www.npmjs.com/package/couchbase) module. + */ + interface couchbase extends Instrumentation {} - /** - * This plugin automatically instruments the - * [amqplib](https://github.com/squaremo/amqp.node) module. - */ - interface amqplib extends Instrumentation {} + /** + * This plugin automatically instruments the + * [cucumber](https://www.npmjs.com/package/@cucumber/cucumber) module. + */ + interface cucumber extends Integration {} - /** - * This plugin automatically instruments the - * [aws-sdk](https://github.com/aws/aws-sdk-js) module. - */ - interface aws_sdk extends Instrumentation { /** - * Whether to add a suffix to the service name so that each AWS service has its own service name. - * @default true + * This plugin automatically instruments the + * [cypress](https://github.com/cypress-io/cypress) module. + */ + interface cypress extends Integration {} + + /** + * This plugin automatically instruments the + * [dns](https://nodejs.org/api/dns.html) module. */ - splitByAwsService?: boolean; + interface dns extends Instrumentation {} /** - * Hooks to run before spans are finished. + * This plugin automatically instruments the + * [elasticsearch](https://github.com/elastic/elasticsearch-js) module. */ - hooks?: { + interface elasticsearch extends Instrumentation { /** - * Hook to execute just before the aws span finishes. + * Hooks to run before spans are finished. */ - request?: (span?: Span, response?: anyObject) => any; - }; + hooks?: { + /** + * Hook to execute just before the query span finishes. + */ + query?: (span?: Span, params?: TransportRequestParams) => any; + }; + } /** - * Configuration for individual services to enable/disable them. Message - * queue services can also configure the producer and consumer individually - * by passing an object with a `producer` and `consumer` properties. The - * list of valid service keys is in the service-specific section of - * https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Config.html + * This plugin automatically instruments the + * [express](http://expressjs.com/) module. */ - [key: string]: boolean | Object | undefined; - } + interface express extends HttpServer {} - /** - * This plugin patches the [bunyan](https://github.com/trentm/node-bunyan) - * to automatically inject trace identifiers in log records when the - * [logInjection](interfaces/traceroptions.html#logInjection) option is enabled - * on the tracer. - */ - interface bunyan extends Integration {} + /** + * This plugin automatically instruments the + * [fastify](https://www.fastify.io/) module. + */ + interface fastify extends HttpServer {} - /** - * This plugin automatically instruments the - * [cassandra-driver](https://github.com/datastax/nodejs-driver) module. - */ - interface cassandra_driver extends Instrumentation {} + /** + * This plugin automatically instruments the + * [fetch](https://nodejs.org/api/globals.html#fetch) global. + */ + interface fetch extends HttpClient {} - /** - * This plugin automatically instruments the - * [connect](https://github.com/senchalabs/connect) module. - */ - interface connect extends HttpServer {} + /** + * This plugin patches the [generic-pool](https://github.com/coopernurse/node-pool) + * module to bind the callbacks the the caller context. + */ + interface generic_pool extends Integration {} - /** - * This plugin automatically instruments the - * [couchbase](https://www.npmjs.com/package/couchbase) module. - */ - interface couchbase extends Instrumentation {} + /** + * This plugin automatically instruments the + * [@google-cloud/pubsub](https://github.com/googleapis/nodejs-pubsub) module. + */ + interface google_cloud_pubsub extends Integration {} - /** - * This plugin automatically instruments the - * [cucumber](https://www.npmjs.com/package/@cucumber/cucumber) module. - */ - interface cucumber extends Integration {} + /** @hidden */ + interface ExecutionArgs { + schema: any, + document: any, + rootValue?: any, + contextValue?: any, + variableValues?: any, + operationName?: string, + fieldResolver?: any, + typeResolver?: any, + } - /** - * This plugin automatically instruments the - * [cypress](https://github.com/cypress-io/cypress) module. - */ - interface cypress extends Integration {} + /** + * This plugin automatically instruments the + * [graphql](https://github.com/graphql/graphql-js) module. + * + * The `graphql` integration uses the operation name as the span resource name. + * If no operation name is set, the resource name will always be just `query`, + * `mutation` or `subscription`. + * + * For example: + * + * ```graphql + * # good, the resource name will be `query HelloWorld` + * query HelloWorld { + * hello + * world + * } + * + * # bad, the resource name will be `query` + * { + * hello + * world + * } + * ``` + */ + interface graphql extends Instrumentation { + /** + * The maximum depth of fields/resolvers to instrument. Set to `0` to only + * instrument the operation or to `-1` to instrument all fields/resolvers. + * + * @default -1 + */ + depth?: number; - /** - * This plugin automatically instruments the - * [dns](https://nodejs.org/api/dns.html) module. - */ - interface dns extends Instrumentation {} + /** + * Whether to include the source of the operation within the query as a tag + * on every span. This may contain sensitive information and sould only be + * enabled if sensitive data is always sent as variables and not in the + * query text. + * + * @default false + */ + source?: boolean; - /** - * This plugin automatically instruments the - * [elasticsearch](https://github.com/elastic/elasticsearch-js) module. - */ - interface elasticsearch extends Instrumentation { - /** - * Hooks to run before spans are finished. - */ - hooks?: { /** - * Hook to execute just before the query span finishes. + * An array of variable names to record. Can also be a callback that returns + * the key/value pairs to record. For example, using + * `variables => variables` would record all variables. */ - query?: (span?: Span, params?: TransportRequestParams) => any; - }; - } + variables?: string[] | ((variables: { [key: string]: any }) => { [key: string]: any }); - /** - * This plugin automatically instruments the - * [express](http://expressjs.com/) module. - */ - interface express extends HttpServer {} + /** + * Whether to collapse list items into a single element. (i.e. single + * `users.*.name` span instead of `users.0.name`, `users.1.name`, etc) + * + * @default true + */ + collapse?: boolean; - /** - * This plugin automatically instruments the - * [fastify](https://www.fastify.io/) module. - */ - interface fastify extends HttpServer {} + /** + * Whether to enable signature calculation for the resource name. This can + * be disabled if your GraphQL operations always have a name. Note that when + * disabled all queries will need to be named for this to work properly. + * + * @default true + */ + signature?: boolean; - /** - * This plugin automatically instruments the - * [fetch](https://nodejs.org/api/globals.html#fetch) global. - */ - interface fetch extends HttpClient {} + /** + * An object of optional callbacks to be executed during the respective + * phase of a GraphQL operation. Undefined callbacks default to a noop + * function. + * + * @default {} + */ + hooks?: { + execute?: (span?: Span, args?: ExecutionArgs, res?: any) => void; + validate?: (span?: Span, document?: any, errors?: any) => void; + parse?: (span?: Span, source?: any, document?: any) => void; + } + } - /** - * This plugin patches the [generic-pool](https://github.com/coopernurse/node-pool) - * module to bind the callbacks the the caller context. - */ - interface generic_pool extends Integration {} + /** + * This plugin automatically instruments the + * [grpc](https://github.com/grpc/grpc-node) module. + */ + interface grpc extends Grpc { + /** + * Configuration for gRPC clients. + */ + client?: Grpc, - /** - * This plugin automatically instruments the - * [@google-cloud/pubsub](https://github.com/googleapis/nodejs-pubsub) module. - */ - interface google_cloud_pubsub extends Integration {} + /** + * Configuration for gRPC servers. + */ + server?: Grpc + } - /** @hidden */ - interface ExecutionArgs { - schema: any, - document: any, - rootValue?: any, - contextValue?: any, - variableValues?: any, - operationName?: string, - fieldResolver?: any, - typeResolver?: any, - } + /** + * This plugin automatically instruments the + * [hapi](https://hapijs.com/) module. + */ + interface hapi extends HttpServer {} - /** - * This plugin automatically instruments the - * [graphql](https://github.com/graphql/graphql-js) module. - * - * The `graphql` integration uses the operation name as the span resource name. - * If no operation name is set, the resource name will always be just `query`, - * `mutation` or `subscription`. - * - * For example: - * - * ```graphql - * # good, the resource name will be `query HelloWorld` - * query HelloWorld { - * hello - * world - * } - * - * # bad, the resource name will be `query` - * { - * hello - * world - * } - * ``` - */ - interface graphql extends Instrumentation { /** - * The maximum depth of fields/resolvers to instrument. Set to `0` to only - * instrument the operation or to `-1` to instrument all fields/resolvers. + * This plugin automatically instruments the + * [http](https://nodejs.org/api/http.html) module. * - * @default -1 + * By default any option set at the root will apply to both clients and + * servers. To configure only one or the other, use the `client` and `server` + * options. */ - depth?: number; + interface http extends HttpClient, HttpServer { + /** + * Configuration for HTTP clients. + */ + client?: HttpClient | boolean, + + /** + * Configuration for HTTP servers. + */ + server?: HttpServer | boolean + + /** + * Hooks to run before spans are finished. + */ + hooks?: { + /** + * Hook to execute just before the request span finishes. + */ + request?: ( + span?: Span, + req?: IncomingMessage | ClientRequest, + res?: ServerResponse | IncomingMessage + ) => any; + }; + } /** - * Whether to include the source of the operation within the query as a tag - * on every span. This may contain sensitive information and sould only be - * enabled if sensitive data is always sent as variables and not in the - * query text. + * This plugin automatically instruments the + * [http2](https://nodejs.org/api/http2.html) module. * - * @default false + * By default any option set at the root will apply to both clients and + * servers. To configure only one or the other, use the `client` and `server` + * options. */ - source?: boolean; + interface http2 extends Http2Client, Http2Server { + /** + * Configuration for HTTP clients. + */ + client?: Http2Client | boolean, + + /** + * Configuration for HTTP servers. + */ + server?: Http2Server | boolean + } /** - * An array of variable names to record. Can also be a callback that returns - * the key/value pairs to record. For example, using - * `variables => variables` would record all variables. + * This plugin automatically instruments the + * [ioredis](https://github.com/luin/ioredis) module. */ - variables?: string[] | ((variables: { [key: string]: any }) => { [key: string]: any }); + interface ioredis extends Instrumentation { + /** + * List of commands that should be instrumented. Commands must be in + * lowercase for example 'xread'. + * + * @default /^.*$/ + */ + allowlist?: string | RegExp | ((command: string) => boolean) | (string | RegExp | ((command: string) => boolean))[]; + + /** + * Deprecated in favor of `allowlist`. + * + * @deprecated + * @hidden + */ + whitelist?: string | RegExp | ((command: string) => boolean) | (string | RegExp | ((command: string) => boolean))[]; + + /** + * List of commands that should not be instrumented. Takes precedence over + * allowlist if a command matches an entry in both. Commands must be in + * lowercase for example 'xread'. + * + * @default [] + */ + blocklist?: string | RegExp | ((command: string) => boolean) | (string | RegExp | ((command: string) => boolean))[]; + + /** + * Deprecated in favor of `blocklist`. + * + * @deprecated + * @hidden + */ + blacklist?: string | RegExp | ((command: string) => boolean) | (string | RegExp | ((command: string) => boolean))[]; + + /** + * Whether to use a different service name for each Redis instance based + * on the configured connection name of the client. + * + * @default false + */ + splitByInstance?: boolean; + } /** - * Whether to collapse list items into a single element. (i.e. single - * `users.*.name` span instead of `users.0.name`, `users.1.name`, etc) - * - * @default true + * This plugin automatically instruments the + * [jest](https://github.com/jestjs/jest) module. */ - collapse?: boolean; + interface jest extends Integration {} /** - * Whether to enable signature calculation for the resource name. This can - * be disabled if your GraphQL operations always have a name. Note that when - * disabled all queries will need to be named for this to work properly. - * - * @default true + * This plugin patches the [knex](https://knexjs.org/) + * module to bind the promise callback the the caller context. */ - signature?: boolean; + interface knex extends Integration {} /** - * An object of optional callbacks to be executed during the respective - * phase of a GraphQL operation. Undefined callbacks default to a noop - * function. - * - * @default {} + * This plugin automatically instruments the + * [koa](https://koajs.com/) module. + */ + interface koa extends HttpServer {} + + /** + * This plugin automatically instruments the + * [kafkajs](https://kafka.js.org/) module. + */ + interface kafkajs extends Instrumentation {} + + /** + * This plugin automatically instruments the + * [ldapjs](https://github.com/ldapjs/node-ldapjs/) module. + */ + interface ldapjs extends Instrumentation {} + + /** + * This plugin automatically instruments the + * [mariadb](https://github.com/mariadb-corporation/mariadb-connector-nodejs) module. + */ + interface mariadb extends mysql {} + + /** + * This plugin automatically instruments the + * [memcached](https://github.com/3rd-Eden/memcached) module. + */ + interface memcached extends Instrumentation {} + + /** + * This plugin automatically instruments the + * [microgateway-core](https://github.com/apigee/microgateway-core) module. + */ + interface microgateway_core extends HttpServer {} + + /** + * This plugin automatically instruments the + * [mocha](https://mochajs.org/) module. + */ + interface mocha extends Integration {} + + /** + * This plugin automatically instruments the + * [moleculer](https://moleculer.services/) module. + */ + interface moleculer extends Moleculer { + /** + * Configuration for Moleculer clients. Set to false to disable client + * instrumentation. + */ + client?: boolean | Moleculer; + + /** + * Configuration for Moleculer servers. Set to false to disable server + * instrumentation. + */ + server?: boolean | Moleculer; + } + + /** + * This plugin automatically instruments the + * [mongodb-core](https://github.com/mongodb-js/mongodb-core) module. */ - hooks?: { - execute?: (span?: Span, args?: ExecutionArgs, res?: any) => void; - validate?: (span?: Span, document?: any, errors?: any) => void; - parse?: (span?: Span, source?: any, document?: any) => void; + interface mongodb_core extends Instrumentation { + /** + * Whether to include the query contents in the resource name. + */ + queryInResourceName?: boolean; } - } - /** - * This plugin automatically instruments the - * [grpc](https://github.com/grpc/grpc-node) module. - */ - interface grpc extends Grpc { /** - * Configuration for gRPC clients. + * This plugin automatically instruments the + * [mongoose](https://mongoosejs.com/) module. */ - client?: Grpc, + interface mongoose extends Instrumentation {} /** - * Configuration for gRPC servers. + * This plugin automatically instruments the + * [mysql](https://github.com/mysqljs/mysql) module. */ - server?: Grpc - } - - /** - * This plugin automatically instruments the - * [hapi](https://hapijs.com/) module. - */ - interface hapi extends HttpServer {} + interface mysql extends Instrumentation { + service?: string | ((params: any) => string); + } - /** - * This plugin automatically instruments the - * [http](https://nodejs.org/api/http.html) module. - * - * By default any option set at the root will apply to both clients and - * servers. To configure only one or the other, use the `client` and `server` - * options. - */ - interface http extends HttpClient, HttpServer { /** - * Configuration for HTTP clients. + * This plugin automatically instruments the + * [mysql2](https://github.com/sidorares/node-mysql2) module. */ - client?: HttpClient | boolean, + interface mysql2 extends mysql {} /** - * Configuration for HTTP servers. + * This plugin automatically instruments the + * [net](https://nodejs.org/api/net.html) module. */ - server?: HttpServer | boolean + interface net extends Instrumentation {} /** - * Hooks to run before spans are finished. + * This plugin automatically instruments the + * [next](https://nextjs.org/) module. */ - hooks?: { + interface next extends Instrumentation { /** - * Hook to execute just before the request span finishes. + * Hooks to run before spans are finished. */ - request?: ( - span?: Span, - req?: IncomingMessage | ClientRequest, - res?: ServerResponse | IncomingMessage - ) => any; - }; - } + hooks?: { + /** + * Hook to execute just before the request span finishes. + */ + request?: (span?: Span, req?: IncomingMessage, res?: ServerResponse) => any; + }; + } - /** - * This plugin automatically instruments the - * [http2](https://nodejs.org/api/http2.html) module. - * - * By default any option set at the root will apply to both clients and - * servers. To configure only one or the other, use the `client` and `server` - * options. - */ - interface http2 extends Http2Client, Http2Server { /** - * Configuration for HTTP clients. + * This plugin automatically instruments the + * [openai](https://platform.openai.com/docs/api-reference?lang=node.js) module. + * + * Note that for logs to work you'll need to set the `DD_API_KEY` environment variable. + * You'll also need to adjust any firewall settings to allow the tracer to communicate + * with `http-intake.logs.datadoghq.com`. + * + * Note that for metrics to work you'll need to enable + * [DogStatsD](https://docs.datadoghq.com/developers/dogstatsd/?tab=hostagent#setup) + * in the agent. */ - client?: Http2Client | boolean, + interface openai extends Instrumentation {} /** - * Configuration for HTTP servers. + * This plugin automatically instruments the + * [opensearch](https://github.com/opensearch-project/opensearch-js) module. */ - server?: Http2Server | boolean - } + interface opensearch extends elasticsearch {} - /** - * This plugin automatically instruments the - * [ioredis](https://github.com/luin/ioredis) module. - */ - interface ioredis extends Instrumentation { /** - * List of commands that should be instrumented. - * - * @default /^.*$/ + * This plugin automatically instruments the + * [oracledb](https://github.com/oracle/node-oracledb) module. */ - allowlist?: string | RegExp | ((command: string) => boolean) | (string | RegExp | ((command: string) => boolean))[]; + interface oracledb extends Instrumentation { + /** + * The service name to be used for this plugin. If a function is used, it will be passed the connection parameters and its return value will be used as the service name. + */ + service?: string | ((params: any) => string); + } /** - * Deprecated in favor of `allowlist`. - * - * @deprecated - * @hidden + * This plugin automatically instruments the + * [paperplane](https://github.com/articulate/paperplane) module. */ - whitelist?: string | RegExp | ((command: string) => boolean) | (string | RegExp | ((command: string) => boolean))[]; + interface paperplane extends HttpServer {} /** - * List of commands that should not be instrumented. Takes precedence over - * allowlist if a command matches an entry in both. - * - * @default [] - */ - blocklist?: string | RegExp | ((command: string) => boolean) | (string | RegExp | ((command: string) => boolean))[]; + * This plugin automatically instruments the + * [playwright](https://github.com/microsoft/playwright) module. + */ + interface playwright extends Integration {} /** - * Deprecated in favor of `blocklist`. - * - * @deprecated - * @hidden + * This plugin automatically instruments the + * [pg](https://node-postgres.com/) module. */ - blacklist?: string | RegExp | ((command: string) => boolean) | (string | RegExp | ((command: string) => boolean))[]; + interface pg extends Instrumentation { + /** + * The service name to be used for this plugin. If a function is used, it will be passed the connection parameters and its return value will be used as the service name. + */ + service?: string | ((params: any) => string); + /** + * The database monitoring propagation mode to be used for this plugin. + */ + dbmPropagationMode?: string; + } /** - * Whether to use a different service name for each Redis instance based - * on the configured connection name of the client. - * - * @default false + * This plugin patches the [pino](http://getpino.io) + * to automatically inject trace identifiers in log records when the + * [logInjection](interfaces/traceroptions.html#logInjection) option is enabled + * on the tracer. */ - splitByInstance?: boolean; - } - - /** - * This plugin automatically instruments the - * [jest](https://github.com/facebook/jest) module. - */ - interface jest extends Integration {} - - /** - * This plugin patches the [knex](https://knexjs.org/) - * module to bind the promise callback the the caller context. - */ - interface knex extends Integration {} - - /** - * This plugin automatically instruments the - * [koa](https://koajs.com/) module. - */ - interface koa extends HttpServer {} - - /** - * This plugin automatically instruments the - * [kafkajs](https://kafka.js.org/) module. - */ - interface kafkajs extends Instrumentation {} - - /** - * This plugin automatically instruments the - * [ldapjs](https://github.com/ldapjs/node-ldapjs/) module. - */ - interface ldapjs extends Instrumentation {} + interface pino extends Integration {} + /** + * This plugin automatically patches the [protobufjs](https://protobufjs.github.io/protobuf.js/) + * to collect protobuf message schemas when Datastreams Monitoring is enabled. + */ + interface protobufjs extends Integration {} - /** - * This plugin automatically instruments the - * [mariadb](https://github.com/mariadb-corporation/mariadb-connector-nodejs) module. - */ - interface mariadb extends mysql {} + /** + * This plugin automatically instruments the + * [redis](https://github.com/NodeRedis/node_redis) module. + */ + interface redis extends Instrumentation { + /** + * List of commands that should be instrumented. + * + * @default /^.*$/ + */ + allowlist?: string | RegExp | ((command: string) => boolean) | (string | RegExp | ((command: string) => boolean))[]; - /** - * This plugin automatically instruments the - * [memcached](https://github.com/3rd-Eden/memcached) module. - */ - interface memcached extends Instrumentation {} + /** + * Deprecated in favor of `allowlist`. + * + * deprecated + * @hidden + */ + whitelist?: string | RegExp | ((command: string) => boolean) | (string | RegExp | ((command: string) => boolean))[]; - /** - * This plugin automatically instruments the - * [microgateway-core](https://github.com/apigee/microgateway-core) module. - */ - interface microgateway_core extends HttpServer {} + /** + * List of commands that should not be instrumented. Takes precedence over + * allowlist if a command matches an entry in both. + * + * @default [] + */ + blocklist?: string | RegExp | ((command: string) => boolean) | (string | RegExp | ((command: string) => boolean))[]; - /** - * This plugin automatically instruments the - * [mocha](https://mochajs.org/) module. - */ - interface mocha extends Integration {} + /** + * Deprecated in favor of `blocklist`. + * + * @deprecated + * @hidden + */ + blacklist?: string | RegExp | ((command: string) => boolean) | (string | RegExp | ((command: string) => boolean))[]; + } - /** - * This plugin automatically instruments the - * [moleculer](https://moleculer.services/) module. - */ - interface moleculer extends Moleculer { /** - * Configuration for Moleculer clients. Set to false to disable client - * instrumentation. + * This plugin automatically instruments the + * [restify](http://restify.com/) module. */ - client?: boolean | Moleculer; + interface restify extends HttpServer {} /** - * Configuration for Moleculer servers. Set to false to disable server - * instrumentation. + * This plugin automatically instruments the + * [rhea](https://github.com/amqp/rhea) module. */ - server?: boolean | Moleculer; - } + interface rhea extends Instrumentation {} - /** - * This plugin automatically instruments the - * [mongodb-core](https://github.com/mongodb-js/mongodb-core) module. - */ - interface mongodb_core extends Instrumentation { /** - * Whether to include the query contents in the resource name. + * This plugin automatically instruments the + * [router](https://github.com/pillarjs/router) module. */ - queryInResourceName?: boolean; - } - - /** - * This plugin automatically instruments the - * [mongoose](https://mongoosejs.com/) module. - */ - interface mongoose extends Instrumentation {} - - /** - * This plugin automatically instruments the - * [mysql](https://github.com/mysqljs/mysql) module. - */ - interface mysql extends Instrumentation { - service?: string | ((params: any) => string); - } - - /** - * This plugin automatically instruments the - * [mysql2](https://github.com/sidorares/node-mysql2) module. - */ - interface mysql2 extends mysql {} + interface router extends Integration {} - /** - * This plugin automatically instruments the - * [net](https://nodejs.org/api/net.html) module. - */ - interface net extends Instrumentation {} + /** + * This plugin automatically instruments the + * [selenium-webdriver](https://www.npmjs.com/package/selenium-webdriver) module. + */ + interface selenium extends Integration {} - /** - * This plugin automatically instruments the - * [next](https://nextjs.org/) module. - */ - interface next extends Instrumentation { /** - * Hooks to run before spans are finished. + * This plugin automatically instruments the + * [sharedb](https://github.com/share/sharedb) module. */ - hooks?: { + interface sharedb extends Integration { /** - * Hook to execute just before the request span finishes. + * Hooks to run before spans are finished. */ - request?: (span?: Span, req?: IncomingMessage, res?: ServerResponse) => any; - }; - } - - /** - * This plugin automatically instruments the - * [openai](https://platform.openai.com/docs/api-reference?lang=node.js) module. - * - * Note that for logs to work you'll need to set the `DD_API_KEY` environment variable. - * You'll also need to adjust any firewall settings to allow the tracer to communicate - * with `http-intake.logs.datadoghq.com`. - * - * Note that for metrics to work you'll need to enable - * [DogStatsD](https://docs.datadoghq.com/developers/dogstatsd/?tab=hostagent#setup) - * in the agent. - */ - interface openai extends Instrumentation {} - - /** - * This plugin automatically instruments the - * [opensearch](https://github.com/opensearch-project/opensearch-js) module. - */ - interface opensearch extends elasticsearch {} + hooks?: { + /** + * Hook to execute just when the span is created. + */ + receive?: (span?: Span, request?: any) => any; + + /** + * Hook to execute just when the span is finished. + */ + reply?: (span?: Span, request?: any, response?: any) => any; + }; + } - /** - * This plugin automatically instruments the - * [oracledb](https://github.com/oracle/node-oracledb) module. - */ - interface oracledb extends Instrumentation { /** - * The service name to be used for this plugin. If a function is used, it will be passed the connection parameters and its return value will be used as the service name. + * This plugin automatically instruments the + * [tedious](https://github.com/tediousjs/tedious/) module. */ - service?: string | ((params: any) => string); - } - - /** - * This plugin automatically instruments the - * [paperplane](https://github.com/articulate/paperplane) module. - */ - interface paperplane extends HttpServer {} - - /** - * This plugin automatically instruments the - * [playwright](https://github.com/microsoft/playwright) module. - */ - interface playwright extends Integration {} + interface tedious extends Instrumentation {} - /** - * This plugin automatically instruments the - * [pg](https://node-postgres.com/) module. - */ - interface pg extends Instrumentation { /** - * The service name to be used for this plugin. If a function is used, it will be passed the connection parameters and its return value will be used as the service name. + * This plugin automatically instruments the + * [undici](https://github.com/nodejs/undici) module. */ - service?: string | ((params: any) => string); - } + interface undici extends HttpClient {} - /** - * This plugin patches the [pino](http://getpino.io) - * to automatically inject trace identifiers in log records when the - * [logInjection](interfaces/traceroptions.html#logInjection) option is enabled - * on the tracer. - */ - interface pino extends Integration {} + /** + * This plugin automatically instruments the + * [vitest](https://github.com/vitest-dev/vitest) module. + */ + interface vitest extends Integration {} - /** - * This plugin automatically instruments the - * [redis](https://github.com/NodeRedis/node_redis) module. - */ - interface redis extends Instrumentation { /** - * List of commands that should be instrumented. - * - * @default /^.*$/ + * This plugin patches the [winston](https://github.com/winstonjs/winston) + * to automatically inject trace identifiers in log records when the + * [logInjection](interfaces/traceroptions.html#logInjection) option is enabled + * on the tracer. */ - allowlist?: string | RegExp | ((command: string) => boolean) | (string | RegExp | ((command: string) => boolean))[]; + interface winston extends Integration {} + } + export namespace opentelemetry { /** - * Deprecated in favor of `allowlist`. - * - * deprecated - * @hidden + * A registry for creating named {@link Tracer}s. */ - whitelist?: string | RegExp | ((command: string) => boolean) | (string | RegExp | ((command: string) => boolean))[]; + export interface TracerProvider extends otel.TracerProvider { + /** + * Construct a new TracerProvider to register with @opentelemetry/api + * + * @param config Configuration object for the TracerProvider + * @returns TracerProvider A TracerProvider instance + */ + new(config?: Record): TracerProvider; + + /** + * Returns a Tracer, creating one if one with the given name and version is + * not already created. + * + * @param name The name of the tracer or instrumentation library. + * @param version The version of the tracer or instrumentation library. + * @param options The options of the tracer or instrumentation library. + * @returns Tracer A Tracer with the given name and version + */ + getTracer(name: string, version?: string, options?: any): Tracer; + + /** + * Register this tracer provider with @opentelemetry/api + */ + register(): void; + } /** - * List of commands that should not be instrumented. Takes precedence over - * allowlist if a command matches an entry in both. - * - * @default [] + * Tracer provides an interface for creating {@link Span}s. */ - blocklist?: string | RegExp | ((command: string) => boolean) | (string | RegExp | ((command: string) => boolean))[]; + export interface Tracer extends otel.Tracer { + /** + * Starts a new {@link Span}. Start the span without setting it on context. + * + * This method do NOT modify the current Context. + * + * @param name The name of the span + * @param [options] SpanOptions used for span creation + * @param [context] Context to use to extract parent + * @returns Span The newly created span + * @example + * const span = tracer.startSpan('op'); + * span.setAttribute('key', 'value'); + * span.end(); + */ + startSpan(name: string, options?: SpanOptions, context?: Context): Span; + + /** + * Starts a new {@link Span} and calls the given function passing it the + * created span as first argument. + * Additionally the new span gets set in context and this context is activated + * for the duration of the function call. + * + * @param name The name of the span + * @param [options] SpanOptions used for span creation + * @param [context] Context to use to extract parent + * @param fn function called in the context of the span and receives the newly created span as an argument + * @returns return value of fn + * @example + * const something = tracer.startActiveSpan('op', span => { + * try { + * do some work + * span.setStatus({code: SpanStatusCode.OK}); + * return something; + * } catch (err) { + * span.setStatus({ + * code: SpanStatusCode.ERROR, + * message: err.message, + * }); + * throw err; + * } finally { + * span.end(); + * } + * }); + * + * @example + * const span = tracer.startActiveSpan('op', span => { + * try { + * do some work + * return span; + * } catch (err) { + * span.setStatus({ + * code: SpanStatusCode.ERROR, + * message: err.message, + * }); + * throw err; + * } + * }); + * do some more work + * span.end(); + */ + startActiveSpan unknown>(name: string, options: SpanOptions, context: otel.Context, fn: F): ReturnType; + startActiveSpan unknown>(name: string, options: SpanOptions, fn: F): ReturnType; + startActiveSpan unknown>(name: string, fn: F): ReturnType; + } /** - * Deprecated in favor of `blocklist`. + * An interface that represents a span. A span represents a single operation + * within a trace. Examples of span might include remote procedure calls or a + * in-process function calls to sub-components. A Trace has a single, top-level + * "root" Span that in turn may have zero or more child Spans, which in turn + * may have children. * - * @deprecated - * @hidden + * Spans are created by the {@link Tracer.startSpan} method. */ - blacklist?: string | RegExp | ((command: string) => boolean) | (string | RegExp | ((command: string) => boolean))[]; - } + export interface Span extends otel.Span { + /** + * Returns the {@link SpanContext} object associated with this Span. + * + * Get an immutable, serializable identifier for this span that can be used + * to create new child spans. Returned SpanContext is usable even after the + * span ends. + * + * @returns the SpanContext object associated with this Span. + */ + spanContext(): SpanContext; - /** - * This plugin automatically instruments the - * [restify](http://restify.com/) module. - */ - interface restify extends HttpServer {} + /** + * Sets an attribute to the span. + * + * Sets a single Attribute with the key and value passed as arguments. + * + * @param key the key for this attribute. + * @param value the value for this attribute. Setting a value null or + * undefined is invalid and will result in undefined behavior. + */ + setAttribute(key: string, value: SpanAttributeValue): this; - /** - * This plugin automatically instruments the - * [rhea](https://github.com/amqp/rhea) module. - */ - interface rhea extends Instrumentation {} + /** + * Sets attributes to the span. + * + * @param attributes the attributes that will be added. + * null or undefined attribute values + * are invalid and will result in undefined behavior. + */ + setAttributes(attributes: SpanAttributes): this; - /** - * This plugin automatically instruments the - * [router](https://github.com/pillarjs/router) module. - */ - interface router extends Integration {} + /** + * Adds an event to the Span. + * + * @param name the name of the event. + * @param [attributesOrStartTime] the attributes that will be added; these are + * associated with this event. Can be also a start time + * if type is {@link TimeInput} and 3rd param is undefined + * @param [startTime] start time of the event. + */ + addEvent(name: string, attributesOrStartTime?: SpanAttributes | TimeInput, startTime?: TimeInput): this; - /** - * This plugin automatically instruments the - * [sharedb](https://github.com/share/sharedb) module. - */ - interface sharedb extends Integration { - /** - * Hooks to run before spans are finished. - */ - hooks?: { /** - * Hook to execute just when the span is created. + * Sets a status to the span. If used, this will override the default Span + * status. Default is {@link otel.SpanStatusCode.UNSET}. SetStatus overrides the value + * of previous calls to SetStatus on the Span. + * + * @param status the SpanStatus to set. */ - receive?: (span?: Span, request?: any) => any; + setStatus(status: SpanStatus): this; /** - * Hook to execute just when the span is finished. + * Updates the Span name. + * + * This will override the name provided via {@link Tracer.startSpan}. + * + * Upon this update, any sampling behavior based on Span name will depend on + * the implementation. + * + * @param name the Span name. */ - reply?: (span?: Span, request?: any, response?: any) => any; - }; - } + updateName(name: string): this; - /** - * This plugin automatically instruments the - * [tedious](https://github.com/tediousjs/tedious/) module. - */ - interface tedious extends Instrumentation {} + /** + * Marks the end of Span execution. + * + * Call to End of a Span MUST not have any effects on child spans. Those may + * still be running and can be ended later. + * + * Do not return `this`. The Span generally should not be used after it + * is ended so chaining is not desired in this context. + * + * @param [endTime] the time to set as Span's end time. If not provided, + * use the current time as the span's end time. + */ + end(endTime?: TimeInput): void; - /** - * This plugin patches the [winston](https://github.com/winstonjs/winston) - * to automatically inject trace identifiers in log records when the - * [logInjection](interfaces/traceroptions.html#logInjection) option is enabled - * on the tracer. - */ - interface winston extends Integration {} -} + /** + * Returns the flag whether this span will be recorded. + * + * @returns true if this Span is active and recording information like events + * with the `AddEvent` operation and attributes using `setAttributes`. + */ + isRecording(): boolean; -export namespace opentelemetry { - /** - * A registry for creating named {@link Tracer}s. - */ - export interface TracerProvider extends otel.TracerProvider { - /** - * Construct a new TracerProvider to register with @opentelemetry/api - * - * @returns TracerProvider A TracerProvider instance - */ - new(): TracerProvider; + /** + * Sets exception as a span event + * @param exception the exception the only accepted values are string or Error + * @param [time] the time to set as Span's event time. If not provided, + * use the current time. + */ + recordException(exception: Exception, time?: TimeInput): void; - /** - * Returns a Tracer, creating one if one with the given name and version is - * not already created. - * - * This function may return different Tracer types (e.g. - * {@link NoopTracerProvider} vs. a functional tracer). - * - * @param name The name of the tracer or instrumentation library. - * @param version The version of the tracer or instrumentation library. - * @param options The options of the tracer or instrumentation library. - * @returns Tracer A Tracer with the given name and version - */ - getTracer(name: string, version?: string): Tracer; + /** + * Causally links another span to the current span + * @param {otel.SpanContext} context The context of the span to link to. + * @param {SpanAttributes} attributes An optional key value pair of arbitrary values. + * @returns {void} + */ + addLink(context: otel.SpanContext, attributes?: SpanAttributes): void; + } /** - * Register this tracer provider with @opentelemetry/api + * A SpanContext represents the portion of a {@link Span} which must be + * serialized and propagated along side of a {@link otel.Baggage}. */ - register(): void; - } + export interface SpanContext extends otel.SpanContext { + /** + * The ID of the trace that this span belongs to. It is worldwide unique + * with practically sufficient probability by being made as 16 randomly + * generated bytes, encoded as a 32 lowercase hex characters corresponding to + * 128 bits. + */ + traceId: string; - /** - * Tracer provides an interface for creating {@link Span}s. - */ - export interface Tracer extends otel.Tracer { - /** - * Starts a new {@link Span}. Start the span without setting it on context. - * - * This method do NOT modify the current Context. - * - * @param name The name of the span - * @param [options] SpanOptions used for span creation - * @param [context] Context to use to extract parent - * @returns Span The newly created span - * @example - * const span = tracer.startSpan('op'); - * span.setAttribute('key', 'value'); - * span.end(); - */ - startSpan(name: string, options?: SpanOptions, context?: Context): Span; - - /** - * Starts a new {@link Span} and calls the given function passing it the - * created span as first argument. - * Additionally the new span gets set in context and this context is activated - * for the duration of the function call. - * - * @param name The name of the span - * @param [options] SpanOptions used for span creation - * @param [context] Context to use to extract parent - * @param fn function called in the context of the span and receives the newly created span as an argument - * @returns return value of fn - * @example - * const something = tracer.startActiveSpan('op', span => { - * try { - * do some work - * span.setStatus({code: SpanStatusCode.OK}); - * return something; - * } catch (err) { - * span.setStatus({ - * code: SpanStatusCode.ERROR, - * message: err.message, - * }); - * throw err; - * } finally { - * span.end(); - * } - * }); - * - * @example - * const span = tracer.startActiveSpan('op', span => { - * try { - * do some work - * return span; - * } catch (err) { - * span.setStatus({ - * code: SpanStatusCode.ERROR, - * message: err.message, - * }); - * throw err; - * } - * }); - * do some more work - * span.end(); - */ - startActiveSpan unknown>(name: string, fn: F): ReturnType; - startActiveSpan unknown>(name: string, options: SpanOptions, fn: F): ReturnType; - startActiveSpan unknown>(name: string, options: SpanOptions, context: otel.Context, fn: F): ReturnType; - } + /** + * The ID of the Span. It is globally unique with practically sufficient + * probability by being made as 8 randomly generated bytes, encoded as a 16 + * lowercase hex characters corresponding to 64 bits. + */ + spanId: string; - /** - * An interface that represents a span. A span represents a single operation - * within a trace. Examples of span might include remote procedure calls or a - * in-process function calls to sub-components. A Trace has a single, top-level - * "root" Span that in turn may have zero or more child Spans, which in turn - * may have children. - * - * Spans are created by the {@link Tracer.startSpan} method. - */ - export interface Span extends otel.Span { - /** - * Returns the {@link SpanContext} object associated with this Span. - * - * Get an immutable, serializable identifier for this span that can be used - * to create new child spans. Returned SpanContext is usable even after the - * span ends. - * - * @returns the SpanContext object associated with this Span. - */ - spanContext(): SpanContext; + /** + * Only true if the SpanContext was propagated from a remote parent. + */ + isRemote?: boolean; - /** - * Sets an attribute to the span. - * - * Sets a single Attribute with the key and value passed as arguments. - * - * @param key the key for this attribute. - * @param value the value for this attribute. Setting a value null or - * undefined is invalid and will result in undefined behavior. - */ - setAttribute(key: string, value: SpanAttributeValue): this; + /** + * Trace flags to propagate. + * + * It is represented as 1 byte (bitmap). Bit to represent whether trace is + * sampled or not. When set, the least significant bit documents that the + * caller may have recorded trace data. A caller who does not record trace + * data out-of-band leaves this flag unset. + * + * see {@link otel.TraceFlags} for valid flag values. + */ + traceFlags: number; - /** - * Sets attributes to the span. - * - * @param attributes the attributes that will be added. - * null or undefined attribute values - * are invalid and will result in undefined behavior. - */ - setAttributes(attributes: SpanAttributes): this; + /** + * Tracing-system-specific info to propagate. + * + * The tracestate field value is a `list` as defined below. The `list` is a + * series of `list-members` separated by commas `,`, and a list-member is a + * key/value pair separated by an equals sign `=`. Spaces and horizontal tabs + * surrounding `list-members` are ignored. There can be a maximum of 32 + * `list-members` in a `list`. + * More Info: https://www.w3.org/TR/trace-context/#tracestate-field + * + * Examples: + * Single tracing system (generic format): + * tracestate: rojo=00f067aa0ba902b7 + * Multiple tracing systems (with different formatting): + * tracestate: rojo=00f067aa0ba902b7,congo=t61rcWkgMzE + */ + traceState?: TraceState; + } - /** - * Adds an event to the Span. - * - * @param name the name of the event. - * @param [attributesOrStartTime] the attributes that will be added; these are - * associated with this event. Can be also a start time - * if type is {@type TimeInput} and 3rd param is undefined - * @param [startTime] start time of the event. - */ - addEvent(name: string, attributesOrStartTime?: SpanAttributes | TimeInput, startTime?: TimeInput): this; + export type Context = otel.Context; + export type Exception = otel.Exception; + export type SpanAttributes = otel.SpanAttributes; + export type SpanAttributeValue = otel.SpanAttributeValue; + export type SpanOptions = otel.SpanOptions; + export type SpanStatus = otel.SpanStatus; + export type TimeInput = otel.TimeInput; + export type TraceState = otel.TraceState; + } + /** + * Iast configuration used in `tracer` and `tracer.experimental` options + */ + interface IastOptions { /** - * Sets a status to the span. If used, this will override the default Span - * status. Default is {@link SpanStatusCode.UNSET}. SetStatus overrides the value - * of previous calls to SetStatus on the Span. - * - * @param status the SpanStatus to set. + * Whether to enable IAST. + * @default false */ - setStatus(status: SpanStatus): this; + enabled?: boolean, /** - * Updates the Span name. - * - * This will override the name provided via {@link Tracer.startSpan}. - * - * Upon this update, any sampling behavior based on Span name will depend on - * the implementation. - * - * @param name the Span name. + * Controls the percentage of requests that iast will analyze + * @default 30 */ - updateName(name: string): this; + requestSampling?: number, /** - * Marks the end of Span execution. - * - * Call to End of a Span MUST not have any effects on child spans. Those may - * still be running and can be ended later. - * - * Do not return `this`. The Span generally should not be used after it - * is ended so chaining is not desired in this context. - * - * @param [endTime] the time to set as Span's end time. If not provided, - * use the current time as the span's end time. + * Controls how many request can be analyzing code vulnerabilities at the same time + * @default 2 */ - end(endTime?: TimeInput): void; + maxConcurrentRequests?: number, /** - * Returns the flag whether this span will be recorded. - * - * @returns true if this Span is active and recording information like events - * with the `AddEvent` operation and attributes using `setAttributes`. + * Controls how many code vulnerabilities can be detected in the same request + * @default 2 */ - isRecording(): boolean; + maxContextOperations?: number, /** - * Sets exception as a span event - * @param exception the exception the only accepted values are string or Error - * @param [time] the time to set as Span's event time. If not provided, - * use the current time. + * Defines the pattern to ignore cookie names in the vulnerability hash calculation + * @default ".{32,}" */ - recordException(exception: Exception, time?: TimeInput): void; - } + cookieFilterPattern?: string, - /** - * A SpanContext represents the portion of a {@link Span} which must be - * serialized and propagated along side of a {@link Baggage}. - */ - export interface SpanContext extends otel.SpanContext { /** - * The ID of the trace that this span belongs to. It is worldwide unique - * with practically sufficient probability by being made as 16 randomly - * generated bytes, encoded as a 32 lowercase hex characters corresponding to - * 128 bits. + * Whether to enable vulnerability deduplication */ - traceId: string; + deduplicationEnabled?: boolean, /** - * The ID of the Span. It is globally unique with practically sufficient - * probability by being made as 8 randomly generated bytes, encoded as a 16 - * lowercase hex characters corresponding to 64 bits. + * Whether to enable vulnerability redaction + * @default true */ - spanId: string; + redactionEnabled?: boolean, /** - * Only true if the SpanContext was propagated from a remote parent. + * Specifies a regex that will redact sensitive source names in vulnerability reports. */ - isRemote?: boolean; + redactionNamePattern?: string, /** - * Trace flags to propagate. - * - * It is represented as 1 byte (bitmap). Bit to represent whether trace is - * sampled or not. When set, the least significant bit documents that the - * caller may have recorded trace data. A caller who does not record trace - * data out-of-band leaves this flag unset. - * - * see {@link TraceFlags} for valid flag values. + * Specifies a regex that will redact sensitive source values in vulnerability reports. */ - traceFlags: number; + redactionValuePattern?: string, /** - * Tracing-system-specific info to propagate. - * - * The tracestate field value is a `list` as defined below. The `list` is a - * series of `list-members` separated by commas `,`, and a list-member is a - * key/value pair separated by an equals sign `=`. Spaces and horizontal tabs - * surrounding `list-members` are ignored. There can be a maximum of 32 - * `list-members` in a `list`. - * More Info: https://www.w3.org/TR/trace-context/#tracestate-field - * - * Examples: - * Single tracing system (generic format): - * tracestate: rojo=00f067aa0ba902b7 - * Multiple tracing systems (with different formatting): - * tracestate: rojo=00f067aa0ba902b7,congo=t61rcWkgMzE + * Specifies the verbosity of the sent telemetry. Default 'INFORMATION' */ - traceState?: TraceState; + telemetryVerbosity?: string } - - export type Context = otel.Context; - export type Exception = otel.Exception; - export type SpanAttributes = otel.SpanAttributes; - export type SpanAttributeValue = otel.SpanAttributeValue; - export type SpanOptions = otel.SpanOptions; - export type SpanStatus = otel.SpanStatus; - export type TimeInput = otel.TimeInput; - export type TraceState = otel.TraceState; } /** @@ -1968,6 +2205,6 @@ export namespace opentelemetry { * start tracing. If not initialized, or initialized and disabled, it will use * a no-op implementation. */ -export declare const tracer: Tracer; +declare const tracer: Tracer; -export default tracer; +export = tracer; diff --git a/init.js b/init.js index 3e0e977da36..ecdb37daee8 100644 --- a/init.js +++ b/init.js @@ -1,7 +1,58 @@ 'use strict' -const tracer = require('.') +const path = require('path') +const Module = require('module') +const semver = require('semver') +const log = require('./packages/dd-trace/src/log') +const { isTrue } = require('./packages/dd-trace/src/util') +const telemetry = require('./packages/dd-trace/src/telemetry/init-telemetry') -tracer.init() +let initBailout = false +let clobberBailout = false +const forced = isTrue(process.env.DD_INJECT_FORCE) -module.exports = tracer +if (process.env.DD_INJECTION_ENABLED) { + // If we're running via single-step install, and we're not in the app's + // node_modules, then we should not initialize the tracer. This prevents + // single-step-installed tracer from clobbering the manually-installed tracer. + let resolvedInApp + const entrypoint = process.argv[1] + try { + resolvedInApp = Module.createRequire(entrypoint).resolve('dd-trace') + } catch (e) { + // Ignore. If we can't resolve the module, we assume it's not in the app. + } + if (resolvedInApp) { + const ourselves = path.join(__dirname, 'index.js') + if (ourselves !== resolvedInApp) { + clobberBailout = true + } + } + + // If we're running via single-step install, and the runtime doesn't match + // the engines field in package.json, then we should not initialize the tracer. + if (!clobberBailout) { + const { engines } = require('./package.json') + const version = process.versions.node + if (!semver.satisfies(version, engines.node)) { + initBailout = true + telemetry([ + { name: 'abort', tags: ['reason:incompatible_runtime'] }, + { name: 'abort.runtime', tags: [] } + ]) + log.info('Aborting application instrumentation due to incompatible_runtime.') + log.info(`Found incompatible runtime nodejs ${version}, Supported runtimes: nodejs ${engines.node}.`) + if (forced) { + log.info('DD_INJECT_FORCE enabled, allowing unsupported runtimes and continuing.') + } + } + } +} + +if (!clobberBailout && (!initBailout || forced)) { + const tracer = require('.') + tracer.init() + module.exports = tracer + telemetry('complete', [`injection_forced:${forced && initBailout ? 'true' : 'false'}`]) + log.info('Application instrumentation bootstrapping complete') +} diff --git a/initialize.mjs b/initialize.mjs new file mode 100644 index 00000000000..777f45cc046 --- /dev/null +++ b/initialize.mjs @@ -0,0 +1,55 @@ +/** + * This file serves one of two purposes, depending on how it's used. + * + * If used with --import, it will import init.js and register the loader hook. + * If used with --loader, it will act as the loader hook, except that it will + * also import init.js inside the source code of the entrypoint file. + * + * The result is that no matter how this file is used, so long as it's with + * one of the two flags, the tracer will always be initialized, and the loader + * hook will always be active for ESM support. + */ + +import { isMainThread } from 'worker_threads' + +import { fileURLToPath } from 'node:url' +import { + load as origLoad, + resolve as origResolve, + getFormat as origGetFormat, + getSource as origGetSource +} from 'import-in-the-middle/hook.mjs' + +let hasInsertedInit = false +function insertInit (result) { + if (!hasInsertedInit) { + hasInsertedInit = true + result.source = ` +import '${fileURLToPath(new URL('./init.js', import.meta.url))}'; +${result.source}` + } + return result +} + +export async function load (...args) { + return insertInit(await origLoad(...args)) +} + +export const resolve = origResolve + +export const getFormat = origGetFormat + +export async function getSource (...args) { + return insertInit(await origGetSource(...args)) +} + +if (isMainThread) { + // Need this IIFE for versions of Node.js without top-level await. + (async () => { + await import('./init.js') + const { register } = await import('node:module') + if (register) { + register('./loader-hook.mjs', import.meta.url) + } + })() +} diff --git a/integration-tests/CODEOWNERS b/integration-tests/CODEOWNERS new file mode 100644 index 00000000000..538c14b1a96 --- /dev/null +++ b/integration-tests/CODEOWNERS @@ -0,0 +1,2 @@ +# CODEOWNERS for testing purposes +ci-visibility/subproject @datadog-dd-trace-js diff --git a/integration-tests/appsec/index.spec.js b/integration-tests/appsec/index.spec.js new file mode 100644 index 00000000000..a7c65b3932d --- /dev/null +++ b/integration-tests/appsec/index.spec.js @@ -0,0 +1,342 @@ +'use strict' + +const getPort = require('get-port') +const path = require('path') +const Axios = require('axios') +const { assert } = require('chai') +const { createSandbox, FakeAgent, spawnProc } = require('../helpers') + +describe('RASP', () => { + let axios, sandbox, cwd, appPort, appFile, agent, proc, stdioHandler + + function stdOutputHandler (data) { + stdioHandler && stdioHandler(data) + } + + before(async () => { + sandbox = await createSandbox(['express', 'axios']) + appPort = await getPort() + cwd = sandbox.folder + appFile = path.join(cwd, 'appsec/rasp/index.js') + axios = Axios.create({ + baseURL: `http://localhost:${appPort}` + }) + }) + + after(async () => { + await sandbox.remove() + }) + + function startServer (abortOnUncaughtException) { + beforeEach(async () => { + let execArgv = process.execArgv + if (abortOnUncaughtException) { + execArgv = ['--abort-on-uncaught-exception', ...execArgv] + } + agent = await new FakeAgent().start() + proc = await spawnProc(appFile, { + cwd, + execArgv, + env: { + DD_TRACE_AGENT_PORT: agent.port, + APP_PORT: appPort, + DD_APPSEC_ENABLED: true, + DD_APPSEC_RASP_ENABLED: true, + DD_APPSEC_RULES: path.join(cwd, 'appsec/rasp/rasp_rules.json') + } + }, stdOutputHandler, stdOutputHandler) + }) + + afterEach(async () => { + proc.kill() + await agent.stop() + }) + } + + async function assertExploitDetected () { + await agent.assertMessageReceived(({ headers, payload }) => { + assert.property(payload[0][0].meta, '_dd.appsec.json') + assert.include(payload[0][0].meta['_dd.appsec.json'], '"test-rule-id-2"') + }) + } + + describe('--abort-on-uncaught-exception is not configured', () => { + startServer(false) + + async function testNotCrashedAfterBlocking (path) { + let hasOutput = false + stdioHandler = () => { + hasOutput = true + } + + try { + await axios.get(`${path}?host=localhost/ifconfig.pro`) + + assert.fail('Request should have failed') + } catch (e) { + if (!e.response) { + throw e + } + + assert.strictEqual(e.response.status, 403) + await assertExploitDetected() + } + + return new Promise((resolve, reject) => { + setTimeout(() => { + if (hasOutput) { + reject(new Error('Unexpected output in stdout/stderr after blocking request')) + } else { + resolve() + } + }, 50) + }) + } + + async function testCustomErrorHandlerIsNotExecuted (path) { + let hasOutput = false + try { + stdioHandler = () => { + hasOutput = true + } + + await axios.get(`${path}?host=localhost/ifconfig.pro`) + + assert.fail('Request should have failed') + } catch (e) { + if (!e.response) { + throw e + } + + assert.strictEqual(e.response.status, 403) + await assertExploitDetected() + + return new Promise((resolve, reject) => { + setTimeout(() => { + if (hasOutput) { + reject(new Error('uncaughtExceptionCaptureCallback executed')) + } else { + resolve() + } + }, 10) + }) + } + } + + async function testAppCrashesAsExpected () { + let hasOutput = false + stdioHandler = () => { + hasOutput = true + } + + try { + await axios.get('/crash') + } catch (e) { + return new Promise((resolve, reject) => { + setTimeout(() => { + if (hasOutput) { + resolve() + } else { + reject(new Error('Output expected after crash')) + } + }, 50) + }) + } + + assert.fail('Request should have failed') + } + + describe('ssrf', () => { + it('should crash when error is not an AbortError', async () => { + await testAppCrashesAsExpected() + }) + + it('should not crash when customer has his own setUncaughtExceptionCaptureCallback', async () => { + let hasOutput = false + stdioHandler = () => { + hasOutput = true + } + + try { + await axios.get('/crash-and-recovery-A') + } catch (e) { + return new Promise((resolve, reject) => { + setTimeout(() => { + if (hasOutput) { + reject(new Error('Unexpected output in stdout/stderr after blocking request')) + } else { + resolve() + } + }, 50) + }) + } + + assert.fail('Request should have failed') + }) + + it('should not crash when customer has his own uncaughtException', async () => { + let hasOutput = false + stdioHandler = () => { + hasOutput = true + } + + try { + await axios.get('/crash-and-recovery-B') + } catch (e) { + return new Promise((resolve, reject) => { + setTimeout(() => { + if (hasOutput) { + reject(new Error('Unexpected output in stdout/stderr after blocking request')) + } else { + resolve() + } + }, 50) + }) + } + + assert.fail('Request should have failed') + }) + + it('should block manually', async () => { + let response + + try { + response = await axios.get('/ssrf/http/manual-blocking?host=localhost/ifconfig.pro') + } catch (e) { + if (!e.response) { + throw e + } + response = e.response + assert.strictEqual(response.status, 418) + return await assertExploitDetected() + } + + assert.fail('Request should have failed') + }) + + it('should block in a domain', async () => { + let response + + try { + response = await axios.get('/ssrf/http/should-block-in-domain?host=localhost/ifconfig.pro') + } catch (e) { + if (!e.response) { + throw e + } + response = e.response + assert.strictEqual(response.status, 403) + return await assertExploitDetected() + } + + assert.fail('Request should have failed') + }) + + it('should crash as expected after block in domain request', async () => { + try { + await axios.get('/ssrf/http/should-block-in-domain?host=localhost/ifconfig.pro') + } catch (e) { + return await testAppCrashesAsExpected() + } + + assert.fail('Request should have failed') + }) + + it('should block when error is unhandled', async () => { + try { + await axios.get('/ssrf/http/unhandled-error?host=localhost/ifconfig.pro') + } catch (e) { + if (!e.response) { + throw e + } + + assert.strictEqual(e.response.status, 403) + return await assertExploitDetected() + } + + assert.fail('Request should have failed') + }) + + it('should crash as expected after a requiest block when error is unhandled', async () => { + try { + await axios.get('/ssrf/http/unhandled-error?host=localhost/ifconfig.pro') + } catch (e) { + return await testAppCrashesAsExpected() + } + + assert.fail('Request should have failed') + }) + + it('should not execute custom uncaughtExceptionCaptureCallback when it is blocked', async () => { + return testCustomErrorHandlerIsNotExecuted('/ssrf/http/custom-uncaught-exception-capture-callback') + }) + + it('should not execute custom uncaughtException listener', async () => { + return testCustomErrorHandlerIsNotExecuted('/ssrf/http/custom-uncaughtException-listener') + }) + + it('should not crash when app send data after blocking', () => { + return testNotCrashedAfterBlocking('/ssrf/http/unhandled-async-write-A') + }) + + it('should not crash when app stream data after blocking', () => { + return testNotCrashedAfterBlocking('/ssrf/http/unhandled-async-write-B') + }) + + it('should not crash when setHeader, writeHead or end after blocking', () => { + return testNotCrashedAfterBlocking('/ssrf/http/unhandled-async-write-C') + }) + + it('should not crash when appendHeader, flushHeaders, removeHeader after blocking', () => { + return testNotCrashedAfterBlocking('/ssrf/http/unhandled-async-write-D') + }) + + it('should not crash when writeContinue after blocking', () => { + return testNotCrashedAfterBlocking('/ssrf/http/unhandled-async-write-E') + }) + + it('should not crash when writeProcessing after blocking', () => { + return testNotCrashedAfterBlocking('/ssrf/http/unhandled-async-write-F') + }) + + it('should not crash when writeEarlyHints after blocking', () => { + return testNotCrashedAfterBlocking('/ssrf/http/unhandled-async-write-G') + }) + + it('should not crash when res.json after blocking', () => { + return testNotCrashedAfterBlocking('/ssrf/http/unhandled-async-write-H') + }) + + it('should not crash when is blocked using axios', () => { + return testNotCrashedAfterBlocking('/ssrf/http/unhandled-axios') + }) + + it('should not crash when is blocked with unhandled rejection', () => { + return testNotCrashedAfterBlocking('/ssrf/http/unhandled-promise') + }) + }) + }) + + describe('--abort-on-uncaught-exception is configured', () => { + startServer(true) + + describe('ssrf', () => { + it('should not block', async () => { + let response + + try { + response = await axios.get('/ssrf/http/manual-blocking?host=localhost/ifconfig.pro') + } catch (e) { + if (!e.response) { + throw e + } + response = e.response + } + + // not blocked + assert.notEqual(response.status, 418) + assert.notEqual(response.status, 403) + await assertExploitDetected() + }) + }) + }) +}) diff --git a/integration-tests/appsec/rasp/index.js b/integration-tests/appsec/rasp/index.js new file mode 100644 index 00000000000..a2035c6c3b4 --- /dev/null +++ b/integration-tests/appsec/rasp/index.js @@ -0,0 +1,210 @@ +'use strict' +const tracer = require('dd-trace') +tracer.init({ + flushInterval: 0 +}) + +const path = require('path') +const fs = require('fs') +const http = require('https') +const express = require('express') +const axios = require('axios') + +const app = express() +const port = process.env.APP_PORT || 3000 + +function makeOutgoingRequestAndCbAfterTimeout (req, res, cb) { + let finished = false + setTimeout(() => { + if (!finished && cb) { + cb() + } + }, 10) + + http.get(`https://${req.query.host}`, () => { + finished = true + res.send('end') + }) +} + +function streamFile (res) { + const stream = fs.createReadStream(path.join(__dirname, 'streamtest.txt'), { encoding: 'utf8' }) + stream.pipe(res, { end: false }) + stream.on('end', () => res.end('end')) +} + +function httpGetPromise (host) { + return new Promise((resolve, reject) => { + const clientRequest = http.get(`https://${host}`, () => { + resolve() + }) + clientRequest.on('error', reject) + }) +} + +app.get('/crash', () => { + process.nextTick(() => { + throw new Error('Crash') + }) +}) + +app.get('/crash-and-recovery-A', (req, res) => { + process.setUncaughtExceptionCaptureCallback(() => { + res.writeHead(500) + res.end('error') + + process.setUncaughtExceptionCaptureCallback(null) + }) + + process.nextTick(() => { + throw new Error('Crash') + }) +}) + +app.get('/crash-and-recovery-B', (req, res) => { + function exceptionHandler () { + res.writeHead(500) + res.end('error') + + process.off('uncaughtException', exceptionHandler) + } + + process.on('uncaughtException', exceptionHandler) + + process.nextTick(() => { + throw new Error('Crash') + }) +}) + +app.get('/ssrf/http/manual-blocking', (req, res) => { + const clientRequest = http.get(`https://${req.query.host}`, () => { + res.send('end') + }) + + clientRequest.on('error', (err) => { + if (err.name === 'DatadogRaspAbortError') { + res.writeHead(418) + res.end('aborted') + } else { + res.writeHead(500) + res.end('error') + } + }) +}) + +app.get('/ssrf/http/custom-uncaught-exception-capture-callback', (req, res) => { + process.setUncaughtExceptionCaptureCallback(() => { + // wanted a log to force error on tests + // eslint-disable-next-line no-console + console.log('Custom uncaught exception capture callback') + res.writeHead(500) + res.end('error') + }) + + http.get(`https://${req.query.host}`, () => { + res.send('end') + }) +}) + +app.get('/ssrf/http/should-block-in-domain', (req, res) => { + // eslint-disable-next-line n/no-deprecated-api + const d = require('node:domain').create() + d.run(() => { + http.get(`https://${req.query.host}`, () => { + res.send('end') + }) + }) +}) + +app.get('/ssrf/http/custom-uncaughtException-listener', (req, res) => { + process.on('uncaughtException', () => { + // wanted a log to force error on tests + // eslint-disable-next-line no-console + console.log('Custom uncaught exception capture callback') + res.writeHead(500) + res.end('error') + }) + + http.get(`https://${req.query.host}`, () => { + res.send('end') + }) +}) + +app.get('/ssrf/http/unhandled-error', (req, res) => { + makeOutgoingRequestAndCbAfterTimeout(req, res) +}) + +app.get('/ssrf/http/unhandled-async-write-A', (req, res) => { + makeOutgoingRequestAndCbAfterTimeout(req, res, () => { + res.send('Late end') + }) +}) + +app.get('/ssrf/http/unhandled-async-write-B', (req, res) => { + makeOutgoingRequestAndCbAfterTimeout(req, res, () => { + streamFile(res) + }) +}) + +app.get('/ssrf/http/unhandled-async-write-C', (req, res) => { + makeOutgoingRequestAndCbAfterTimeout(req, res, () => { + res.setHeader('key', 'value') + res.writeHead(200, 'OK', ['key2', 'value2']) + res.write('test\n') + res.end('end') + }) +}) + +app.get('/ssrf/http/unhandled-async-write-D', (req, res) => { + makeOutgoingRequestAndCbAfterTimeout(req, res, () => { + res.setHeader('key', 'value') + res.appendHeader?.('key2', 'value2') + res.removeHeader('key') + res.flushHeaders() + res.end('end') + }) +}) + +app.get('/ssrf/http/unhandled-async-write-E', (req, res) => { + makeOutgoingRequestAndCbAfterTimeout(req, res, () => { + res.writeContinue() + res.end() + }) +}) + +app.get('/ssrf/http/unhandled-async-write-F', (req, res) => { + makeOutgoingRequestAndCbAfterTimeout(req, res, () => { + res.writeProcessing() + res.end() + }) +}) + +app.get('/ssrf/http/unhandled-async-write-G', (req, res) => { + makeOutgoingRequestAndCbAfterTimeout(req, res, () => { + const earlyHintsLink = '; rel=preload; as=style' + res.writeEarlyHints?.({ + link: earlyHintsLink + }) + res.end() + }) +}) + +app.get('/ssrf/http/unhandled-async-write-H', (req, res) => { + makeOutgoingRequestAndCbAfterTimeout(req, res, () => { + res.json({ key: 'value' }) + }) +}) + +app.get('/ssrf/http/unhandled-axios', (req, res) => { + axios.get(`https://${req.query.host}`) + .then(() => res.end('end')) +}) + +app.get('/ssrf/http/unhandled-promise', (req, res) => { + httpGetPromise(req.query.host) + .then(() => res.end('end')) +}) + +app.listen(port, () => { + process.send({ port }) +}) diff --git a/integration-tests/appsec/rasp/rasp_rules.json b/integration-tests/appsec/rasp/rasp_rules.json new file mode 100644 index 00000000000..6e6913b0311 --- /dev/null +++ b/integration-tests/appsec/rasp/rasp_rules.json @@ -0,0 +1,59 @@ +{ + "version": "2.2", + "metadata": { + "rules_version": "1.99.0" + }, + "rules": [ + { + "id": "test-rule-id-2", + "name": "Server-side request forgery exploit", + "enabled": true, + "tags": { + "type": "ssrf", + "category": "vulnerability_trigger", + "cwe": "918", + "capec": "1000/225/115/664", + "confidence": "0", + "module": "rasp" + }, + "conditions": [ + { + "parameters": { + "resource": [ + { + "address": "server.io.net.url" + } + ], + "params": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + }, + { + "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" + } + ] + }, + "operator": "ssrf_detector" + } + ], + "transformers": [], + "on_match": [ + "block", + "stack_trace" + ] + } + ] +} + diff --git a/integration-tests/appsec/rasp/streamtest.txt b/integration-tests/appsec/rasp/streamtest.txt new file mode 100644 index 00000000000..9daeafb9864 --- /dev/null +++ b/integration-tests/appsec/rasp/streamtest.txt @@ -0,0 +1 @@ +test diff --git a/integration-tests/automatic-log-submission.spec.js b/integration-tests/automatic-log-submission.spec.js new file mode 100644 index 00000000000..eade717dcf1 --- /dev/null +++ b/integration-tests/automatic-log-submission.spec.js @@ -0,0 +1,207 @@ +'use strict' + +const { exec } = require('child_process') + +const { assert } = require('chai') +const getPort = require('get-port') + +const { + createSandbox, + getCiVisAgentlessConfig, + getCiVisEvpProxyConfig +} = require('./helpers') +const { FakeCiVisIntake } = require('./ci-visibility-intake') +const webAppServer = require('./ci-visibility/web-app-server') +const { NODE_MAJOR } = require('../version') + +const cucumberVersion = NODE_MAJOR <= 16 ? '9' : 'latest' + +describe('test visibility automatic log submission', () => { + let sandbox, cwd, receiver, childProcess, webAppPort + let testOutput = '' + + before(async () => { + sandbox = await createSandbox([ + 'mocha', + `@cucumber/cucumber@${cucumberVersion}`, + 'jest', + 'winston', + 'chai@4' + ], true) + cwd = sandbox.folder + webAppPort = await getPort() + webAppServer.listen(webAppPort) + }) + + after(async () => { + await sandbox.remove() + await new Promise(resolve => webAppServer.close(resolve)) + }) + + beforeEach(async function () { + const port = await getPort() + receiver = await new FakeCiVisIntake(port).start() + }) + + afterEach(async () => { + testOutput = '' + childProcess.kill() + await receiver.stop() + }) + + const testFrameworks = [ + { + name: 'mocha', + command: 'mocha ./ci-visibility/automatic-log-submission/automatic-log-submission-test.js' + }, + { + name: 'jest', + command: 'node ./node_modules/jest/bin/jest --config ./ci-visibility/automatic-log-submission/config-jest.js' + }, + { + name: 'cucumber', + command: './node_modules/.bin/cucumber-js ci-visibility/automatic-log-submission-cucumber/*.feature' + } + ] + + testFrameworks.forEach(({ name, command }) => { + context(`with ${name}`, () => { + it('can automatically submit logs', (done) => { + let logIds, testIds + + const logsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.includes('/api/v2/logs'), payloads => { + payloads.forEach(({ headers }) => { + assert.equal(headers['dd-api-key'], '1') + }) + const logMessages = payloads.flatMap(({ logMessage }) => logMessage) + const [url] = payloads.flatMap(({ url }) => url) + + assert.equal(url, '/api/v2/logs?ddsource=winston&service=my-service') + assert.equal(logMessages.length, 2) + + logMessages.forEach(({ dd, level }) => { + assert.equal(level, 'info') + assert.equal(dd.service, 'my-service') + assert.hasAllKeys(dd, ['trace_id', 'span_id', 'service']) + }) + + assert.includeMembers(logMessages.map(({ message }) => message), [ + 'Hello simple log!', + 'sum function being called' + ]) + + logIds = { + logSpanId: logMessages[0].dd.span_id, + logTraceId: logMessages[0].dd.trace_id + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testEventContent = events.find(event => event.type === 'test').content + + testIds = { + testSpanId: testEventContent.span_id.toString(), + testTraceId: testEventContent.trace_id.toString() + } + }) + + childProcess = exec(command, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + DD_AGENTLESS_LOG_SUBMISSION_ENABLED: '1', + DD_AGENTLESS_LOG_SUBMISSION_URL: `http://localhost:${receiver.port}`, + DD_API_KEY: '1', + DD_SERVICE: 'my-service' + }, + stdio: 'pipe' + } + ) + childProcess.on('exit', () => { + Promise.all([logsPromise, eventsPromise]).then(() => { + const { logSpanId, logTraceId } = logIds + const { testSpanId, testTraceId } = testIds + assert.include(testOutput, 'Hello simple log!') + assert.include(testOutput, 'sum function being called') + // cucumber has `cucumber.step`, and that's the active span, not the test. + // logs are queried by trace id, so it should be OK + if (name !== 'cucumber') { + assert.include(testOutput, `"span_id":"${testSpanId}"`) + assert.equal(logSpanId, testSpanId) + } + assert.include(testOutput, `"trace_id":"${testTraceId}"`) + assert.equal(logTraceId, testTraceId) + done() + }).catch(done) + }) + + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + }) + + it('does not submit logs when DD_AGENTLESS_LOG_SUBMISSION_ENABLED is not set', (done) => { + childProcess = exec(command, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + DD_AGENTLESS_LOG_SUBMISSION_URL: `http://localhost:${receiver.port}`, + DD_SERVICE: 'my-service' + }, + stdio: 'pipe' + } + ) + childProcess.on('exit', () => { + assert.include(testOutput, 'Hello simple log!') + assert.notInclude(testOutput, 'span_id') + done() + }) + + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + }) + + it('does not submit logs when DD_AGENTLESS_LOG_SUBMISSION_ENABLED is set but DD_API_KEY is not', (done) => { + childProcess = exec(command, + { + cwd, + env: { + ...getCiVisEvpProxyConfig(receiver.port), + DD_AGENTLESS_LOG_SUBMISSION_ENABLED: '1', + DD_AGENTLESS_LOG_SUBMISSION_URL: `http://localhost:${receiver.port}`, + DD_SERVICE: 'my-service', + DD_TRACE_DEBUG: '1', + DD_TRACE_LOG_LEVEL: 'warn', + DD_API_KEY: '' + }, + stdio: 'pipe' + } + ) + childProcess.on('exit', () => { + assert.include(testOutput, 'Hello simple log!') + assert.include(testOutput, 'no automatic log submission will be performed') + done() + }) + + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + }) + }) + }) +}) diff --git a/integration-tests/ci-visibility-intake.js b/integration-tests/ci-visibility-intake.js index 2efbba2de03..c133a7a31fe 100644 --- a/integration-tests/ci-visibility-intake.js +++ b/integration-tests/ci-visibility-intake.js @@ -5,27 +5,49 @@ const codec = msgpack.createCodec({ int64: true }) const http = require('http') const multer = require('multer') const upload = multer() +const zlib = require('zlib') const { FakeAgent } = require('./helpers') const DEFAULT_SETTINGS = { code_coverage: true, tests_skipping: true, - itr_enabled: true + itr_enabled: true, + early_flake_detection: { + enabled: false, + slow_test_retries: { + '5s': 3 + } + } } const DEFAULT_SUITES_TO_SKIP = [] const DEFAULT_GIT_UPLOAD_STATUS = 200 +const DEFAULT_KNOWN_TESTS_UPLOAD_STATUS = 200 const DEFAULT_INFO_RESPONSE = { endpoints: ['/evp_proxy/v2'] } +const DEFAULT_CORRELATION_ID = '1234' +const DEFAULT_KNOWN_TESTS = ['test-suite1.js.test-name1', 'test-suite2.js.test-name2'] let settings = DEFAULT_SETTINGS let suitesToSkip = DEFAULT_SUITES_TO_SKIP let gitUploadStatus = DEFAULT_GIT_UPLOAD_STATUS let infoResponse = DEFAULT_INFO_RESPONSE +let correlationId = DEFAULT_CORRELATION_ID +let knownTests = DEFAULT_KNOWN_TESTS +let knownTestsStatusCode = DEFAULT_KNOWN_TESTS_UPLOAD_STATUS +let waitingTime = 0 class FakeCiVisIntake extends FakeAgent { + setKnownTestsResponseCode (statusCode) { + knownTestsStatusCode = statusCode + } + + setKnownTests (newKnownTestsResponse) { + knownTests = newKnownTestsResponse + } + setInfoResponse (newInfoResponse) { infoResponse = newInfoResponse } @@ -38,10 +60,18 @@ class FakeCiVisIntake extends FakeAgent { suitesToSkip = newSuitesToSkip } + setItrCorrelationId (newCorrelationId) { + correlationId = newCorrelationId + } + setSettings (newSettings) { settings = newSettings } + setWaitingTime (newWaitingTime) { + waitingTime = newWaitingTime + } + async start () { const app = express() app.use(bodyParser.raw({ limit: Infinity, type: 'application/msgpack' })) @@ -64,18 +94,21 @@ class FakeCiVisIntake extends FakeAgent { }) }) - app.post(['/api/v2/citestcycle', '/evp_proxy/v2/api/v2/citestcycle'], (req, res) => { - res.status(200).send('OK') - this.emit('message', { - headers: req.headers, - payload: msgpack.decode(req.body, { codec }), - url: req.url - }) + // It can be slowed down with setWaitingTime + app.post(['/api/v2/citestcycle', '/evp_proxy/:version/api/v2/citestcycle'], (req, res) => { + this.waitingTimeoutId = setTimeout(() => { + res.status(200).send('OK') + this.emit('message', { + headers: req.headers, + payload: msgpack.decode(req.body, { codec }), + url: req.url + }) + }, waitingTime || 0) }) app.post([ '/api/v2/git/repository/search_commits', - '/evp_proxy/v2/api/v2/git/repository/search_commits' + '/evp_proxy/:version/api/v2/git/repository/search_commits' ], (req, res) => { res.status(gitUploadStatus).send(JSON.stringify({ data: [] })) this.emit('message', { @@ -87,7 +120,7 @@ class FakeCiVisIntake extends FakeAgent { app.post([ '/api/v2/git/repository/packfile', - '/evp_proxy/v2/api/v2/git/repository/packfile' + '/evp_proxy/:version/api/v2/git/repository/packfile' ], (req, res) => { res.status(202).send('') this.emit('message', { @@ -98,7 +131,7 @@ class FakeCiVisIntake extends FakeAgent { app.post([ '/api/v2/citestcov', - '/evp_proxy/v2/api/v2/citestcov' + '/evp_proxy/:version/api/v2/citestcov' ], upload.any(), (req, res) => { res.status(200).send('OK') @@ -122,7 +155,7 @@ class FakeCiVisIntake extends FakeAgent { app.post([ '/api/v2/libraries/tests/services/setting', - '/evp_proxy/v2/api/v2/libraries/tests/services/setting' + '/evp_proxy/:version/api/v2/libraries/tests/services/setting' ], (req, res) => { res.status(200).send(JSON.stringify({ data: { @@ -137,10 +170,13 @@ class FakeCiVisIntake extends FakeAgent { app.post([ '/api/v2/ci/tests/skippable', - '/evp_proxy/v2/api/v2/ci/tests/skippable' + '/evp_proxy/:version/api/v2/ci/tests/skippable' ], (req, res) => { res.status(200).send(JSON.stringify({ - data: suitesToSkip + data: suitesToSkip, + meta: { + correlation_id: correlationId + } })) this.emit('message', { headers: req.headers, @@ -148,6 +184,39 @@ class FakeCiVisIntake extends FakeAgent { }) }) + app.post([ + '/api/v2/ci/libraries/tests', + '/evp_proxy/:version/api/v2/ci/libraries/tests' + ], (req, res) => { + // The endpoint returns compressed data if 'accept-encoding' is set to 'gzip' + const isGzip = req.headers['accept-encoding'] === 'gzip' + const data = JSON.stringify({ + data: { + attributes: { + tests: knownTests + } + } + }) + res.setHeader('content-type', 'application/json') + if (isGzip) { + res.setHeader('content-encoding', 'gzip') + } + res.status(knownTestsStatusCode).send(isGzip ? zlib.gzipSync(data) : data) + this.emit('message', { + headers: req.headers, + url: req.url + }) + }) + + app.post('/api/v2/logs', express.json(), (req, res) => { + res.status(200).send('OK') + this.emit('message', { + headers: req.headers, + url: req.url, + logMessage: req.body + }) + }) + return new Promise((resolve, reject) => { const timeoutObj = setTimeout(() => { reject(new Error('Intake timed out starting up')) @@ -166,8 +235,13 @@ class FakeCiVisIntake extends FakeAgent { settings = DEFAULT_SETTINGS suitesToSkip = DEFAULT_SUITES_TO_SKIP gitUploadStatus = DEFAULT_GIT_UPLOAD_STATUS + knownTestsStatusCode = DEFAULT_KNOWN_TESTS_UPLOAD_STATUS infoResponse = DEFAULT_INFO_RESPONSE this.removeAllListeners() + if (this.waitingTimeoutId) { + clearTimeout(this.waitingTimeoutId) + } + waitingTime = 0 return super.stop() } diff --git a/integration-tests/ci-visibility.spec.js b/integration-tests/ci-visibility.spec.js deleted file mode 100644 index b9cf69c4c41..00000000000 --- a/integration-tests/ci-visibility.spec.js +++ /dev/null @@ -1,1590 +0,0 @@ -'use strict' - -const { fork, exec } = require('child_process') -const path = require('path') - -const { assert } = require('chai') -const getPort = require('get-port') - -const { - createSandbox, - getCiVisAgentlessConfig, - getCiVisEvpProxyConfig -} = require('./helpers') -const { FakeCiVisIntake } = require('./ci-visibility-intake') - -const { - TEST_CODE_COVERAGE_ENABLED, - TEST_ITR_SKIPPING_ENABLED, - TEST_ITR_TESTS_SKIPPED, - TEST_CODE_COVERAGE_LINES_PCT, - TEST_SUITE, - TEST_STATUS, - TEST_SKIPPED_BY_ITR, - TEST_ITR_SKIPPING_TYPE, - TEST_ITR_SKIPPING_COUNT, - TEST_ITR_UNSKIPPABLE, - TEST_ITR_FORCED_RUN -} = require('../packages/dd-trace/src/plugins/util/test') -const { ERROR_MESSAGE } = require('../packages/dd-trace/src/constants') - -const hookFile = 'dd-trace/loader-hook.mjs' - -const mochaCommonOptions = { - name: 'mocha', - expectedStdout: '2 passing', - extraStdout: 'end event: can add event listeners to mocha' -} - -const jestCommonOptions = { - name: 'jest', - dependencies: ['jest', 'chai', 'jest-jasmine2'], - expectedStdout: 'Test Suites: 2 passed', - expectedCoverageFiles: [ - 'ci-visibility/test/sum.js', - 'ci-visibility/test/ci-visibility-test.js', - 'ci-visibility/test/ci-visibility-test-2.js' - ] -} - -const testFrameworks = [ - { - ...mochaCommonOptions, - testFile: 'ci-visibility/run-mocha.js', - dependencies: ['mocha', 'chai', 'nyc'], - expectedCoverageFiles: [ - 'ci-visibility/run-mocha.js', - 'ci-visibility/test/sum.js', - 'ci-visibility/test/ci-visibility-test.js', - 'ci-visibility/test/ci-visibility-test-2.js' - ], - runTestsWithCoverageCommand: './node_modules/nyc/bin/nyc.js -r=text-summary node ./ci-visibility/run-mocha.js', - type: 'commonJS' - }, - { - ...mochaCommonOptions, - testFile: 'ci-visibility/run-mocha.mjs', - dependencies: ['mocha', 'chai', 'nyc', '@istanbuljs/esm-loader-hook'], - expectedCoverageFiles: [ - 'ci-visibility/run-mocha.mjs', - 'ci-visibility/test/sum.js', - 'ci-visibility/test/ci-visibility-test.js', - 'ci-visibility/test/ci-visibility-test-2.js' - ], - runTestsWithCoverageCommand: - `./node_modules/nyc/bin/nyc.js -r=text-summary ` + - `node --loader=./node_modules/@istanbuljs/esm-loader-hook/index.js ` + - `--loader=${hookFile} ./ci-visibility/run-mocha.mjs`, - type: 'esm' - }, - { - ...jestCommonOptions, - testFile: 'ci-visibility/run-jest.js', - runTestsWithCoverageCommand: 'node ./ci-visibility/run-jest.js', - type: 'commonJS' - }, - { - ...jestCommonOptions, - testFile: 'ci-visibility/run-jest.mjs', - runTestsWithCoverageCommand: `node --loader=${hookFile} ./ci-visibility/run-jest.mjs`, - type: 'esm' - } -] - -testFrameworks.forEach(({ - name, - dependencies, - testFile, - expectedStdout, - extraStdout, - expectedCoverageFiles, - runTestsWithCoverageCommand, - type -}) => { - // temporary fix for failing esm tests on the CI, skip for now for the release and comeback to solve the issue - if (type === 'esm') { - return - } - - // to avoid this error: @istanbuljs/esm-loader-hook@0.2.0: The engine "node" - // is incompatible with this module. Expected version ">=16.12.0". Got "14.21.3" - // if (type === 'esm' && name === 'mocha' && semver.satisfies(process.version, '<16.12.0')) { - // return - // } - describe(`${name} ${type}`, () => { - let receiver - let childProcess - let sandbox - let cwd - let startupTestFile - let testOutput = '' - - before(async function () { - // add an explicit timeout to make esm tests less flaky - this.timeout(50000) - sandbox = await createSandbox(dependencies, true) - cwd = sandbox.folder - startupTestFile = path.join(cwd, testFile) - }) - - after(async function () { - await sandbox.remove() - }) - - beforeEach(async function () { - const port = await getPort() - receiver = await new FakeCiVisIntake(port).start() - }) - - afterEach(async () => { - childProcess.kill() - testOutput = '' - await receiver.stop() - }) - - if (name === 'mocha') { - it('does not change mocha config if CI Visibility fails to init', (done) => { - receiver.assertPayloadReceived(() => { - const error = new Error('it should not report tests') - done(error) - }, ({ url }) => url === '/api/v2/citestcycle', 3000).catch(() => {}) - - const { DD_CIVISIBILITY_AGENTLESS_URL, ...restEnvVars } = getCiVisAgentlessConfig(receiver.port) - - // `runMocha` is only executed when using the CLI, which is where we modify mocha config - // if CI Visibility is init - childProcess = exec('mocha ./ci-visibility/test/ci-visibility-test.js', { - cwd, - env: { - ...restEnvVars, - DD_TRACE_DEBUG: 1, - DD_TRACE_LOG_LEVEL: 'error', - DD_SITE: '= invalid = url' - }, - stdio: 'pipe' - }) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.on('exit', () => { - assert.include(testOutput, 'Invalid URL') - assert.include(testOutput, '1 passing') // we only run one file here - done() - }) - }).timeout(50000) - - it('does not init CI Visibility when running in parallel mode', (done) => { - receiver.assertPayloadReceived(() => { - const error = new Error('it should not report tests') - done(error) - }, ({ url }) => url === '/api/v2/citestcycle', 3000).catch(() => {}) - - childProcess = fork(testFile, { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - RUN_IN_PARALLEL: true, - DD_TRACE_DEBUG: 1, - DD_TRACE_LOG_LEVEL: 'warn' - }, - stdio: 'pipe' - }) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.on('message', () => { - assert.include(testOutput, 'Unable to initialize CI Visibility because Mocha is running in parallel mode.') - done() - }) - }) - } - - if (name === 'jest') { - it('works when sharding', (done) => { - receiver.payloadReceived(({ url }) => url === '/api/v2/citestcycle').then(events => { - const testSuiteEvents = events.payload.events.filter(event => event.type === 'test_suite_end') - assert.equal(testSuiteEvents.length, 3) - const testSuites = testSuiteEvents.map(span => span.content.meta[TEST_SUITE]) - - assert.includeMembers(testSuites, - [ - 'ci-visibility/sharding-test/sharding-test-5.js', - 'ci-visibility/sharding-test/sharding-test-4.js', - 'ci-visibility/sharding-test/sharding-test-1.js' - ] - ) - - const testSession = events.payload.events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') - - // We run the second shard - receiver.setSuitesToSkip([ - { - type: 'suite', - attributes: { - suite: 'ci-visibility/sharding-test/sharding-test-2.js' - } - }, - { - type: 'suite', - attributes: { - suite: 'ci-visibility/sharding-test/sharding-test-3.js' - } - } - ]) - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - TESTS_TO_RUN: 'sharding-test/sharding-test', - TEST_SHARD: '2/2' - }, - stdio: 'inherit' - } - ) - - receiver.payloadReceived(({ url }) => url === '/api/v2/citestcycle').then(secondShardEvents => { - const testSuiteEvents = secondShardEvents.payload.events.filter(event => event.type === 'test_suite_end') - - // The suites for this shard are to be skipped - assert.equal(testSuiteEvents.length, 2) - - testSuiteEvents.forEach(testSuite => { - assert.propertyVal(testSuite.content.meta, TEST_STATUS, 'skip') - assert.propertyVal(testSuite.content.meta, TEST_SKIPPED_BY_ITR, 'true') - }) - - const testSession = secondShardEvents - .payload - .events - .find(event => event.type === 'test_session_end').content - - assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'true') - assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_TYPE, 'suite') - assert.propertyVal(testSession.metrics, TEST_ITR_SKIPPING_COUNT, 2) - - done() - }) - }) - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - TESTS_TO_RUN: 'sharding-test/sharding-test', - TEST_SHARD: '1/2' - }, - stdio: 'inherit' - } - ) - }) - it('does not crash when jest is badly initialized', (done) => { - childProcess = fork('ci-visibility/run-jest-bad-init.js', { - cwd, - env: { - DD_TRACE_AGENT_PORT: receiver.port - }, - stdio: 'pipe' - }) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.on('message', () => { - assert.notInclude(testOutput, 'TypeError') - assert.include(testOutput, expectedStdout) - done() - }) - }) - it('does not crash when jest uses jest-jasmine2', (done) => { - childProcess = fork(testFile, { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - OLD_RUNNER: 1, - NODE_OPTIONS: '-r dd-trace/ci/init', - RUN_IN_PARALLEL: true - }, - stdio: 'pipe' - }) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.on('message', () => { - assert.notInclude(testOutput, 'TypeError') - done() - }) - }) - describe('when jest is using workers to run tests in parallel', () => { - it('reports tests when using the agent', (done) => { - receiver.setInfoResponse({ endpoints: [] }) - childProcess = fork(testFile, { - cwd, - env: { - DD_TRACE_AGENT_PORT: receiver.port, - NODE_OPTIONS: '-r dd-trace/ci/init', - RUN_IN_PARALLEL: true - }, - stdio: 'pipe' - }) - - receiver.gatherPayloads(({ url }) => url === '/v0.4/traces', 5000).then(tracesRequests => { - const testSpans = tracesRequests.flatMap(trace => trace.payload).flatMap(request => request) - assert.equal(testSpans.length, 2) - const spanTypes = testSpans.map(span => span.type) - assert.includeMembers(spanTypes, ['test']) - assert.notInclude(spanTypes, ['test_session_end', 'test_suite_end', 'test_module_end']) - receiver.setInfoResponse({ endpoints: ['/evp_proxy/v2'] }) - done() - }).catch(done) - }) - it('reports tests when using agentless', (done) => { - childProcess = fork(testFile, { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - RUN_IN_PARALLEL: true - }, - stdio: 'pipe' - }) - - receiver.gatherPayloads(({ url }) => url === '/api/v2/citestcycle', 5000).then(eventsRequests => { - const eventTypes = eventsRequests.map(({ payload }) => payload) - .flatMap(({ events }) => events) - .map(event => event.type) - - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - done() - }).catch(done) - }) - it('reports tests when using evp proxy', (done) => { - childProcess = fork(testFile, { - cwd, - env: { - ...getCiVisEvpProxyConfig(receiver.port), - RUN_IN_PARALLEL: true - }, - stdio: 'pipe' - }) - - receiver.gatherPayloads(({ url }) => url === '/evp_proxy/v2/api/v2/citestcycle', 5000) - .then(eventsRequests => { - const eventTypes = eventsRequests.map(({ payload }) => payload) - .flatMap(({ events }) => events) - .map(event => event.type) - - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - done() - }).catch(done) - }) - }) - it('reports timeout error message', (done) => { - childProcess = fork(testFile, { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - NODE_OPTIONS: '-r dd-trace/ci/init', - RUN_IN_PARALLEL: true, - TESTS_TO_RUN: 'timeout-test/timeout-test.js' - }, - stdio: 'pipe' - }) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.on('message', () => { - assert.include(testOutput, 'Exceeded timeout of 100 ms for a test') - done() - }) - }) - it('reports parsing errors in the test file', (done) => { - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const suites = events.filter(event => event.type === 'test_suite_end') - assert.equal(suites.length, 2) - - const resourceNames = suites.map(suite => suite.content.resource) - - assert.includeMembers(resourceNames, [ - 'test_suite.ci-visibility/test-parsing-error/parsing-error-2.js', - 'test_suite.ci-visibility/test-parsing-error/parsing-error.js' - ]) - suites.forEach(suite => { - assert.equal(suite.content.meta[TEST_STATUS], 'fail') - assert.include(suite.content.meta[ERROR_MESSAGE], 'chao') - }) - }) - childProcess = fork(testFile, { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - TESTS_TO_RUN: 'test-parsing-error/parsing-error' - }, - stdio: 'pipe' - }) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - it('does not report total code coverage % if user has not configured coverage manually', (done) => { - receiver.setSettings({ - itr_enabled: true, - code_coverage: true, - tests_skipping: false - }) - - receiver.assertPayloadReceived(({ payload }) => { - const testSession = payload.events.find(event => event.type === 'test_session_end').content - assert.notProperty(testSession.metrics, TEST_CODE_COVERAGE_LINES_PCT) - }, ({ url }) => url === '/api/v2/citestcycle').then(() => done()).catch(done) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - DISABLE_CODE_COVERAGE: '1' - }, - stdio: 'inherit' - } - ) - }) - it('reports total code coverage % even when ITR is disabled', (done) => { - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false - }) - - receiver.assertPayloadReceived(({ payload }) => { - const testSession = payload.events.find(event => event.type === 'test_session_end').content - assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) - }, ({ url }) => url === '/api/v2/citestcycle').then(() => done()).catch(done) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'inherit' - } - ) - }) - } - - it('can run tests and report spans', (done) => { - receiver.setInfoResponse({ endpoints: [] }) - receiver.payloadReceived(({ url }) => url === '/v0.4/traces').then(({ payload }) => { - const testSpans = payload.flatMap(trace => trace) - const resourceNames = testSpans.map(span => span.resource) - - assert.includeMembers(resourceNames, - [ - 'ci-visibility/test/ci-visibility-test.js.ci visibility can report tests', - 'ci-visibility/test/ci-visibility-test-2.js.ci visibility 2 can report tests 2' - ] - ) - - const areAllTestSpans = testSpans.every(span => span.name === `${name}.test`) - assert.isTrue(areAllTestSpans) - - assert.include(testOutput, expectedStdout) - - if (extraStdout) { - assert.include(testOutput, extraStdout) - } - - done() - }) - - childProcess = fork(startupTestFile, { - cwd, - env: { - DD_TRACE_AGENT_PORT: receiver.port, - NODE_OPTIONS: type === 'esm' ? `-r dd-trace/ci/init --loader=${hookFile}` : '-r dd-trace/ci/init' - }, - stdio: 'pipe' - }) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - }) - const envVarSettings = ['DD_TRACING_ENABLED', 'DD_TRACE_ENABLED'] - - envVarSettings.forEach(envVar => { - context(`when ${envVar}=false`, () => { - it('does not report spans but still runs tests', (done) => { - receiver.assertMessageReceived(() => { - done(new Error('Should not create spans')) - }).catch(() => {}) - - childProcess = fork(startupTestFile, { - cwd, - env: { - DD_TRACE_AGENT_PORT: receiver.port, - NODE_OPTIONS: '-r dd-trace/ci/init', - [envVar]: 'false' - }, - stdio: 'pipe' - }) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.on('message', () => { - assert.include(testOutput, expectedStdout) - done() - }) - }) - }) - }) - context('when no ci visibility init is used', () => { - it('does not crash', (done) => { - childProcess = fork(startupTestFile, { - cwd, - env: { - DD_TRACE_AGENT_PORT: receiver.port, - NODE_OPTIONS: '-r dd-trace/init' - }, - stdio: 'pipe' - }) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.on('message', () => { - assert.notInclude(testOutput, 'TypeError') - assert.notInclude(testOutput, 'Uncaught error outside test suite') - assert.include(testOutput, expectedStdout) - done() - }) - }) - }) - - describe('agentless', () => { - it('reports errors in test sessions', (done) => { - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_STATUS, 'fail') - const errorMessage = name === 'mocha' ? 'Failed tests: 1' : 'Failed test suites: 1. Failed tests: 1' - assert.include(testSession.meta[ERROR_MESSAGE], errorMessage) - }) - - let TESTS_TO_RUN = 'test/fail-test' - if (name === 'mocha') { - TESTS_TO_RUN = JSON.stringify([ - './test/fail-test.js' - ]) - } - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - TESTS_TO_RUN - }, - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - it('does not init if DD_API_KEY is not set', (done) => { - receiver.assertMessageReceived(() => { - done(new Error('Should not create spans')) - }).catch(() => {}) - - childProcess = fork(startupTestFile, { - cwd, - env: { - DD_CIVISIBILITY_AGENTLESS_ENABLED: 1, - NODE_OPTIONS: '-r dd-trace/ci/init' - }, - stdio: 'pipe' - }) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.on('message', () => { - assert.include(testOutput, expectedStdout) - assert.include(testOutput, 'DD_CIVISIBILITY_AGENTLESS_ENABLED is set, ' + - 'but neither DD_API_KEY nor DATADOG_API_KEY are set in your environment, ' + - 'so dd-trace will not be initialized.' - ) - done() - }) - }) - it('can report git metadata', (done) => { - const searchCommitsRequestPromise = receiver.payloadReceived( - ({ url }) => url === '/api/v2/git/repository/search_commits' - ) - const packfileRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/git/repository/packfile') - const eventsRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcycle') - - Promise.all([ - searchCommitsRequestPromise, - packfileRequestPromise, - eventsRequestPromise - ]).then(([searchCommitRequest, packfileRequest, eventsRequest]) => { - assert.propertyVal(searchCommitRequest.headers, 'dd-api-key', '1') - assert.propertyVal(packfileRequest.headers, 'dd-api-key', '1') - - const eventTypes = eventsRequest.payload.events.map(event => event.type) - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 - ) - assert.equal(numSuites, 2) - - done() - }).catch(done) - - childProcess = fork(startupTestFile, { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'pipe' - }) - }) - it('can report code coverage', (done) => { - let testOutput - const itrConfigRequestPromise = receiver.payloadReceived( - ({ url }) => url === '/api/v2/libraries/tests/services/setting' - ) - const codeCovRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcov') - const eventsRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcycle') - - Promise.all([ - itrConfigRequestPromise, - codeCovRequestPromise, - eventsRequestPromise - ]).then(([itrConfigRequest, codeCovRequest, eventsRequest]) => { - assert.propertyVal(itrConfigRequest.headers, 'dd-api-key', '1') - - const [coveragePayload] = codeCovRequest.payload - assert.propertyVal(codeCovRequest.headers, 'dd-api-key', '1') - - assert.propertyVal(coveragePayload, 'name', 'coverage1') - assert.propertyVal(coveragePayload, 'filename', 'coverage1.msgpack') - assert.propertyVal(coveragePayload, 'type', 'application/msgpack') - assert.include(coveragePayload.content, { - version: 2 - }) - const allCoverageFiles = codeCovRequest.payload - .flatMap(coverage => coverage.content.coverages) - .flatMap(file => file.files) - .map(file => file.filename) - - assert.includeMembers(allCoverageFiles, expectedCoverageFiles) - assert.exists(coveragePayload.content.coverages[0].test_session_id) - assert.exists(coveragePayload.content.coverages[0].test_suite_id) - - const testSession = eventsRequest.payload.events.find(event => event.type === 'test_session_end').content - assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) - - const eventTypes = eventsRequest.payload.events.map(event => event.type) - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 - ) - assert.equal(numSuites, 2) - }).catch(done) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'pipe' - } - ) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.on('exit', () => { - // coverage report - if (name === 'mocha') { - assert.include(testOutput, 'Lines ') - } - done() - }) - }) - it('does not report code coverage if disabled by the API', (done) => { - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false - }) - - receiver.assertPayloadReceived(() => { - const error = new Error('it should not report code coverage') - done(error) - }, ({ url }) => url === '/api/v2/citestcov').catch(() => {}) - - receiver.assertPayloadReceived(({ headers, payload }) => { - assert.propertyVal(headers, 'dd-api-key', '1') - const eventTypes = payload.events.map(event => event.type) - assert.includeMembers(eventTypes, ['test', 'test_session_end', 'test_module_end', 'test_suite_end']) - const testSession = payload.events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') - assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'false') - assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'false') - assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) - const testModule = payload.events.find(event => event.type === 'test_module_end').content - assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') - assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'false') - assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'false') - }, ({ url }) => url === '/api/v2/citestcycle').then(() => done()).catch(done) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'inherit' - } - ) - }) - it('can skip suites received by the intelligent test runner API and still reports code coverage', (done) => { - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test.js' - } - }]) - - const skippableRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/ci/tests/skippable') - const coverageRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcov') - const eventsRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcycle') - - Promise.all([ - skippableRequestPromise, - coverageRequestPromise, - eventsRequestPromise - ]).then(([skippableRequest, coverageRequest, eventsRequest]) => { - assert.propertyVal(skippableRequest.headers, 'dd-api-key', '1') - const [coveragePayload] = coverageRequest.payload - assert.propertyVal(coverageRequest.headers, 'dd-api-key', '1') - assert.propertyVal(coveragePayload, 'name', 'coverage1') - assert.propertyVal(coveragePayload, 'filename', 'coverage1.msgpack') - assert.propertyVal(coveragePayload, 'type', 'application/msgpack') - - assert.propertyVal(eventsRequest.headers, 'dd-api-key', '1') - const eventTypes = eventsRequest.payload.events.map(event => event.type) - const skippedSuite = eventsRequest.payload.events.find(event => - event.content.resource === 'test_suite.ci-visibility/test/ci-visibility-test.js' - ).content - assert.propertyVal(skippedSuite.meta, TEST_STATUS, 'skip') - assert.propertyVal(skippedSuite.meta, TEST_SKIPPED_BY_ITR, 'true') - - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 - ) - assert.equal(numSuites, 2) - const testSession = eventsRequest.payload.events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'true') - assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_TYPE, 'suite') - assert.propertyVal(testSession.metrics, TEST_ITR_SKIPPING_COUNT, 1) - const testModule = eventsRequest.payload.events.find(event => event.type === 'test_module_end').content - assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'true') - assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_TYPE, 'suite') - assert.propertyVal(testModule.metrics, TEST_ITR_SKIPPING_COUNT, 1) - done() - }).catch(done) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'inherit' - } - ) - }) - it('marks the test session as skipped if every suite is skipped', (done) => { - receiver.setSuitesToSkip( - [ - { - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test.js' - } - }, - { - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test-2.js' - } - } - ] - ) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_STATUS, 'skip') - }) - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - it('does not skip tests if git metadata upload fails', (done) => { - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test.js' - } - }]) - - receiver.setGitUploadStatus(404) - - receiver.assertPayloadReceived(() => { - const error = new Error('should not request skippable') - done(error) - }, ({ url }) => url === '/api/v2/ci/tests/skippable').catch(() => {}) - - receiver.assertPayloadReceived(({ headers, payload }) => { - assert.propertyVal(headers, 'dd-api-key', '1') - const eventTypes = payload.events.map(event => event.type) - // because they are not skipped - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 - ) - assert.equal(numSuites, 2) - const testSession = payload.events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') - assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - const testModule = payload.events.find(event => event.type === 'test_module_end').content - assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') - assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - }, ({ url }) => url === '/api/v2/citestcycle').then(() => done()).catch(done) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'inherit' - } - ) - }) - it('does not skip tests if test skipping is disabled by the API', (done) => { - receiver.setSettings({ - itr_enabled: true, - code_coverage: true, - tests_skipping: false - }) - - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test.js' - } - }]) - - receiver.assertPayloadReceived(() => { - const error = new Error('should not request skippable') - done(error) - }, ({ url }) => url === '/api/v2/ci/tests/skippable').catch(() => {}) - - receiver.assertPayloadReceived(({ headers, payload }) => { - assert.propertyVal(headers, 'dd-api-key', '1') - const eventTypes = payload.events.map(event => event.type) - // because they are not skipped - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 - ) - assert.equal(numSuites, 2) - }, ({ url }) => url === '/api/v2/citestcycle').then(() => done()).catch(done) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'inherit' - } - ) - }) - it('does not skip suites if suite is marked as unskippable', (done) => { - receiver.setSuitesToSkip([ - { - type: 'suite', - attributes: { - suite: 'ci-visibility/unskippable-test/test-to-skip.js' - } - }, - { - type: 'suite', - attributes: { - suite: 'ci-visibility/unskippable-test/test-unskippable.js' - } - } - ]) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const suites = events.filter(event => event.type === 'test_suite_end') - - assert.equal(suites.length, 2) - - const testSession = events.find(event => event.type === 'test_session_end').content - const testModule = events.find(event => event.type === 'test_module_end').content - assert.propertyVal(testSession.meta, TEST_ITR_FORCED_RUN, 'true') - assert.propertyVal(testSession.meta, TEST_ITR_UNSKIPPABLE, 'true') - assert.propertyVal(testModule.meta, TEST_ITR_FORCED_RUN, 'true') - assert.propertyVal(testModule.meta, TEST_ITR_UNSKIPPABLE, 'true') - - const skippedSuite = suites.find( - event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-to-skip.js' - ) - const forcedToRunSuite = suites.find( - event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-unskippable.js' - ) - - assert.propertyVal(skippedSuite.content.meta, TEST_STATUS, 'skip') - assert.notProperty(skippedSuite.content.meta, TEST_ITR_UNSKIPPABLE) - assert.notProperty(skippedSuite.content.meta, TEST_ITR_FORCED_RUN) - - assert.propertyVal(forcedToRunSuite.content.meta, TEST_STATUS, 'pass') - assert.propertyVal(forcedToRunSuite.content.meta, TEST_ITR_UNSKIPPABLE, 'true') - assert.propertyVal(forcedToRunSuite.content.meta, TEST_ITR_FORCED_RUN, 'true') - }, 25000) - - let TESTS_TO_RUN = 'unskippable-test/test-' - if (name === 'mocha') { - TESTS_TO_RUN = JSON.stringify([ - './unskippable-test/test-to-skip.js', - './unskippable-test/test-unskippable.js' - ]) - } - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - TESTS_TO_RUN - }, - stdio: 'inherit' - } - ) - - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - it('only sets forced to run if suite was going to be skipped by ITR', (done) => { - receiver.setSuitesToSkip([ - { - type: 'suite', - attributes: { - suite: 'ci-visibility/unskippable-test/test-to-skip.js' - } - } - ]) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const suites = events.filter(event => event.type === 'test_suite_end') - - assert.equal(suites.length, 2) - - const testSession = events.find(event => event.type === 'test_session_end').content - const testModule = events.find(event => event.type === 'test_module_end').content - assert.notProperty(testSession.meta, TEST_ITR_FORCED_RUN) - assert.propertyVal(testSession.meta, TEST_ITR_UNSKIPPABLE, 'true') - assert.notProperty(testModule.meta, TEST_ITR_FORCED_RUN) - assert.propertyVal(testModule.meta, TEST_ITR_UNSKIPPABLE, 'true') - - const skippedSuite = suites.find( - event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-to-skip.js' - ).content - const nonSkippedSuite = suites.find( - event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-unskippable.js' - ).content - - assert.propertyVal(skippedSuite.meta, TEST_STATUS, 'skip') - - assert.propertyVal(nonSkippedSuite.meta, TEST_STATUS, 'pass') - assert.propertyVal(nonSkippedSuite.meta, TEST_ITR_UNSKIPPABLE, 'true') - // it was not forced to run because it wasn't going to be skipped - assert.notProperty(nonSkippedSuite.meta, TEST_ITR_FORCED_RUN) - }, 25000) - - let TESTS_TO_RUN = 'unskippable-test/test-' - if (name === 'mocha') { - TESTS_TO_RUN = JSON.stringify([ - './unskippable-test/test-to-skip.js', - './unskippable-test/test-unskippable.js' - ]) - } - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - TESTS_TO_RUN - }, - stdio: 'inherit' - } - ) - - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - it('sets _dd.ci.itr.tests_skipped to false if the received suite is not skipped', (done) => { - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: 'ci-visibility/test/not-existing-test.js' - } - }]) - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') - assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - const testModule = events.find(event => event.type === 'test_module_end').content - assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') - assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - }, 25000) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - }) - - describe('evp proxy', () => { - context('if the agent is not event platform proxy compatible', () => { - it('does not do any intelligent test runner request', (done) => { - receiver.setInfoResponse({ endpoints: [] }) - - receiver.assertPayloadReceived(() => { - const error = new Error('should not request search_commits') - done(error) - }, ({ url }) => url === '/evp_proxy/v2/api/v2/git/repository/search_commits').catch(() => {}) - receiver.assertPayloadReceived(() => { - const error = new Error('should not request search_commits') - done(error) - }, ({ url }) => url === '/api/v2/git/repository/search_commits').catch(() => {}) - receiver.assertPayloadReceived(() => { - const error = new Error('should not request setting') - done(error) - }, ({ url }) => url === '/api/v2/libraries/tests/services/setting').catch(() => {}) - receiver.assertPayloadReceived(() => { - const error = new Error('should not request setting') - done(error) - }, ({ url }) => url === '/evp_proxy/v2/api/v2/libraries/tests/services/setting').catch(() => {}) - - receiver.assertPayloadReceived(({ payload }) => { - const testSpans = payload.flatMap(trace => trace) - const resourceNames = testSpans.map(span => span.resource) - - assert.includeMembers(resourceNames, - [ - 'ci-visibility/test/ci-visibility-test.js.ci visibility can report tests', - 'ci-visibility/test/ci-visibility-test-2.js.ci visibility 2 can report tests 2' - ] - ) - }, ({ url }) => url === '/v0.4/traces').then(() => done()).catch(done) - - childProcess = fork(startupTestFile, { - cwd, - env: getCiVisEvpProxyConfig(receiver.port), - stdio: 'pipe' - }) - }) - }) - it('reports errors in test sessions', (done) => { - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_STATUS, 'fail') - const errorMessage = name === 'mocha' ? 'Failed tests: 1' : 'Failed test suites: 1. Failed tests: 1' - assert.include(testSession.meta[ERROR_MESSAGE], errorMessage) - }) - - let TESTS_TO_RUN = 'test/fail-test' - if (name === 'mocha') { - TESTS_TO_RUN = JSON.stringify([ - './test/fail-test.js' - ]) - } - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: { - ...getCiVisEvpProxyConfig(receiver.port), - TESTS_TO_RUN - }, - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - it('can report git metadata', (done) => { - const infoRequestPromise = receiver.payloadReceived(({ url }) => url === '/info') - const searchCommitsRequestPromise = receiver.payloadReceived( - ({ url }) => url === '/evp_proxy/v2/api/v2/git/repository/search_commits' - ) - const packFileRequestPromise = receiver.payloadReceived( - ({ url }) => url === '/evp_proxy/v2/api/v2/git/repository/packfile' - ) - const eventsRequestPromise = receiver.payloadReceived(({ url }) => url === '/evp_proxy/v2/api/v2/citestcycle') - - Promise.all([ - infoRequestPromise, - searchCommitsRequestPromise, - packFileRequestPromise, - eventsRequestPromise - ]).then(([infoRequest, searchCommitsRequest, packfileRequest, eventsRequest]) => { - assert.notProperty(infoRequest.headers, 'dd-api-key') - - assert.notProperty(searchCommitsRequest.headers, 'dd-api-key') - assert.propertyVal(searchCommitsRequest.headers, 'x-datadog-evp-subdomain', 'api') - - assert.notProperty(packfileRequest.headers, 'dd-api-key') - assert.propertyVal(packfileRequest.headers, 'x-datadog-evp-subdomain', 'api') - - const eventTypes = eventsRequest.payload.events.map(event => event.type) - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 - ) - assert.equal(numSuites, 2) - done() - }).catch(done) - - childProcess = fork(startupTestFile, { - cwd, - env: getCiVisEvpProxyConfig(receiver.port), - stdio: 'pipe' - }) - }) - it('can report code coverage', (done) => { - let testOutput - const itrConfigRequestPromise = receiver.payloadReceived( - ({ url }) => url === '/evp_proxy/v2/api/v2/libraries/tests/services/setting' - ) - const codeCovRequestPromise = receiver.payloadReceived(({ url }) => url === '/evp_proxy/v2/api/v2/citestcov') - const eventsRequestPromise = receiver.payloadReceived(({ url }) => url === '/evp_proxy/v2/api/v2/citestcycle') - - Promise.all([ - itrConfigRequestPromise, - codeCovRequestPromise, - eventsRequestPromise - ]).then(([itrConfigRequest, codeCovRequest, eventsRequest]) => { - assert.notProperty(itrConfigRequest.headers, 'dd-api-key') - assert.propertyVal(itrConfigRequest.headers, 'x-datadog-evp-subdomain', 'api') - - const [coveragePayload] = codeCovRequest.payload - assert.notProperty(codeCovRequest.headers, 'dd-api-key') - - assert.propertyVal(coveragePayload, 'name', 'coverage1') - assert.propertyVal(coveragePayload, 'filename', 'coverage1.msgpack') - assert.propertyVal(coveragePayload, 'type', 'application/msgpack') - assert.include(coveragePayload.content, { - version: 2 - }) - const allCoverageFiles = codeCovRequest.payload - .flatMap(coverage => coverage.content.coverages) - .flatMap(file => file.files) - .map(file => file.filename) - - assert.includeMembers(allCoverageFiles, expectedCoverageFiles) - assert.exists(coveragePayload.content.coverages[0].test_session_id) - assert.exists(coveragePayload.content.coverages[0].test_suite_id) - - const testSession = eventsRequest.payload.events.find(event => event.type === 'test_session_end').content - assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) - - const eventTypes = eventsRequest.payload.events.map(event => event.type) - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 - ) - assert.equal(numSuites, 2) - }).catch(done) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisEvpProxyConfig(receiver.port), - stdio: 'pipe' - } - ) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.on('exit', () => { - // coverage report - if (name === 'mocha') { - assert.include(testOutput, 'Lines ') - } - done() - }) - }) - it('does not report code coverage if disabled by the API', (done) => { - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false - }) - - receiver.assertPayloadReceived(() => { - const error = new Error('it should not report code coverage') - done(error) - }, ({ url }) => url === '/evp_proxy/v2/api/v2/citestcov').catch(() => {}) - - receiver.assertPayloadReceived(({ headers, payload }) => { - assert.notProperty(headers, 'dd-api-key') - assert.propertyVal(headers, 'x-datadog-evp-subdomain', 'citestcycle-intake') - const eventTypes = payload.events.map(event => event.type) - assert.includeMembers(eventTypes, ['test', 'test_session_end', 'test_module_end', 'test_suite_end']) - const testSession = payload.events.find(event => event.type === 'test_session_end').content - assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) - }, ({ url }) => url === '/evp_proxy/v2/api/v2/citestcycle').then(() => done()).catch(done) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisEvpProxyConfig(receiver.port), - stdio: 'inherit' - } - ) - }) - it('can skip suites received by the intelligent test runner API and still reports code coverage', (done) => { - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test.js' - } - }]) - - const skippableRequestPromise = receiver.payloadReceived( - ({ url }) => url === '/evp_proxy/v2/api/v2/ci/tests/skippable' - ) - const coverageRequestPromise = receiver.payloadReceived(({ url }) => url === '/evp_proxy/v2/api/v2/citestcov') - const eventsRequestPromise = receiver.payloadReceived(({ url }) => url === '/evp_proxy/v2/api/v2/citestcycle') - - Promise.all([ - skippableRequestPromise, - coverageRequestPromise, - eventsRequestPromise - ]).then(([skippableRequest, coverageRequest, eventsRequest]) => { - assert.notProperty(skippableRequest.headers, 'dd-api-key') - assert.propertyVal(skippableRequest.headers, 'x-datadog-evp-subdomain', 'api') - - const [coveragePayload] = coverageRequest.payload - assert.notProperty(coverageRequest.headers, 'dd-api-key') - assert.propertyVal(coverageRequest.headers, 'x-datadog-evp-subdomain', 'citestcov-intake') - assert.propertyVal(coveragePayload, 'name', 'coverage1') - assert.propertyVal(coveragePayload, 'filename', 'coverage1.msgpack') - assert.propertyVal(coveragePayload, 'type', 'application/msgpack') - - assert.notProperty(eventsRequest.headers, 'dd-api-key') - assert.propertyVal(eventsRequest.headers, 'x-datadog-evp-subdomain', 'citestcycle-intake') - const eventTypes = eventsRequest.payload.events.map(event => event.type) - const skippedSuite = eventsRequest.payload.events.find(event => - event.content.resource === 'test_suite.ci-visibility/test/ci-visibility-test.js' - ).content - assert.propertyVal(skippedSuite.meta, TEST_STATUS, 'skip') - assert.propertyVal(skippedSuite.meta, TEST_SKIPPED_BY_ITR, 'true') - - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 - ) - assert.equal(numSuites, 2) - done() - }).catch(done) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisEvpProxyConfig(receiver.port), - stdio: 'inherit' - } - ) - }) - it('marks the test session as skipped if every suite is skipped', (done) => { - receiver.setSuitesToSkip( - [ - { - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test.js' - } - }, - { - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test-2.js' - } - } - ] - ) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_STATUS, 'skip') - }) - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - it('marks the test session as skipped if every suite is skipped', (done) => { - receiver.setSuitesToSkip( - [ - { - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test.js' - } - }, - { - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test-2.js' - } - } - ] - ) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_STATUS, 'skip') - }) - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisEvpProxyConfig(receiver.port), - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - it('does not skip tests if git metadata upload fails', (done) => { - receiver.assertPayloadReceived(() => { - const error = new Error('should not request skippable') - done(error) - }, ({ url }) => url === '/evp_proxy/v2/api/v2/ci/tests/skippable').catch(() => {}) - - receiver.assertPayloadReceived(({ headers, payload }) => { - assert.notProperty(headers, 'dd-api-key') - assert.propertyVal(headers, 'x-datadog-evp-subdomain', 'citestcycle-intake') - const eventTypes = payload.events.map(event => event.type) - // because they are not skipped - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 - ) - assert.equal(numSuites, 2) - }, ({ url }) => url === '/evp_proxy/v2/api/v2/citestcycle').then(() => done()).catch(done) - - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test.js' - } - }]) - - receiver.setGitUploadStatus(404) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisEvpProxyConfig(receiver.port), - stdio: 'inherit' - } - ) - }) - it('does not skip tests if test skipping is disabled by the API', (done) => { - receiver.assertPayloadReceived(() => { - const error = new Error('should not request skippable') - done(error) - }, ({ url }) => url === '/evp_proxy/v2/api/v2/ci/tests/skippable').catch(() => {}) - - receiver.assertPayloadReceived(({ headers, payload }) => { - assert.notProperty(headers, 'dd-api-key') - assert.propertyVal(headers, 'x-datadog-evp-subdomain', 'citestcycle-intake') - const eventTypes = payload.events.map(event => event.type) - // because they are not skipped - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 - ) - assert.equal(numSuites, 2) - }, ({ url }) => url === '/evp_proxy/v2/api/v2/citestcycle').then(() => done()).catch(done) - - receiver.setSettings({ - itr_enabled: true, - code_coverage: true, - tests_skipping: false - }) - - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test.js' - } - }]) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisEvpProxyConfig(receiver.port), - stdio: 'inherit' - } - ) - }) - it('sets _dd.ci.itr.tests_skipped to false if the received suite is not skipped', (done) => { - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: 'ci-visibility/test/not-existing-test.js' - } - }]) - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') - assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - const testModule = events.find(event => event.type === 'test_module_end').content - assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') - assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - }, 25000) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisEvpProxyConfig(receiver.port), - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - }) - }) -}) diff --git a/integration-tests/ci-visibility/automatic-log-submission-cucumber/automatic-log-submission.feature b/integration-tests/ci-visibility/automatic-log-submission-cucumber/automatic-log-submission.feature new file mode 100644 index 00000000000..bcce6b75bea --- /dev/null +++ b/integration-tests/ci-visibility/automatic-log-submission-cucumber/automatic-log-submission.feature @@ -0,0 +1,4 @@ +Feature: Automatic Log Submission + Scenario: Run Automatic Log Submission + When we run a test + Then I should have made a log diff --git a/integration-tests/ci-visibility/automatic-log-submission-cucumber/support/logger.js b/integration-tests/ci-visibility/automatic-log-submission-cucumber/support/logger.js new file mode 100644 index 00000000000..5480f1ee574 --- /dev/null +++ b/integration-tests/ci-visibility/automatic-log-submission-cucumber/support/logger.js @@ -0,0 +1,10 @@ +const { createLogger, format, transports } = require('winston') + +module.exports = createLogger({ + level: 'info', + exitOnError: false, + format: format.json(), + transports: [ + new transports.Console() + ] +}) diff --git a/integration-tests/ci-visibility/automatic-log-submission-cucumber/support/steps.js b/integration-tests/ci-visibility/automatic-log-submission-cucumber/support/steps.js new file mode 100644 index 00000000000..2d1bdb4e906 --- /dev/null +++ b/integration-tests/ci-visibility/automatic-log-submission-cucumber/support/steps.js @@ -0,0 +1,14 @@ +const { expect } = require('chai') +const { When, Then } = require('@cucumber/cucumber') + +const logger = require('./logger') +const sum = require('./sum') + +Then('I should have made a log', async function () { + expect(true).to.equal(true) + expect(sum(1, 2)).to.equal(3) +}) + +When('we run a test', async function () { + logger.log('info', 'Hello simple log!') +}) diff --git a/integration-tests/ci-visibility/automatic-log-submission-cucumber/support/sum.js b/integration-tests/ci-visibility/automatic-log-submission-cucumber/support/sum.js new file mode 100644 index 00000000000..cce61142972 --- /dev/null +++ b/integration-tests/ci-visibility/automatic-log-submission-cucumber/support/sum.js @@ -0,0 +1,6 @@ +const logger = require('./logger') + +module.exports = function (a, b) { + logger.log('info', 'sum function being called') + return a + b +} diff --git a/integration-tests/ci-visibility/automatic-log-submission/automatic-log-submission-test.js b/integration-tests/ci-visibility/automatic-log-submission/automatic-log-submission-test.js new file mode 100644 index 00000000000..cfc60b8d3b0 --- /dev/null +++ b/integration-tests/ci-visibility/automatic-log-submission/automatic-log-submission-test.js @@ -0,0 +1,13 @@ +const { expect } = require('chai') + +const logger = require('./logger') +const sum = require('./sum') + +describe('test', () => { + it('should return true', () => { + logger.log('info', 'Hello simple log!') + + expect(true).to.be.true + expect(sum(1, 2)).to.equal(3) + }) +}) diff --git a/integration-tests/ci-visibility/automatic-log-submission/config-jest.js b/integration-tests/ci-visibility/automatic-log-submission/config-jest.js new file mode 100644 index 00000000000..56afa0d36db --- /dev/null +++ b/integration-tests/ci-visibility/automatic-log-submission/config-jest.js @@ -0,0 +1,8 @@ +module.exports = { + projects: [], + testPathIgnorePatterns: ['/node_modules/'], + cache: false, + testMatch: [ + '**/ci-visibility/automatic-log-submission/automatic-log-submission-*' + ] +} diff --git a/integration-tests/ci-visibility/automatic-log-submission/logger.js b/integration-tests/ci-visibility/automatic-log-submission/logger.js new file mode 100644 index 00000000000..5480f1ee574 --- /dev/null +++ b/integration-tests/ci-visibility/automatic-log-submission/logger.js @@ -0,0 +1,10 @@ +const { createLogger, format, transports } = require('winston') + +module.exports = createLogger({ + level: 'info', + exitOnError: false, + format: format.json(), + transports: [ + new transports.Console() + ] +}) diff --git a/integration-tests/ci-visibility/automatic-log-submission/sum.js b/integration-tests/ci-visibility/automatic-log-submission/sum.js new file mode 100644 index 00000000000..cce61142972 --- /dev/null +++ b/integration-tests/ci-visibility/automatic-log-submission/sum.js @@ -0,0 +1,6 @@ +const logger = require('./logger') + +module.exports = function (a, b) { + logger.log('info', 'sum function being called') + return a + b +} diff --git a/integration-tests/ci-visibility/features-flaky/flaky.feature b/integration-tests/ci-visibility/features-flaky/flaky.feature new file mode 100644 index 00000000000..c24e4a61319 --- /dev/null +++ b/integration-tests/ci-visibility/features-flaky/flaky.feature @@ -0,0 +1,4 @@ +Feature: Farewell + Scenario: Say flaky + When the greeter says flaky + Then I should have heard "flaky" diff --git a/integration-tests/ci-visibility/features-flaky/support/steps.js b/integration-tests/ci-visibility/features-flaky/support/steps.js new file mode 100644 index 00000000000..2e4a335cfb7 --- /dev/null +++ b/integration-tests/ci-visibility/features-flaky/support/steps.js @@ -0,0 +1,18 @@ +const assert = require('assert') +const { When, Then } = require('@cucumber/cucumber') + +let globalCounter = 0 + +Then('I should have heard {string}', function (expectedResponse) { + assert.equal(this.whatIHeard, expectedResponse) +}) + +When('the greeter says flaky', function () { + // It's important that the first time this fails. The reason is the following: + // In `getWrappedRunTestCase` we were returning the first result from + // `runTestCaseFunction`, so if the first time it passed, the EFD logic was + // not kicking in. By making it fail, `runTestCaseResult` is false (fail), + // and the EFD logic is tested correctly, i.e. the test passes as long as a single + // attempt has passed. + this.whatIHeard = globalCounter++ % 2 === 1 ? 'flaky' : 'not flaky' +}) diff --git a/integration-tests/ci-visibility/features-retry/flaky.feature b/integration-tests/ci-visibility/features-retry/flaky.feature new file mode 100644 index 00000000000..c24e4a61319 --- /dev/null +++ b/integration-tests/ci-visibility/features-retry/flaky.feature @@ -0,0 +1,4 @@ +Feature: Farewell + Scenario: Say flaky + When the greeter says flaky + Then I should have heard "flaky" diff --git a/integration-tests/ci-visibility/features-retry/support/steps.js b/integration-tests/ci-visibility/features-retry/support/steps.js new file mode 100644 index 00000000000..50da213fb75 --- /dev/null +++ b/integration-tests/ci-visibility/features-retry/support/steps.js @@ -0,0 +1,15 @@ +const assert = require('assert') +const { When, Then } = require('@cucumber/cucumber') + +let globalCounter = 0 + +Then('I should have heard {string}', function (expectedResponse) { + assert.equal(this.whatIHeard, expectedResponse) +}) + +When('the greeter says flaky', function () { + if (++globalCounter < 3) { + throw new Error('Not good enough!') + } + this.whatIHeard = 'flaky' +}) diff --git a/integration-tests/ci-visibility/features-selenium/selenium.feature b/integration-tests/ci-visibility/features-selenium/selenium.feature new file mode 100644 index 00000000000..f2b26212d40 --- /dev/null +++ b/integration-tests/ci-visibility/features-selenium/selenium.feature @@ -0,0 +1,4 @@ +Feature: Selenium + Scenario: Run selenium + When we run selenium + Then I should have run selenium diff --git a/integration-tests/ci-visibility/features-selenium/support/steps.js b/integration-tests/ci-visibility/features-selenium/support/steps.js new file mode 100644 index 00000000000..307c947189f --- /dev/null +++ b/integration-tests/ci-visibility/features-selenium/support/steps.js @@ -0,0 +1,37 @@ +const { expect } = require('chai') +const { By, Builder } = require('selenium-webdriver') +const chrome = require('selenium-webdriver/chrome') +const { When, Then, Before, After } = require('@cucumber/cucumber') + +let driver +let title +let helloWorldText + +const options = new chrome.Options() +options.addArguments('--headless') + +Before(async function () { + const build = new Builder().forBrowser('chrome').setChromeOptions(options) + driver = await build.build() +}) + +After(async function () { + await driver.quit() +}) + +Then('I should have run selenium', async function () { + expect(title).to.equal('Hello World') + expect(helloWorldText).to.equal('Hello World') +}) + +When('we run selenium', async function () { + await driver.get(process.env.WEB_APP_URL) + + title = await driver.getTitle() + + await driver.manage().setTimeouts({ implicit: 500 }) + + const helloWorld = await driver.findElement(By.className('hello-world')) + + helloWorldText = await helloWorld.getText() +}) diff --git a/integration-tests/ci-visibility/features/farewell.feature b/integration-tests/ci-visibility/features/farewell.feature index 9d42ec47053..7ab87854150 100644 --- a/integration-tests/ci-visibility/features/farewell.feature +++ b/integration-tests/ci-visibility/features/farewell.feature @@ -2,3 +2,6 @@ Feature: Farewell Scenario: Say farewell When the greeter says farewell Then I should have heard "farewell" + Scenario: Say whatever + When the greeter says whatever + Then I should have heard "whatever" diff --git a/integration-tests/ci-visibility/features/support/steps.js b/integration-tests/ci-visibility/features/support/steps.js index dd21bc14b46..5882703295c 100644 --- a/integration-tests/ci-visibility/features/support/steps.js +++ b/integration-tests/ci-visibility/features/support/steps.js @@ -4,12 +4,15 @@ class Greeter { sayFarewell () { return 'farewell' } + sayGreetings () { return 'greetings' } + sayYo () { return 'yo' } + sayYeah () { return 'yeah whatever' } @@ -38,3 +41,7 @@ When('the greeter says yeah', function () { When('the greeter says greetings', function () { this.whatIHeard = new Greeter().sayGreetings() }) + +When('the greeter says whatever', function () { + this.whatIHeard = 'whatever' +}) diff --git a/integration-tests/ci-visibility/jest-custom-test-sequencer.js b/integration-tests/ci-visibility/jest-custom-test-sequencer.js new file mode 100644 index 00000000000..b78e0afc531 --- /dev/null +++ b/integration-tests/ci-visibility/jest-custom-test-sequencer.js @@ -0,0 +1,22 @@ +const Sequencer = require('@jest/test-sequencer').default + +// From example in https://jestjs.io/docs/configuration#testsequencer-string +class CustomSequencer extends Sequencer { + shard (tests, { shardIndex, shardCount }) { + // Log used to show that the custom sequencer is being used + // eslint-disable-next-line + console.log('Running shard with a custom sequencer', shardIndex) + const shardSize = Math.ceil(tests.length / shardCount) + const shardStart = shardSize * (shardIndex - 1) + const shardEnd = shardSize * shardIndex + + return [...tests].sort((a, b) => (a.path > b.path ? 1 : -1)).slice(shardStart, shardEnd) + } + + sort (tests) { + const copyTests = [...tests] + return copyTests.sort((testA, testB) => (testA.path > testB.path ? 1 : -1)) + } +} + +module.exports = CustomSequencer diff --git a/integration-tests/ci-visibility/jest-flaky/flaky-fails.js b/integration-tests/ci-visibility/jest-flaky/flaky-fails.js new file mode 100644 index 00000000000..b61b804d990 --- /dev/null +++ b/integration-tests/ci-visibility/jest-flaky/flaky-fails.js @@ -0,0 +1,6 @@ +describe('test-flaky-test-retries', () => { + it('can retry failed tests', () => { + // eslint-disable-next-line + expect(1).toEqual(2) + }) +}) diff --git a/integration-tests/ci-visibility/jest-flaky/flaky-passes.js b/integration-tests/ci-visibility/jest-flaky/flaky-passes.js new file mode 100644 index 00000000000..aa2c19ccf1d --- /dev/null +++ b/integration-tests/ci-visibility/jest-flaky/flaky-passes.js @@ -0,0 +1,13 @@ +let counter = 0 + +describe('test-flaky-test-retries', () => { + it('can retry flaky tests', () => { + // eslint-disable-next-line + expect(++counter).toEqual(3) + }) + + it('will not retry passed tests', () => { + // eslint-disable-next-line + expect(3).toEqual(3) + }) +}) diff --git a/integration-tests/ci-visibility/playwright-tests-automatic-retry/automatic-retry-test.js b/integration-tests/ci-visibility/playwright-tests-automatic-retry/automatic-retry-test.js new file mode 100644 index 00000000000..ac0cc8e33c1 --- /dev/null +++ b/integration-tests/ci-visibility/playwright-tests-automatic-retry/automatic-retry-test.js @@ -0,0 +1,15 @@ +const { test, expect } = require('@playwright/test') + +test.beforeEach(async ({ page }) => { + await page.goto(process.env.PW_BASE_URL) +}) + +test.describe('playwright', () => { + test('should eventually pass after retrying', async ({ page }) => { + const shouldFail = test.info().retry < 2 + + await expect(page.locator('.hello-world')).toHaveText([ + shouldFail ? 'Hello Warld' : 'Hello World' + ]) + }) +}) diff --git a/integration-tests/ci-visibility/playwright-tests-error/before-all-timeout-test.js b/integration-tests/ci-visibility/playwright-tests-error/before-all-timeout-test.js new file mode 100644 index 00000000000..9736f2b801d --- /dev/null +++ b/integration-tests/ci-visibility/playwright-tests-error/before-all-timeout-test.js @@ -0,0 +1,19 @@ +const { test, expect } = require('@playwright/test') + +test.beforeEach(async ({ page }) => { + await page.goto(process.env.PW_BASE_URL) +}) + +const waitFor = (ms) => new Promise(resolve => setTimeout(resolve, ms)) + +test.describe('playwright', () => { + test.beforeAll(async () => { + // timeout error + await waitFor(3100) + }) + test('should work with passing tests', async ({ page }) => { + await expect(page.locator('.hello-world')).toHaveText([ + 'Hello World' + ]) + }) +}) diff --git a/integration-tests/ci-visibility/playwright-tests-max-failures/failing-test-and-another-test.js b/integration-tests/ci-visibility/playwright-tests-max-failures/failing-test-and-another-test.js new file mode 100644 index 00000000000..317b97f4175 --- /dev/null +++ b/integration-tests/ci-visibility/playwright-tests-max-failures/failing-test-and-another-test.js @@ -0,0 +1,17 @@ +const { test, expect } = require('@playwright/test') + +test.beforeEach(async ({ page }) => { + await page.goto(process.env.PW_BASE_URL) +}) + +test('should work with failing tests', async ({ page }) => { + await expect(page.locator('.hello-world')).toHaveText([ + 'Hello Warld' + ]) +}) + +test('does not crash afterwards', async ({ page }) => { + await expect(page.locator('.hello-world')).toHaveText([ + 'Hello World' + ]) +}) diff --git a/integration-tests/ci-visibility/run-jest.js b/integration-tests/ci-visibility/run-jest.js index 50f569aa902..a1f236be7a2 100644 --- a/integration-tests/ci-visibility/run-jest.js +++ b/integration-tests/ci-visibility/run-jest.js @@ -7,7 +7,8 @@ const options = { testRegex: process.env.TESTS_TO_RUN ? new RegExp(process.env.TESTS_TO_RUN) : /test\/ci-visibility-test/, coverage: !process.env.DISABLE_CODE_COVERAGE, runInBand: true, - shard: process.env.TEST_SHARD || undefined + shard: process.env.TEST_SHARD || undefined, + testSequencer: process.env.CUSTOM_TEST_SEQUENCER } if (process.env.RUN_IN_PARALLEL) { @@ -19,6 +20,14 @@ if (process.env.OLD_RUNNER) { options.testRunner = 'jest-jasmine2' } +if (process.env.ENABLE_JSDOM) { + options.testEnvironment = 'jsdom' +} + +if (process.env.COLLECT_COVERAGE_FROM) { + options.collectCoverageFrom = process.env.COLLECT_COVERAGE_FROM.split(',') +} + jest.runCLI( options, options.projects diff --git a/integration-tests/ci-visibility/run-jest.mjs b/integration-tests/ci-visibility/run-jest.mjs index a35ddda382c..a9ecb24d0c6 100644 --- a/integration-tests/ci-visibility/run-jest.mjs +++ b/integration-tests/ci-visibility/run-jest.mjs @@ -22,6 +22,10 @@ if (process.env.OLD_RUNNER) { options.testRunner = 'jest-jasmine2' } +if (process.env.ENABLE_JSDOM) { + options.testEnvironment = 'jsdom' +} + jest.runCLI( options, options.projects diff --git a/integration-tests/ci-visibility/run-workerpool.js b/integration-tests/ci-visibility/run-workerpool.js new file mode 100644 index 00000000000..8a77c9e315b --- /dev/null +++ b/integration-tests/ci-visibility/run-workerpool.js @@ -0,0 +1,23 @@ +// eslint-disable-next-line +const workerpool = require('workerpool') +const pool = workerpool.pool({ workerType: 'process' }) + +function add (a, b) { + return a + b +} + +pool + .exec(add, [3, 4]) + .then((result) => { + // eslint-disable-next-line no-console + console.log('result', result) // outputs 7 + return pool.terminate() + }) + .catch(function (err) { + // eslint-disable-next-line no-console + console.error(err) + process.exit(1) + }) + .then(() => { + process.exit(0) + }) diff --git a/integration-tests/ci-visibility/subproject/cypress-config.json b/integration-tests/ci-visibility/subproject/cypress-config.json new file mode 100644 index 00000000000..3ad19f9f90a --- /dev/null +++ b/integration-tests/ci-visibility/subproject/cypress-config.json @@ -0,0 +1,9 @@ +{ + "video": false, + "screenshotOnRunFailure": false, + "pluginsFile": "cypress/plugins-old/index.js", + "supportFile": "cypress/support/e2e.js", + "integrationFolder": "cypress/e2e", + "defaultCommandTimeout": 100, + "nodeVersion": "system" +} diff --git a/integration-tests/ci-visibility/subproject/cypress.config.js b/integration-tests/ci-visibility/subproject/cypress.config.js new file mode 100644 index 00000000000..9a786e4ef75 --- /dev/null +++ b/integration-tests/ci-visibility/subproject/cypress.config.js @@ -0,0 +1,11 @@ +module.exports = { + defaultCommandTimeout: 100, + e2e: { + setupNodeEvents (on, config) { + return require('dd-trace/ci/cypress/plugin')(on, config) + }, + specPattern: process.env.SPEC_PATTERN || 'cypress/e2e/**/*.cy.js' + }, + video: false, + screenshotOnRunFailure: false +} diff --git a/integration-tests/ci-visibility/subproject/cypress/e2e/spec.cy.js b/integration-tests/ci-visibility/subproject/cypress/e2e/spec.cy.js new file mode 100644 index 00000000000..c03d23b677c --- /dev/null +++ b/integration-tests/ci-visibility/subproject/cypress/e2e/spec.cy.js @@ -0,0 +1,8 @@ +/* eslint-disable */ +describe('context', () => { + it('passes', () => { + cy.visit('/') + .get('.hello-world') + .should('have.text', 'Hello World') + }) +}) diff --git a/integration-tests/ci-visibility/subproject/cypress/plugins-old/index.js b/integration-tests/ci-visibility/subproject/cypress/plugins-old/index.js new file mode 100644 index 00000000000..f80695694a9 --- /dev/null +++ b/integration-tests/ci-visibility/subproject/cypress/plugins-old/index.js @@ -0,0 +1 @@ +module.exports = require('dd-trace/ci/cypress/plugin') diff --git a/integration-tests/ci-visibility/subproject/cypress/support/e2e.js b/integration-tests/ci-visibility/subproject/cypress/support/e2e.js new file mode 100644 index 00000000000..c169d8b1bb2 --- /dev/null +++ b/integration-tests/ci-visibility/subproject/cypress/support/e2e.js @@ -0,0 +1,2 @@ +// eslint-disable-next-line +require('dd-trace/ci/cypress/support') diff --git a/integration-tests/ci-visibility/subproject/features/greetings.feature b/integration-tests/ci-visibility/subproject/features/greetings.feature new file mode 100644 index 00000000000..7f8097b87d5 --- /dev/null +++ b/integration-tests/ci-visibility/subproject/features/greetings.feature @@ -0,0 +1,4 @@ +Feature: Greetings + Scenario: Say greetings + When the greeter says greetings + Then I should have heard "greetings" diff --git a/integration-tests/ci-visibility/subproject/features/support/steps.js b/integration-tests/ci-visibility/subproject/features/support/steps.js new file mode 100644 index 00000000000..6a946067ca9 --- /dev/null +++ b/integration-tests/ci-visibility/subproject/features/support/steps.js @@ -0,0 +1,15 @@ +const assert = require('assert') +const { When, Then } = require('@cucumber/cucumber') +class Greeter { + sayGreetings () { + return 'greetings' + } +} + +When('the greeter says greetings', function () { + this.whatIHeard = new Greeter().sayGreetings() +}) + +Then('I should have heard {string}', function (expectedResponse) { + assert.equal(this.whatIHeard, expectedResponse) +}) diff --git a/integration-tests/ci-visibility/subproject/package.json b/integration-tests/ci-visibility/subproject/package.json new file mode 100644 index 00000000000..dc1d9050f8e --- /dev/null +++ b/integration-tests/ci-visibility/subproject/package.json @@ -0,0 +1,6 @@ +{ + "name": "subproject", + "private": true, + "version": "1.0.0", + "description": "app within repo" +} diff --git a/integration-tests/ci-visibility/subproject/playwright-tests/landing-page-test.js b/integration-tests/ci-visibility/subproject/playwright-tests/landing-page-test.js new file mode 100644 index 00000000000..34e6eb2c3aa --- /dev/null +++ b/integration-tests/ci-visibility/subproject/playwright-tests/landing-page-test.js @@ -0,0 +1,13 @@ +const { test, expect } = require('@playwright/test') + +test.beforeEach(async ({ page }) => { + await page.goto(process.env.PW_BASE_URL) +}) + +test.describe('playwright', () => { + test('should work with passing tests', async ({ page }) => { + await expect(page.locator('.hello-world')).toHaveText([ + 'Hello World' + ]) + }) +}) diff --git a/integration-tests/ci-visibility/subproject/playwright.config.js b/integration-tests/ci-visibility/subproject/playwright.config.js new file mode 100644 index 00000000000..3be77049e3b --- /dev/null +++ b/integration-tests/ci-visibility/subproject/playwright.config.js @@ -0,0 +1,18 @@ +// Playwright config file for integration tests +const { devices } = require('@playwright/test') + +const config = { + baseURL: process.env.PW_BASE_URL, + testDir: './playwright-tests', + timeout: 30000, + reporter: 'line', + projects: [ + { + name: 'chromium', + use: devices['Desktop Chrome'] + } + ], + testMatch: '**/*-test.js' +} + +module.exports = config diff --git a/integration-tests/ci-visibility/subproject/subproject-test.js b/integration-tests/ci-visibility/subproject/subproject-test.js new file mode 100644 index 00000000000..5300f1926d6 --- /dev/null +++ b/integration-tests/ci-visibility/subproject/subproject-test.js @@ -0,0 +1,9 @@ +// eslint-disable-next-line +const { expect } = require('chai') + +describe('subproject-test', () => { + it('can run', () => { + // eslint-disable-next-line + expect(1).to.equal(1) + }) +}) diff --git a/integration-tests/ci-visibility/subproject/vitest-test.mjs b/integration-tests/ci-visibility/subproject/vitest-test.mjs new file mode 100644 index 00000000000..d495a14a98c --- /dev/null +++ b/integration-tests/ci-visibility/subproject/vitest-test.mjs @@ -0,0 +1,7 @@ +import { describe, test, expect } from 'vitest' + +describe('context', () => { + test('can report passed test', () => { + expect(1 + 2).to.equal(3) + }) +}) diff --git a/integration-tests/ci-visibility/test-api-manual/test.fake.js b/integration-tests/ci-visibility/test-api-manual/test.fake.js index 11f35dd8e87..a3256bc6f42 100644 --- a/integration-tests/ci-visibility/test-api-manual/test.fake.js +++ b/integration-tests/ci-visibility/test-api-manual/test.fake.js @@ -31,7 +31,7 @@ describe('can run tests', () => { }) test('integration test', () => { // Just for testing purposes, so we don't create a custom span - if (!process.env.DD_CIVISIBILITY_MANUAL_API_ENABLED) { + if (process.env.DD_CIVISIBILITY_MANUAL_API_ENABLED === 'false') { return Promise.resolve() } const testSpan = tracer.scope().active() diff --git a/integration-tests/ci-visibility/test-early-flake-detection/__snapshots__/jest-snapshot.js.snap b/integration-tests/ci-visibility/test-early-flake-detection/__snapshots__/jest-snapshot.js.snap new file mode 100644 index 00000000000..54a48eb466f --- /dev/null +++ b/integration-tests/ci-visibility/test-early-flake-detection/__snapshots__/jest-snapshot.js.snap @@ -0,0 +1,3 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`test can do snapshot 1`] = `3`; diff --git a/integration-tests/ci-visibility/test-early-flake-detection/jest-snapshot.js b/integration-tests/ci-visibility/test-early-flake-detection/jest-snapshot.js new file mode 100644 index 00000000000..a1948824aad --- /dev/null +++ b/integration-tests/ci-visibility/test-early-flake-detection/jest-snapshot.js @@ -0,0 +1,6 @@ +describe('test', () => { + it('can do snapshot', () => { + // eslint-disable-next-line + expect(1 + 2).toMatchSnapshot() + }) +}) diff --git a/integration-tests/ci-visibility/test-early-flake-detection/mocha-parameterized.js b/integration-tests/ci-visibility/test-early-flake-detection/mocha-parameterized.js new file mode 100644 index 00000000000..b286dfeb359 --- /dev/null +++ b/integration-tests/ci-visibility/test-early-flake-detection/mocha-parameterized.js @@ -0,0 +1,8 @@ +const { expect } = require('chai') +const forEach = require('mocha-each') + +describe('parameterized', () => { + forEach(['parameter 1', 'parameter 2']).it('test %s', (value) => { + expect(value.startsWith('parameter')).to.be.true + }) +}) diff --git a/integration-tests/ci-visibility/test-early-flake-detection/occasionally-failing-test.js b/integration-tests/ci-visibility/test-early-flake-detection/occasionally-failing-test.js new file mode 100644 index 00000000000..22b6d91935b --- /dev/null +++ b/integration-tests/ci-visibility/test-early-flake-detection/occasionally-failing-test.js @@ -0,0 +1,9 @@ +const { expect } = require('chai') + +let globalCounter = 0 + +describe('fail', () => { + it('occasionally fails', () => { + expect((globalCounter++) % 2).to.equal(0) + }) +}) diff --git a/integration-tests/ci-visibility/test-early-flake-detection/skipped-and-todo-test.js b/integration-tests/ci-visibility/test-early-flake-detection/skipped-and-todo-test.js new file mode 100644 index 00000000000..b778a31711e --- /dev/null +++ b/integration-tests/ci-visibility/test-early-flake-detection/skipped-and-todo-test.js @@ -0,0 +1,14 @@ +const { expect } = require('chai') + +describe('ci visibility', () => { + it('can report tests', () => { + expect(1 + 2).to.equal(3) + }) + // only run for jest tests + if (typeof jest !== 'undefined') { + it.todo('todo will not be retried') + } + it.skip('skip will not be retried', () => { + expect(1 + 2).to.equal(4) + }) +}) diff --git a/integration-tests/ci-visibility/test-early-flake-detection/test-parameterized.js b/integration-tests/ci-visibility/test-early-flake-detection/test-parameterized.js new file mode 100644 index 00000000000..8ff884c6c28 --- /dev/null +++ b/integration-tests/ci-visibility/test-early-flake-detection/test-parameterized.js @@ -0,0 +1,7 @@ +const { expect } = require('chai') + +describe('parameterized', () => { + test.each(['parameter 1', 'parameter 2'])('test %s', (value) => { + expect(value.startsWith('parameter')).toEqual(true) + }) +}) diff --git a/integration-tests/ci-visibility/test-early-flake-detection/test.js b/integration-tests/ci-visibility/test-early-flake-detection/test.js new file mode 100644 index 00000000000..e3306f69374 --- /dev/null +++ b/integration-tests/ci-visibility/test-early-flake-detection/test.js @@ -0,0 +1,7 @@ +const { expect } = require('chai') + +describe('ci visibility', () => { + it('can report tests', () => { + expect(1 + 2).to.equal(3) + }) +}) diff --git a/integration-tests/ci-visibility/test-early-flake-detection/weird-test-names.js b/integration-tests/ci-visibility/test-early-flake-detection/weird-test-names.js new file mode 100644 index 00000000000..60b30a65fb0 --- /dev/null +++ b/integration-tests/ci-visibility/test-early-flake-detection/weird-test-names.js @@ -0,0 +1,11 @@ +const { expect } = require('chai') + +it('no describe can do stuff', () => { + expect(1).to.equal(1) +}) + +describe('describe ', () => { + it('trailing space ', () => { + expect(1).to.equal(1) + }) +}) diff --git a/integration-tests/ci-visibility/test-flaky-test-retries/eventually-passing-test.js b/integration-tests/ci-visibility/test-flaky-test-retries/eventually-passing-test.js new file mode 100644 index 00000000000..de08821128d --- /dev/null +++ b/integration-tests/ci-visibility/test-flaky-test-retries/eventually-passing-test.js @@ -0,0 +1,9 @@ +const { expect } = require('chai') + +let counter = 0 + +describe('test-flaky-test-retries', () => { + it('can retry failed tests', () => { + expect(++counter).to.equal(3) + }) +}) diff --git a/integration-tests/ci-visibility/test-total-code-coverage/test-run.js b/integration-tests/ci-visibility/test-total-code-coverage/test-run.js new file mode 100644 index 00000000000..2256f75f069 --- /dev/null +++ b/integration-tests/ci-visibility/test-total-code-coverage/test-run.js @@ -0,0 +1,8 @@ +const { expect } = require('chai') +const sum = require('./used-dependency') + +describe('test-run', () => { + it('can report tests', () => { + expect(sum(1, 2)).to.equal(3) + }) +}) diff --git a/integration-tests/ci-visibility/test-total-code-coverage/test-skipped.js b/integration-tests/ci-visibility/test-total-code-coverage/test-skipped.js new file mode 100644 index 00000000000..1410740bfa3 --- /dev/null +++ b/integration-tests/ci-visibility/test-total-code-coverage/test-skipped.js @@ -0,0 +1,8 @@ +const { expect } = require('chai') +const sum = require('./unused-dependency') + +describe('test-skipped', () => { + it('can report tests', () => { + expect(sum(1, 2)).to.equal(3) + }) +}) diff --git a/integration-tests/ci-visibility/test-total-code-coverage/unused-dependency.js b/integration-tests/ci-visibility/test-total-code-coverage/unused-dependency.js new file mode 100644 index 00000000000..2012896b44c --- /dev/null +++ b/integration-tests/ci-visibility/test-total-code-coverage/unused-dependency.js @@ -0,0 +1,3 @@ +module.exports = function (a, b) { + return a + b +} diff --git a/integration-tests/ci-visibility/test-total-code-coverage/used-dependency.js b/integration-tests/ci-visibility/test-total-code-coverage/used-dependency.js new file mode 100644 index 00000000000..2012896b44c --- /dev/null +++ b/integration-tests/ci-visibility/test-total-code-coverage/used-dependency.js @@ -0,0 +1,3 @@ +module.exports = function (a, b) { + return a + b +} diff --git a/integration-tests/ci-visibility/test/selenium-no-framework.js b/integration-tests/ci-visibility/test/selenium-no-framework.js new file mode 100644 index 00000000000..cca24586bfd --- /dev/null +++ b/integration-tests/ci-visibility/test/selenium-no-framework.js @@ -0,0 +1,30 @@ +const { By, Builder } = require('selenium-webdriver') +const chrome = require('selenium-webdriver/chrome') + +async function run () { + const options = new chrome.Options() + options.addArguments('--headless') + const build = new Builder().forBrowser('chrome').setChromeOptions(options) + const driver = await build.build() + + await driver.get(process.env.WEB_APP_URL) + + await driver.getTitle() + + await driver.manage().setTimeouts({ implicit: 500 }) + + const helloWorld = await driver.findElement(By.className('hello-world')) + + await helloWorld.getText() + + return driver.quit() +} + +run() + .then(() => { + process.exit(0) + }).catch((err) => { + // eslint-disable-next-line no-console + console.error(err) + process.exit(1) + }) diff --git a/integration-tests/ci-visibility/test/selenium-test.js b/integration-tests/ci-visibility/test/selenium-test.js new file mode 100644 index 00000000000..71260f8da95 --- /dev/null +++ b/integration-tests/ci-visibility/test/selenium-test.js @@ -0,0 +1,32 @@ +const { By, Builder } = require('selenium-webdriver') +const chrome = require('selenium-webdriver/chrome') +const { expect } = require('chai') + +const options = new chrome.Options() +options.addArguments('--headless') + +describe('selenium', function () { + let driver + + beforeEach(async function () { + const build = new Builder().forBrowser('chrome').setChromeOptions(options) + driver = await build.build() + }) + + it('can run selenium tests', async function () { + await driver.get(process.env.WEB_APP_URL) + + const title = await driver.getTitle() + expect(title).to.equal('Hello World') + + await driver.manage().setTimeouts({ implicit: 500 }) + + const helloWorld = await driver.findElement(By.className('hello-world')) + + const value = await helloWorld.getText() + + expect(value).to.equal('Hello World') + }) + + afterEach(async () => await driver.quit()) +}) diff --git a/integration-tests/ci-visibility/unskippable-test/test-to-run.js b/integration-tests/ci-visibility/unskippable-test/test-to-run.js new file mode 100644 index 00000000000..f093d1e39ed --- /dev/null +++ b/integration-tests/ci-visibility/unskippable-test/test-to-run.js @@ -0,0 +1,7 @@ +const { expect } = require('chai') + +describe('test-to-run', () => { + it('can report tests', () => { + expect(1 + 2).to.equal(3) + }) +}) diff --git a/integration-tests/ci-visibility/vitest-tests/coverage-test.mjs b/integration-tests/ci-visibility/vitest-tests/coverage-test.mjs new file mode 100644 index 00000000000..63ee3600ef9 --- /dev/null +++ b/integration-tests/ci-visibility/vitest-tests/coverage-test.mjs @@ -0,0 +1,8 @@ +import { describe, test, expect } from 'vitest' +import { sum } from './sum' + +describe('code coverage', () => { + test('passes', () => { + expect(sum(1, 2)).to.equal(3) + }) +}) diff --git a/integration-tests/ci-visibility/vitest-tests/early-flake-detection.mjs b/integration-tests/ci-visibility/vitest-tests/early-flake-detection.mjs new file mode 100644 index 00000000000..a85036dac8e --- /dev/null +++ b/integration-tests/ci-visibility/vitest-tests/early-flake-detection.mjs @@ -0,0 +1,33 @@ +import { describe, test, expect } from 'vitest' +import { sum } from './sum' + +let numAttempt = 0 +let numOtherAttempt = 0 + +describe('early flake detection', () => { + test('can retry tests that eventually pass', { repeats: process.env.SHOULD_REPEAT && 2 }, () => { + expect(sum(1, 2)).to.equal(numAttempt++ > 1 ? 3 : 4) + }) + + test('can retry tests that always pass', { repeats: process.env.SHOULD_REPEAT && 2 }, () => { + if (process.env.ALWAYS_FAIL) { + expect(sum(1, 2)).to.equal(4) + } else { + expect(sum(1, 2)).to.equal(3) + } + }) + + test('does not retry if it is not new', () => { + expect(sum(1, 2)).to.equal(3) + }) + + test.skip('does not retry if the test is skipped', () => { + expect(sum(1, 2)).to.equal(3) + }) + + if (process.env.SHOULD_ADD_EVENTUALLY_FAIL) { + test('can retry tests that eventually fail', () => { + expect(sum(1, 2)).to.equal(numOtherAttempt++ < 3 ? 3 : 4) + }) + } +}) diff --git a/integration-tests/ci-visibility/vitest-tests/flaky-test-retries.mjs b/integration-tests/ci-visibility/vitest-tests/flaky-test-retries.mjs new file mode 100644 index 00000000000..fa2693311bf --- /dev/null +++ b/integration-tests/ci-visibility/vitest-tests/flaky-test-retries.mjs @@ -0,0 +1,18 @@ +import { describe, test, expect } from 'vitest' +import { sum } from './sum' + +let numAttempt = 0 + +describe('flaky test retries', () => { + test('can retry tests that eventually pass', () => { + expect(sum(1, 2)).to.equal(numAttempt++) + }) + + test('can retry tests that never pass', () => { + expect(sum(1, 2)).to.equal(0) + }) + + test('does not retry if unnecessary', () => { + expect(sum(1, 2)).to.equal(3) + }) +}) diff --git a/integration-tests/ci-visibility/vitest-tests/sum.mjs b/integration-tests/ci-visibility/vitest-tests/sum.mjs new file mode 100644 index 00000000000..f1c6520acbd --- /dev/null +++ b/integration-tests/ci-visibility/vitest-tests/sum.mjs @@ -0,0 +1,3 @@ +export function sum (a, b) { + return a + b +} diff --git a/integration-tests/ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs b/integration-tests/ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs new file mode 100644 index 00000000000..a97f95e0df1 --- /dev/null +++ b/integration-tests/ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs @@ -0,0 +1,26 @@ +import { describe, test, expect, beforeEach, afterEach } from 'vitest' +import { sum } from './sum' + +describe('context', () => { + beforeEach(() => { + throw new Error('failed before each') + }) + test('can report failed test', () => { + expect(sum(1, 2)).to.equal(4) + }) + test('can report more', () => { + expect(sum(1, 2)).to.equal(3) + }) +}) + +describe('other context', () => { + afterEach(() => { + throw new Error('failed after each') + }) + test('can report passed test', () => { + expect(sum(1, 2)).to.equal(3) + }) + test('can report more', () => { + expect(sum(1, 2)).to.equal(3) + }) +}) diff --git a/integration-tests/ci-visibility/vitest-tests/test-visibility-failed-suite.mjs b/integration-tests/ci-visibility/vitest-tests/test-visibility-failed-suite.mjs new file mode 100644 index 00000000000..f2df345a87f --- /dev/null +++ b/integration-tests/ci-visibility/vitest-tests/test-visibility-failed-suite.mjs @@ -0,0 +1,29 @@ +import { describe, test, expect, beforeEach, afterEach } from 'vitest' +import { sum } from './sum' + +let preparedValue = 1 + +describe('test-visibility-failed-suite-first-describe', () => { + beforeEach(() => { + preparedValue = 2 + }) + test('can report failed test', () => { + expect(sum(1, 2)).to.equal(4) + }) + test('can report more', () => { + expect(sum(1, 2)).to.equal(3) + expect(preparedValue).to.equal(2) + }) +}) + +describe('test-visibility-failed-suite-second-describe', () => { + afterEach(() => { + preparedValue = 1 + }) + test('can report passed test', () => { + expect(sum(1, 2)).to.equal(3) + }) + test('can report more', () => { + expect(sum(1, 2)).to.equal(3) + }) +}) diff --git a/integration-tests/ci-visibility/vitest-tests/test-visibility-passed-suite.mjs b/integration-tests/ci-visibility/vitest-tests/test-visibility-passed-suite.mjs new file mode 100644 index 00000000000..9b2ade6d83d --- /dev/null +++ b/integration-tests/ci-visibility/vitest-tests/test-visibility-passed-suite.mjs @@ -0,0 +1,47 @@ +import { describe, test, expect } from 'vitest' +import { sum } from './sum' + +describe('context', () => { + test('can report passed test', () => { + expect(sum(1, 2)).to.equal(3) + }) + test('can report more', () => { + expect(sum(1, 2)).to.equal(3) + }) +}) + +describe('other context', () => { + test('can report passed test', () => { + expect(sum(1, 2)).to.equal(3) + }) + test('can report more', () => { + expect(sum(1, 2)).to.equal(3) + }) + test.skip('can skip', () => { + expect(sum(1, 2)).to.equal(3) + }) + test.todo('can todo', () => { + expect(sum(1, 2)).to.equal(3) + }) + // eslint-disable-next-line + test('can programmatic skip', (context) => { + // eslint-disable-next-line + context.skip() + expect(sum(1, 2)).to.equal(3) + }) +}) + +test('no suite', () => { + expect(sum(1, 2)).to.equal(3) +}) + +test.skip('skip no suite', () => { + expect(sum(1, 2)).to.equal(3) +}) + +// eslint-disable-next-line +test('programmatic skip no suite', (context) => { + // eslint-disable-next-line + context.skip() + expect(sum(1, 2)).to.equal(3) +}) diff --git a/integration-tests/ci-visibility/web-app-server.js b/integration-tests/ci-visibility/web-app-server.js index cb683e06bb8..28b4d957b7b 100644 --- a/integration-tests/ci-visibility/web-app-server.js +++ b/integration-tests/ci-visibility/web-app-server.js @@ -8,6 +8,14 @@ module.exports = http.createServer((req, res) => { res.end(` + Hello World +
Hello World
diff --git a/integration-tests/config-jest-multiproject.js b/integration-tests/config-jest-multiproject.js new file mode 100644 index 00000000000..e06aec35930 --- /dev/null +++ b/integration-tests/config-jest-multiproject.js @@ -0,0 +1,20 @@ +module.exports = { + projects: [ + { + displayName: 'standard', + testPathIgnorePatterns: ['/node_modules/'], + cache: false, + testMatch: [ + '**/ci-visibility/test/ci-visibility-test*' + ] + }, + { + displayName: 'node', + testPathIgnorePatterns: ['/node_modules/'], + cache: false, + testMatch: [ + '**/ci-visibility/test/ci-visibility-test*' + ] + } + ] +} diff --git a/integration-tests/config-jest.js b/integration-tests/config-jest.js new file mode 100644 index 00000000000..f30aec0ad35 --- /dev/null +++ b/integration-tests/config-jest.js @@ -0,0 +1,8 @@ +module.exports = { + projects: process.env.PROJECTS ? JSON.parse(process.env.PROJECTS) : [__dirname], + testPathIgnorePatterns: ['/node_modules/'], + cache: false, + testMatch: [ + process.env.TESTS_TO_RUN || '**/ci-visibility/test/ci-visibility-test*' + ] +} diff --git a/integration-tests/cucumber/cucumber.spec.js b/integration-tests/cucumber/cucumber.spec.js index dfdbf13ebb6..35c4b3b2060 100644 --- a/integration-tests/cucumber/cucumber.spec.js +++ b/integration-tests/cucumber/cucumber.spec.js @@ -25,213 +25,246 @@ const { TEST_ITR_SKIPPING_COUNT, TEST_CODE_COVERAGE_LINES_PCT, TEST_ITR_FORCED_RUN, - TEST_ITR_UNSKIPPABLE + TEST_ITR_UNSKIPPABLE, + TEST_SOURCE_FILE, + TEST_SOURCE_START, + TEST_EARLY_FLAKE_ENABLED, + TEST_EARLY_FLAKE_ABORT_REASON, + TEST_IS_NEW, + TEST_IS_RETRY, + TEST_NAME, + CUCUMBER_IS_PARALLEL, + TEST_SUITE, + TEST_CODE_OWNERS, + TEST_SESSION_NAME, + TEST_LEVEL_EVENT_TYPES } = require('../../packages/dd-trace/src/plugins/util/test') +const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') -const hookFile = 'dd-trace/loader-hook.mjs' const isOldNode = semver.satisfies(process.version, '<=16') const versions = ['7.0.0', isOldNode ? '9' : 'latest'] -const moduleType = [ - { - type: 'commonJS', - runTestsCommand: './node_modules/.bin/cucumber-js ci-visibility/features/*.feature', - runTestsWithCoverageCommand: - './node_modules/nyc/bin/nyc.js -r=text-summary ' + - 'node ./node_modules/.bin/cucumber-js ci-visibility/features/*.feature', - parallelModeCommand: `./node_modules/.bin/cucumber-js ` + - `ci-visibility/features/farewell.feature --parallel 2 --publish-quiet`, - featuresPath: 'ci-visibility/features/', - fileExtension: 'js' - }, - { - type: 'esm', - runTestsCommand: `node --loader=${hookFile} ./node_modules/.bin/cucumber-js ci-visibility/features-esm/*.feature`, - runTestsWithCoverageCommand: - `./node_modules/nyc/bin/nyc.js -r=text-summary ` + - `node --loader=./node_modules/@istanbuljs/esm-loader-hook/index.js ` + - `--loader=${hookFile} ./node_modules/.bin/cucumber-js ci-visibility/features-esm/*.feature`, - parallelModeCommand: - `node --loader=${hookFile} ./node_modules/.bin/cucumber-js ` + - `ci-visibility/features-esm/farewell.feature --parallel 2 --publish-quiet`, - featuresPath: 'ci-visibility/features-esm/', - fileExtension: 'mjs' - } -] +const runTestsCommand = './node_modules/.bin/cucumber-js ci-visibility/features/*.feature' +const runTestsWithCoverageCommand = './node_modules/nyc/bin/nyc.js -r=text-summary ' + + 'node ./node_modules/.bin/cucumber-js ci-visibility/features/*.feature' +const parallelModeCommand = './node_modules/.bin/cucumber-js ci-visibility/features/*.feature --parallel 2' +const featuresPath = 'ci-visibility/features/' +const fileExtension = 'js' versions.forEach(version => { - moduleType.forEach(({ - type, - runTestsCommand, - runTestsWithCoverageCommand, - parallelModeCommand, - featuresPath, - fileExtension - }) => { - // temporary fix for failing esm tests on the CI, skip for now for the release and comeback to solve the issue - if (type === 'esm') { - return - } - - // esm support by cucumber was only added on >= 8.0.0 - // if (type === 'esm' && semver.satisfies(version, '<8.0.0')) { - // return - // } - - describe(`cucumber@${version} ${type}`, () => { - let sandbox, cwd, receiver, childProcess - before(async function () { - // add an explicit timeout to make tests less flaky - this.timeout(50000) - - sandbox = await createSandbox([`@cucumber/cucumber@${version}`, 'assert', - 'nyc', '@istanbuljs/esm-loader-hook'], true) - cwd = sandbox.folder - }) + // TODO: add esm tests + describe(`cucumber@${version} commonJS`, () => { + let sandbox, cwd, receiver, childProcess, testOutput - after(async function () { - // add an explicit timeout to make tests less flaky - this.timeout(50000) + before(async function () { + // add an explicit timeout to make tests less flaky + this.timeout(50000) - await sandbox.remove() - }) + sandbox = await createSandbox([`@cucumber/cucumber@${version}`, 'assert', 'nyc'], true) + cwd = sandbox.folder + }) - beforeEach(async function () { - const port = await getPort() - receiver = await new FakeCiVisIntake(port).start() - }) + after(async function () { + // add an explicit timeout to make tests less flaky + this.timeout(50000) - afterEach(async () => { - childProcess.kill() - await receiver.stop() - }) - const reportMethods = ['agentless', 'evp proxy'] + await sandbox.remove() + }) - it('does not crash with parallel mode', (done) => { - let testOutput - childProcess = exec( - parallelModeCommand, - { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - DD_TRACE_DEBUG: 1, - DD_TRACE_LOG_LEVEL: 'warn' - }, - stdio: 'inherit' - } - ) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.on('exit', (code) => { - assert.notInclude(testOutput, 'TypeError') - assert.include(testOutput, 'Unable to initialize CI Visibility because Cucumber is running in parallel mode.') - assert.equal(code, 0) - done() + beforeEach(async function () { + const port = await getPort() + receiver = await new FakeCiVisIntake(port).start() + }) + + afterEach(async () => { + testOutput = '' + childProcess.kill() + await receiver.stop() + }) + + const reportMethods = ['agentless', 'evp proxy'] + + reportMethods.forEach((reportMethod) => { + context(`reporting via ${reportMethod}`, () => { + let envVars, isAgentless + beforeEach(() => { + isAgentless = reportMethod === 'agentless' + envVars = isAgentless ? getCiVisAgentlessConfig(receiver.port) : getCiVisEvpProxyConfig(receiver.port) }) - }).timeout(50000) - - reportMethods.forEach((reportMethod) => { - context(`reporting via ${reportMethod}`, () => { - let envVars, isAgentless - beforeEach(() => { - isAgentless = reportMethod === 'agentless' - envVars = isAgentless ? getCiVisAgentlessConfig(receiver.port) : getCiVisEvpProxyConfig(receiver.port) - }) - it('can run and report tests', (done) => { - receiver.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { - const events = payloads.flatMap(({ payload }) => payload.events) - - const testSessionEvent = events.find(event => event.type === 'test_session_end') - const testModuleEvent = events.find(event => event.type === 'test_module_end') - const testSuiteEvents = events.filter(event => event.type === 'test_suite_end') - const testEvents = events.filter(event => event.type === 'test') - - const stepEvents = events.filter(event => event.type === 'span') - - const { content: testSessionEventContent } = testSessionEvent - const { content: testModuleEventContent } = testModuleEvent - - assert.exists(testSessionEventContent.test_session_id) - assert.exists(testSessionEventContent.meta[TEST_COMMAND]) - assert.exists(testSessionEventContent.meta[TEST_TOOLCHAIN]) - assert.equal(testSessionEventContent.resource.startsWith('test_session.'), true) - assert.equal(testSessionEventContent.meta[TEST_STATUS], 'fail') - - assert.exists(testModuleEventContent.test_session_id) - assert.exists(testModuleEventContent.test_module_id) - assert.exists(testModuleEventContent.meta[TEST_COMMAND]) - assert.exists(testModuleEventContent.meta[TEST_MODULE]) - assert.equal(testModuleEventContent.resource.startsWith('test_module.'), true) - assert.equal(testModuleEventContent.meta[TEST_STATUS], 'fail') - assert.equal( - testModuleEventContent.test_session_id.toString(10), - testSessionEventContent.test_session_id.toString(10) - ) + const runModes = ['serial'] + + if (version !== '7.0.0') { // only on latest or 9 if node is old + runModes.push('parallel') + } + + runModes.forEach((runMode) => { + it(`(${runMode}) can run and report tests`, (done) => { + const runCommand = runMode === 'parallel' ? parallelModeCommand : runTestsCommand + + const receiverPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const metadataDicts = payloads.flatMap(({ payload }) => payload.metadata) + metadataDicts.forEach(metadata => { + for (const testLevel of TEST_LEVEL_EVENT_TYPES) { + assert.equal(metadata[testLevel][TEST_SESSION_NAME], 'my-test-session') + } + }) - assert.includeMembers(testSuiteEvents.map(suite => suite.content.resource), [ - `test_suite.${featuresPath}farewell.feature`, - `test_suite.${featuresPath}greetings.feature` - ]) - assert.includeMembers(testSuiteEvents.map(suite => suite.content.meta[TEST_STATUS]), [ - 'pass', - 'fail' - ]) + const events = payloads.flatMap(({ payload }) => payload.events) - testSuiteEvents.forEach(({ - content: { - meta, - test_suite_id: testSuiteId, - test_module_id: testModuleId, - test_session_id: testSessionId - } - }) => { - assert.exists(meta[TEST_COMMAND]) - assert.exists(meta[TEST_MODULE]) - assert.exists(testSuiteId) - assert.equal(testModuleId.toString(10), testModuleEventContent.test_module_id.toString(10)) - assert.equal(testSessionId.toString(10), testSessionEventContent.test_session_id.toString(10)) - }) + const testSessionEvent = events.find(event => event.type === 'test_session_end') + const testModuleEvent = events.find(event => event.type === 'test_module_end') + const testSuiteEvents = events.filter(event => event.type === 'test_suite_end') + const testEvents = events.filter(event => event.type === 'test') - assert.includeMembers(testEvents.map(test => test.content.resource), [ - `${featuresPath}farewell.feature.Say farewell`, - `${featuresPath}greetings.feature.Say greetings`, - `${featuresPath}greetings.feature.Say yeah`, - `${featuresPath}greetings.feature.Say yo`, - `${featuresPath}greetings.feature.Say skip` - ]) - assert.includeMembers(testEvents.map(test => test.content.meta[TEST_STATUS]), [ - 'pass', - 'pass', - 'pass', - 'fail', - 'skip' - ]) + const stepEvents = events.filter(event => event.type === 'span') + + const { content: testSessionEventContent } = testSessionEvent + const { content: testModuleEventContent } = testModuleEvent - testEvents.forEach(({ - content: { - meta, - test_suite_id: testSuiteId, - test_module_id: testModuleId, - test_session_id: testSessionId + if (runMode === 'parallel') { + assert.equal(testSessionEventContent.meta[CUCUMBER_IS_PARALLEL], 'true') } - }) => { - assert.exists(meta[TEST_COMMAND]) - assert.exists(meta[TEST_MODULE]) - assert.exists(testSuiteId) - assert.equal(testModuleId.toString(10), testModuleEventContent.test_module_id.toString(10)) - assert.equal(testSessionId.toString(10), testSessionEventContent.test_session_id.toString(10)) - }) - stepEvents.forEach(stepEvent => { - assert.equal(stepEvent.content.name, 'cucumber.step') - assert.property(stepEvent.content.meta, 'cucumber.step') - }) - }, 5000).then(() => done()).catch(done) + assert.exists(testSessionEventContent.test_session_id) + assert.exists(testSessionEventContent.meta[TEST_COMMAND]) + assert.exists(testSessionEventContent.meta[TEST_TOOLCHAIN]) + assert.equal(testSessionEventContent.resource.startsWith('test_session.'), true) + assert.equal(testSessionEventContent.meta[TEST_STATUS], 'fail') + + assert.exists(testModuleEventContent.test_session_id) + assert.exists(testModuleEventContent.test_module_id) + assert.exists(testModuleEventContent.meta[TEST_COMMAND]) + assert.exists(testModuleEventContent.meta[TEST_MODULE]) + assert.equal(testModuleEventContent.resource.startsWith('test_module.'), true) + assert.equal(testModuleEventContent.meta[TEST_STATUS], 'fail') + assert.equal( + testModuleEventContent.test_session_id.toString(10), + testSessionEventContent.test_session_id.toString(10) + ) + + assert.includeMembers(testSuiteEvents.map(suite => suite.content.resource), [ + `test_suite.${featuresPath}farewell.feature`, + `test_suite.${featuresPath}greetings.feature` + ]) + assert.includeMembers(testSuiteEvents.map(suite => suite.content.meta[TEST_STATUS]), [ + 'pass', + 'fail' + ]) + + testSuiteEvents.forEach(({ + content: { + meta, + metrics, + test_suite_id: testSuiteId, + test_module_id: testModuleId, + test_session_id: testSessionId + } + }) => { + assert.exists(meta[TEST_COMMAND]) + assert.exists(meta[TEST_MODULE]) + assert.exists(testSuiteId) + assert.equal(testModuleId.toString(10), testModuleEventContent.test_module_id.toString(10)) + assert.equal(testSessionId.toString(10), testSessionEventContent.test_session_id.toString(10)) + assert.isTrue(meta[TEST_SOURCE_FILE].startsWith(featuresPath)) + assert.equal(metrics[TEST_SOURCE_START], 1) + assert.exists(metrics[DD_HOST_CPU_COUNT]) + }) + + assert.includeMembers(testEvents.map(test => test.content.resource), [ + `${featuresPath}farewell.feature.Say farewell`, + `${featuresPath}greetings.feature.Say greetings`, + `${featuresPath}greetings.feature.Say yeah`, + `${featuresPath}greetings.feature.Say yo`, + `${featuresPath}greetings.feature.Say skip` + ]) + assert.includeMembers(testEvents.map(test => test.content.meta[TEST_STATUS]), [ + 'pass', + 'pass', + 'pass', + 'fail', + 'skip' + ]) + + testEvents.forEach(({ + content: { + meta, + metrics, + test_suite_id: testSuiteId, + test_module_id: testModuleId, + test_session_id: testSessionId + } + }) => { + assert.exists(meta[TEST_COMMAND]) + assert.exists(meta[TEST_MODULE]) + assert.exists(testSuiteId) + assert.equal(testModuleId.toString(10), testModuleEventContent.test_module_id.toString(10)) + assert.equal(testSessionId.toString(10), testSessionEventContent.test_session_id.toString(10)) + assert.equal(meta[TEST_SOURCE_FILE].startsWith('ci-visibility/features'), true) + // Can read DD_TAGS + assert.propertyVal(meta, 'test.customtag', 'customvalue') + assert.propertyVal(meta, 'test.customtag2', 'customvalue2') + if (runMode === 'parallel') { + assert.propertyVal(meta, CUCUMBER_IS_PARALLEL, 'true') + } + assert.exists(metrics[DD_HOST_CPU_COUNT]) + }) + + stepEvents.forEach(stepEvent => { + assert.equal(stepEvent.content.name, 'cucumber.step') + assert.property(stepEvent.content.meta, 'cucumber.step') + }) + }, 5000) + + childProcess = exec( + runCommand, + { + cwd, + env: { + ...envVars, + DD_TAGS: 'test.customtag:customvalue,test.customtag2:customvalue2', + DD_TEST_SESSION_NAME: 'my-test-session' + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + receiverPromise.then(() => done()).catch(done) + }) + }) + }) + context('intelligent test runner', () => { + it('can report git metadata', (done) => { + const searchCommitsRequestPromise = receiver.payloadReceived( + ({ url }) => url.endsWith('/api/v2/git/repository/search_commits') + ) + const packfileRequestPromise = receiver + .payloadReceived(({ url }) => url.endsWith('/api/v2/git/repository/packfile')) + const eventsRequestPromise = receiver.payloadReceived(({ url }) => url.endsWith('/api/v2/citestcycle')) + + Promise.all([ + searchCommitsRequestPromise, + packfileRequestPromise, + eventsRequestPromise + ]).then(([searchCommitRequest, packfileRequest, eventsRequest]) => { + if (isAgentless) { + assert.propertyVal(searchCommitRequest.headers, 'dd-api-key', '1') + assert.propertyVal(packfileRequest.headers, 'dd-api-key', '1') + } else { + assert.notProperty(searchCommitRequest.headers, 'dd-api-key') + assert.notProperty(packfileRequest.headers, 'dd-api-key') + } + + const eventTypes = eventsRequest.payload.events.map(event => event.type) + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + ) + assert.equal(numSuites, 2) + + done() + }).catch(done) childProcess = exec( runTestsCommand, @@ -242,375 +275,1117 @@ versions.forEach(version => { } ) }) - context('intelligent test runner', () => { - it('can report git metadata', (done) => { - const searchCommitsRequestPromise = receiver.payloadReceived( - ({ url }) => url.endsWith('/api/v2/git/repository/search_commits') + it('can report code coverage', (done) => { + const libraryConfigRequestPromise = receiver.payloadReceived( + ({ url }) => url.endsWith('/api/v2/libraries/tests/services/setting') + ) + const codeCovRequestPromise = receiver.payloadReceived(({ url }) => url.endsWith('/api/v2/citestcov')) + const eventsRequestPromise = receiver.payloadReceived(({ url }) => url.endsWith('/api/v2/citestcycle')) + + Promise.all([ + libraryConfigRequestPromise, + codeCovRequestPromise, + eventsRequestPromise + ]).then(([libraryConfigRequest, codeCovRequest, eventsRequest]) => { + const [coveragePayload] = codeCovRequest.payload + if (isAgentless) { + assert.propertyVal(libraryConfigRequest.headers, 'dd-api-key', '1') + assert.propertyVal(codeCovRequest.headers, 'dd-api-key', '1') + } else { + assert.notProperty(libraryConfigRequest.headers, 'dd-api-key') + assert.notProperty(codeCovRequest.headers, 'dd-api-key', '1') + } + + assert.propertyVal(coveragePayload, 'name', 'coverage1') + assert.propertyVal(coveragePayload, 'filename', 'coverage1.msgpack') + assert.propertyVal(coveragePayload, 'type', 'application/msgpack') + assert.include(coveragePayload.content, { + version: 2 + }) + const allCoverageFiles = codeCovRequest.payload + .flatMap(coverage => coverage.content.coverages) + .flatMap(file => file.files) + .map(file => file.filename) + + assert.includeMembers(allCoverageFiles, [ + `${featuresPath}support/steps.${fileExtension}`, + `${featuresPath}farewell.feature`, + `${featuresPath}greetings.feature` + ]) + // steps is twice because there are two suites using it + assert.equal( + allCoverageFiles.filter(file => file === `${featuresPath}support/steps.${fileExtension}`).length, + 2 ) - const packfileRequestPromise = receiver - .payloadReceived(({ url }) => url.endsWith('/api/v2/git/repository/packfile')) + assert.exists(coveragePayload.content.coverages[0].test_session_id) + assert.exists(coveragePayload.content.coverages[0].test_suite_id) + + const testSession = eventsRequest + .payload + .events + .find(event => event.type === 'test_session_end') + .content + assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) + + const eventTypes = eventsRequest.payload.events.map(event => event.type) + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + ) + assert.equal(numSuites, 2) + }).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: envVars, + stdio: 'pipe' + } + ) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('exit', () => { + // check that reported coverage is still the same + assert.include(testOutput, 'Lines : 100%') + done() + }) + }) + it('does not report code coverage if disabled by the API', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false + }) + + receiver.assertPayloadReceived(() => { + const error = new Error('it should not report code coverage') + done(error) + }, ({ url }) => url.endsWith('/api/v2/citestcov')).catch(() => {}) + + receiver.assertPayloadReceived(({ payload }) => { + const eventTypes = payload.events.map(event => event.type) + assert.includeMembers(eventTypes, ['test', 'test_session_end', 'test_module_end', 'test_suite_end']) + const testSession = payload.events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'false') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'false') + assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) + const testModule = payload.events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'false') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'false') + }, ({ url }) => url.endsWith('/api/v2/citestcycle')).then(() => done()).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: envVars, + stdio: 'inherit' + } + ) + }) + it('can skip suites received by the intelligent test runner API and still reports code coverage', + (done) => { + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: `${featuresPath}farewell.feature` + } + }]) + + const skippableRequestPromise = receiver + .payloadReceived(({ url }) => url.endsWith('/api/v2/ci/tests/skippable')) + const coverageRequestPromise = receiver.payloadReceived(({ url }) => url.endsWith('/api/v2/citestcov')) const eventsRequestPromise = receiver.payloadReceived(({ url }) => url.endsWith('/api/v2/citestcycle')) Promise.all([ - searchCommitsRequestPromise, - packfileRequestPromise, + skippableRequestPromise, + coverageRequestPromise, eventsRequestPromise - ]).then(([searchCommitRequest, packfileRequest, eventsRequest]) => { + ]).then(([skippableRequest, coverageRequest, eventsRequest]) => { + const [coveragePayload] = coverageRequest.payload if (isAgentless) { - assert.propertyVal(searchCommitRequest.headers, 'dd-api-key', '1') - assert.propertyVal(packfileRequest.headers, 'dd-api-key', '1') + assert.propertyVal(skippableRequest.headers, 'dd-api-key', '1') + assert.propertyVal(coverageRequest.headers, 'dd-api-key', '1') + assert.propertyVal(eventsRequest.headers, 'dd-api-key', '1') } else { - assert.notProperty(searchCommitRequest.headers, 'dd-api-key') - assert.notProperty(packfileRequest.headers, 'dd-api-key') + assert.notProperty(skippableRequest.headers, 'dd-api-key', '1') + assert.notProperty(coverageRequest.headers, 'dd-api-key', '1') + assert.notProperty(eventsRequest.headers, 'dd-api-key', '1') } + assert.propertyVal(coveragePayload, 'name', 'coverage1') + assert.propertyVal(coveragePayload, 'filename', 'coverage1.msgpack') + assert.propertyVal(coveragePayload, 'type', 'application/msgpack') const eventTypes = eventsRequest.payload.events.map(event => event.type) + + const skippedSuite = eventsRequest.payload.events.find(event => + event.content.resource === `test_suite.${featuresPath}farewell.feature` + ).content + assert.propertyVal(skippedSuite.meta, TEST_STATUS, 'skip') + assert.propertyVal(skippedSuite.meta, TEST_SKIPPED_BY_ITR, 'true') + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) const numSuites = eventTypes.reduce( (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 ) assert.equal(numSuites, 2) + const testSession = eventsRequest + .payload.events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'true') + assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_TYPE, 'suite') + assert.propertyVal(testSession.metrics, TEST_ITR_SKIPPING_COUNT, 1) + const testModule = eventsRequest + .payload.events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'true') + assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_TYPE, 'suite') + assert.propertyVal(testModule.metrics, TEST_ITR_SKIPPING_COUNT, 1) done() }).catch(done) childProcess = exec( - runTestsCommand, + runTestsWithCoverageCommand, { cwd, env: envVars, - stdio: 'pipe' + stdio: 'inherit' } ) }) - it('can report code coverage', (done) => { - let testOutput - const itrConfigRequestPromise = receiver.payloadReceived( - ({ url }) => url.endsWith('/api/v2/libraries/tests/services/setting') + it('does not skip tests if git metadata upload fails', (done) => { + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: `${featuresPath}farewell.feature` + } + }]) + + receiver.setGitUploadStatus(404) + + receiver.assertPayloadReceived(() => { + const error = new Error('should not request skippable') + done(error) + }, ({ url }) => url.endsWith('/api/v2/ci/tests/skippable')) + + receiver.assertPayloadReceived(({ payload }) => { + const eventTypes = payload.events.map(event => event.type) + // because they are not skipped + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 ) - const codeCovRequestPromise = receiver.payloadReceived(({ url }) => url.endsWith('/api/v2/citestcov')) - const eventsRequestPromise = receiver.payloadReceived(({ url }) => url.endsWith('/api/v2/citestcycle')) + assert.equal(numSuites, 2) + const testSession = payload.events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + const testModule = payload.events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + }, ({ url }) => url.endsWith('/api/v2/citestcycle')).then(() => done()).catch(done) - Promise.all([ - itrConfigRequestPromise, - codeCovRequestPromise, - eventsRequestPromise - ]).then(([itrConfigRequest, codeCovRequest, eventsRequest]) => { - const [coveragePayload] = codeCovRequest.payload - if (isAgentless) { - assert.propertyVal(itrConfigRequest.headers, 'dd-api-key', '1') - assert.propertyVal(codeCovRequest.headers, 'dd-api-key', '1') - } else { - assert.notProperty(itrConfigRequest.headers, 'dd-api-key') - assert.notProperty(codeCovRequest.headers, 'dd-api-key', '1') + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: envVars, + stdio: 'inherit' + } + ) + }) + it('does not skip tests if test skipping is disabled by the API', (done) => { + receiver.setSettings({ + itr_enabled: true, + code_coverage: true, + tests_skipping: false + }) + + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: `${featuresPath}farewell.feature` + } + }]) + + receiver.assertPayloadReceived(() => { + const error = new Error('should not request skippable') + done(error) + }, ({ url }) => url.endsWith('/api/v2/ci/tests/skippable')) + + receiver.assertPayloadReceived(({ payload }) => { + const eventTypes = payload.events.map(event => event.type) + // because they are not skipped + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + ) + assert.equal(numSuites, 2) + }, ({ url }) => url.endsWith('/api/v2/citestcycle')).then(() => done()).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + }) + it('does not skip suites if suite is marked as unskippable', (done) => { + receiver.setSettings({ + itr_enabled: true, + code_coverage: true, + tests_skipping: true + }) + + receiver.setSuitesToSkip([ + { + type: 'suite', + attributes: { + suite: `${featuresPath}farewell.feature` + } + }, + { + type: 'suite', + attributes: { + suite: `${featuresPath}greetings.feature` } + } + ]) - assert.propertyVal(coveragePayload, 'name', 'coverage1') - assert.propertyVal(coveragePayload, 'filename', 'coverage1.msgpack') - assert.propertyVal(coveragePayload, 'type', 'application/msgpack') - assert.include(coveragePayload.content, { - version: 2 + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const suites = events.filter(event => event.type === 'test_suite_end') + + assert.equal(suites.length, 2) + + const testSession = events.find(event => event.type === 'test_session_end').content + const testModule = events.find(event => event.type === 'test_session_end').content + + assert.propertyVal(testSession.meta, TEST_ITR_UNSKIPPABLE, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_FORCED_RUN, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_UNSKIPPABLE, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_FORCED_RUN, 'true') + + const skippedSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/features/farewell.feature' + ).content + const forcedToRunSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/features/greetings.feature' + ).content + + assert.propertyVal(skippedSuite.meta, TEST_STATUS, 'skip') + assert.notProperty(skippedSuite.meta, TEST_ITR_UNSKIPPABLE) + assert.notProperty(skippedSuite.meta, TEST_ITR_FORCED_RUN) + + assert.propertyVal(forcedToRunSuite.meta, TEST_STATUS, 'fail') + assert.propertyVal(forcedToRunSuite.meta, TEST_ITR_UNSKIPPABLE, 'true') + assert.propertyVal(forcedToRunSuite.meta, TEST_ITR_FORCED_RUN, 'true') + }, 25000) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: envVars, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + it('only sets forced to run if suite was going to be skipped by ITR', (done) => { + receiver.setSettings({ + itr_enabled: true, + code_coverage: true, + tests_skipping: true + }) + + receiver.setSuitesToSkip([ + { + type: 'suite', + attributes: { + suite: `${featuresPath}farewell.feature` + } + } + ]) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const suites = events.filter(event => event.type === 'test_suite_end') + + assert.equal(suites.length, 2) + + const testSession = events.find(event => event.type === 'test_session_end').content + const testModule = events.find(event => event.type === 'test_session_end').content + + assert.propertyVal(testSession.meta, TEST_ITR_UNSKIPPABLE, 'true') + assert.notProperty(testSession.meta, TEST_ITR_FORCED_RUN) + assert.propertyVal(testModule.meta, TEST_ITR_UNSKIPPABLE, 'true') + assert.notProperty(testModule.meta, TEST_ITR_FORCED_RUN) + + const skippedSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/features/farewell.feature' + ) + const failedSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/features/greetings.feature' + ) + + assert.propertyVal(skippedSuite.content.meta, TEST_STATUS, 'skip') + assert.notProperty(skippedSuite.content.meta, TEST_ITR_UNSKIPPABLE) + assert.notProperty(skippedSuite.content.meta, TEST_ITR_FORCED_RUN) + + assert.propertyVal(failedSuite.content.meta, TEST_STATUS, 'fail') + assert.propertyVal(failedSuite.content.meta, TEST_ITR_UNSKIPPABLE, 'true') + assert.notProperty(failedSuite.content.meta, TEST_ITR_FORCED_RUN) + }, 25000) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: envVars, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + it('sets _dd.ci.itr.tests_skipped to false if the received suite is not skipped', (done) => { + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: `${featuresPath}not-existing.feature` + } + }]) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + assert.propertyVal(testSession.metrics, TEST_ITR_SKIPPING_COUNT, 0) + const testModule = events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + assert.propertyVal(testModule.metrics, TEST_ITR_SKIPPING_COUNT, 0) + }, 25000) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: envVars, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + if (!isAgentless) { + context('if the agent is not event platform proxy compatible', () => { + it('does not do any intelligent test runner request', (done) => { + receiver.setInfoResponse({ endpoints: [] }) + + receiver.assertPayloadReceived(() => { + const error = new Error('should not request search_commits') + done(error) + }, ({ url }) => url === '/evp_proxy/v2/api/v2/git/repository/search_commits') + receiver.assertPayloadReceived(() => { + const error = new Error('should not request search_commits') + done(error) + }, ({ url }) => url === '/api/v2/git/repository/search_commits') + receiver.assertPayloadReceived(() => { + const error = new Error('should not request setting') + done(error) + }, ({ url }) => url === '/api/v2/libraries/tests/services/setting') + receiver.assertPayloadReceived(() => { + const error = new Error('should not request setting') + done(error) + }, ({ url }) => url === '/evp_proxy/v2/api/v2/libraries/tests/services/setting') + + receiver.assertPayloadReceived(({ payload }) => { + const testSpans = payload.flatMap(trace => trace) + const resourceNames = testSpans.map(span => span.resource) + + assert.includeMembers(resourceNames, + [ + `${featuresPath}farewell.feature.Say farewell`, + `${featuresPath}greetings.feature.Say greetings`, + `${featuresPath}greetings.feature.Say yeah`, + `${featuresPath}greetings.feature.Say yo`, + `${featuresPath}greetings.feature.Say skip` + ] + ) + }, ({ url }) => url === '/v0.4/traces').then(() => done()).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisEvpProxyConfig(receiver.port), + stdio: 'inherit' + } + ) + }) + }) + } + it('reports itr_correlation_id in test suites', (done) => { + const itrCorrelationId = '4321' + receiver.setItrCorrelationId(itrCorrelationId) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSuites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) + testSuites.forEach(testSuite => { + assert.equal(testSuite.itr_correlation_id, itrCorrelationId) }) - const allCoverageFiles = codeCovRequest.payload - .flatMap(coverage => coverage.content.coverages) - .flatMap(file => file.files) - .map(file => file.filename) - - assert.includeMembers(allCoverageFiles, [ - `${featuresPath}support/steps.${fileExtension}`, - `${featuresPath}farewell.feature`, - `${featuresPath}greetings.feature` - ]) - // steps is twice because there are two suites using it + }, 25000) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: envVars, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + }) + + context('early flake detection', () => { + it('retries new tests', (done) => { + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } + } + }) + // cucumber.ci-visibility/features/farewell.feature.Say whatever will be considered new + receiver.setKnownTests( + { + cucumber: { + 'ci-visibility/features/farewell.feature': ['Say farewell'], + 'ci-visibility/features/greetings.feature': ['Say greetings', 'Say yeah', 'Say yo', 'Say skip'] + } + } + ) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const newTests = tests.filter(test => + test.resource === 'ci-visibility/features/farewell.feature.Say whatever' + ) + newTests.forEach(test => { + assert.propertyVal(test.meta, TEST_IS_NEW, 'true') + }) + const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + // all but one has been retried assert.equal( - allCoverageFiles.filter(file => file === `${featuresPath}support/steps.${fileExtension}`).length, - 2 + newTests.length - 1, + retriedTests.length ) - assert.exists(coveragePayload.content.coverages[0].test_session_id) - assert.exists(coveragePayload.content.coverages[0].test_suite_id) + assert.equal(retriedTests.length, NUM_RETRIES_EFD) + // Test name does not change + newTests.forEach(test => { + assert.equal(test.meta[TEST_NAME], 'Say whatever') + }) + }) + childProcess = exec( + runTestsCommand, + { + cwd, + env: envVars, + stdio: 'pipe' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) - const testSession = eventsRequest - .payload - .events - .find(event => event.type === 'test_session_end') - .content - assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) + it('is disabled if DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED is false', (done) => { + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } + } + }) - const eventTypes = eventsRequest.payload.events.map(event => event.type) - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const newTests = tests.filter(test => + test.meta[TEST_IS_NEW] === 'true' ) - assert.equal(numSuites, 2) + // new tests are not detected + assert.equal(newTests.length, 0) + }) + // cucumber.ci-visibility/features/farewell.feature.Say whatever will be considered new + receiver.setKnownTests({ + cucumber: { + 'ci-visibility/features/farewell.feature': ['Say farewell'], + 'ci-visibility/features/greetings.feature': ['Say greetings', 'Say yeah', 'Say yo', 'Say skip'] + } + }) + + childProcess = exec( + runTestsCommand, + { + cwd, + env: { ...envVars, DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED: 'false' }, + stdio: 'pipe' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() }).catch(done) + }) + }) - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: envVars, - stdio: 'pipe' + it('retries flaky tests and sets exit code to 0 as long as one attempt passes', (done) => { + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD } - ) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() + } + }) + // Tests in "cucumber.ci-visibility/features-flaky/flaky.feature" will be considered new + receiver.setKnownTests({}) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const testSuites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) + + tests.forEach(test => { + assert.propertyVal(test.meta, TEST_IS_NEW, 'true') + }) + // All test suites pass, even though there are failed tests + testSuites.forEach(testSuite => { + assert.propertyVal(testSuite.meta, TEST_STATUS, 'pass') + }) + + const failedAttempts = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + const passedAttempts = tests.filter(test => test.meta[TEST_STATUS] === 'pass') + + // (1 original run + 3 retries) / 2 + assert.equal(failedAttempts.length, 2) + assert.equal(passedAttempts.length, 2) }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() + + childProcess = exec( + './node_modules/.bin/cucumber-js ci-visibility/features-flaky/*.feature', + { + cwd, + env: envVars, + stdio: 'pipe' + } + ) + childProcess.on('exit', (exitCode) => { + assert.equal(exitCode, 0) + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('does not retry tests that are skipped', (done) => { + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } + } + }) + // "cucumber.ci-visibility/features/farewell.feature.Say whatever" will be considered new + // "cucumber.ci-visibility/features/greetings.feature.Say skip" will be considered new + receiver.setKnownTests({ + cucumber: { + 'ci-visibility/features/farewell.feature': ['Say farewell'], + 'ci-visibility/features/greetings.feature': ['Say greetings', 'Say yeah', 'Say yo'] + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const skippedNewTest = tests.filter(test => + test.resource === 'ci-visibility/features/greetings.feature.Say skip' + ) + // not retried + assert.equal(skippedNewTest.length, 1) }) - childProcess.on('exit', () => { - // check that reported coverage is still the same - assert.include(testOutput, 'Lines : 100%') + + childProcess = exec( + runTestsCommand, + { + cwd, + env: envVars, + stdio: 'pipe' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { done() - }) + }).catch(done) }) - it('does not report code coverage if disabled by the API', (done) => { - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false + }) + + it('does not run EFD if the known tests request fails', (done) => { + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } + } + }) + receiver.setKnownTestsResponseCode(500) + receiver.setKnownTests({}) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + assert.equal(tests.length, 6) + const newTests = tests.filter(test => + test.meta[TEST_IS_NEW] === 'true' + ) + assert.equal(newTests.length, 0) }) - receiver.assertPayloadReceived(() => { - const error = new Error('it should not report code coverage') - done(error) - }, ({ url }) => url.endsWith('/api/v2/citestcov')).catch(() => {}) + childProcess = exec( + runTestsCommand, + { cwd, env: envVars, stdio: 'pipe' } + ) - receiver.assertPayloadReceived(({ payload }) => { - const eventTypes = payload.events.map(event => event.type) - assert.includeMembers(eventTypes, ['test', 'test_session_end', 'test_module_end', 'test_suite_end']) - const testSession = payload.events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') - assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'false') - assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'false') - assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) - const testModule = payload.events.find(event => event.type === 'test_module_end').content - assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') - assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'false') - assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'false') - }, ({ url }) => url.endsWith('/api/v2/citestcycle')).then(() => done()).catch(done) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: envVars, - stdio: 'inherit' + it('bails out of EFD if the percentage of new tests is too high', (done) => { + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 0 + } + }) + // tests in cucumber.ci-visibility/features/farewell.feature will be considered new + receiver.setKnownTests( + { + cucumber: { + 'ci-visibility/features/greetings.feature': ['Say greetings', 'Say yeah', 'Say yo', 'Say skip'] } - ) + } + ) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ABORT_REASON, 'faulty') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') + assert.equal(newTests.length, 0) + + const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 0) + }) + + childProcess = exec( + runTestsCommand, + { + cwd, + env: envVars, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) }) - it('can skip suites received by the intelligent test runner API and still reports code coverage', - (done) => { - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: `${featuresPath}farewell.feature` + }) + + if (version !== '7.0.0') { // EFD in parallel mode only supported from cucumber>=11 + context('parallel mode', () => { + it('retries new tests', (done) => { + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } } - }]) - - const skippableRequestPromise = receiver - .payloadReceived(({ url }) => url.endsWith('/api/v2/ci/tests/skippable')) - const coverageRequestPromise = receiver.payloadReceived(({ url }) => url.endsWith('/api/v2/citestcov')) - const eventsRequestPromise = receiver.payloadReceived(({ url }) => url.endsWith('/api/v2/citestcycle')) - - Promise.all([ - skippableRequestPromise, - coverageRequestPromise, - eventsRequestPromise - ]).then(([skippableRequest, coverageRequest, eventsRequest]) => { - const [coveragePayload] = coverageRequest.payload - if (isAgentless) { - assert.propertyVal(skippableRequest.headers, 'dd-api-key', '1') - assert.propertyVal(coverageRequest.headers, 'dd-api-key', '1') - assert.propertyVal(eventsRequest.headers, 'dd-api-key', '1') - } else { - assert.notProperty(skippableRequest.headers, 'dd-api-key', '1') - assert.notProperty(coverageRequest.headers, 'dd-api-key', '1') - assert.notProperty(eventsRequest.headers, 'dd-api-key', '1') + }) + // cucumber.ci-visibility/features/farewell.feature.Say whatever will be considered new + receiver.setKnownTests( + { + cucumber: { + 'ci-visibility/features/farewell.feature': ['Say farewell'], + 'ci-visibility/features/greetings.feature': ['Say greetings', 'Say yeah', 'Say yo', 'Say skip'] + } } - assert.propertyVal(coveragePayload, 'name', 'coverage1') - assert.propertyVal(coveragePayload, 'filename', 'coverage1.msgpack') - assert.propertyVal(coveragePayload, 'type', 'application/msgpack') + ) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) - const eventTypes = eventsRequest.payload.events.map(event => event.type) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + assert.propertyVal(testSession.meta, CUCUMBER_IS_PARALLEL, 'true') - const skippedSuite = eventsRequest.payload.events.find(event => - event.content.resource === `test_suite.${featuresPath}farewell.feature` - ).content - assert.propertyVal(skippedSuite.meta, TEST_STATUS, 'skip') - assert.propertyVal(skippedSuite.meta, TEST_SKIPPED_BY_ITR, 'true') + const tests = events.filter(event => event.type === 'test').map(event => event.content) - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 - ) - assert.equal(numSuites, 2) - const testSession = eventsRequest - .payload.events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'true') - assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_TYPE, 'suite') - assert.propertyVal(testSession.metrics, TEST_ITR_SKIPPING_COUNT, 1) - - const testModule = eventsRequest - .payload.events.find(event => event.type === 'test_module_end').content - assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'true') - assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_TYPE, 'suite') - assert.propertyVal(testModule.metrics, TEST_ITR_SKIPPING_COUNT, 1) - done() - }).catch(done) + const newTests = tests.filter(test => + test.resource === 'ci-visibility/features/farewell.feature.Say whatever' + ) + newTests.forEach(test => { + assert.propertyVal(test.meta, TEST_IS_NEW, 'true') + // Test name does not change + assert.propertyVal(test.meta, TEST_NAME, 'Say whatever') + assert.propertyVal(test.meta, CUCUMBER_IS_PARALLEL, 'true') + }) + const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + // all but one has been retried + assert.equal( + newTests.length - 1, + retriedTests.length + ) + assert.equal(retriedTests.length, NUM_RETRIES_EFD) + }) childProcess = exec( - runTestsWithCoverageCommand, + parallelModeCommand, { cwd, env: envVars, - stdio: 'inherit' + stdio: 'pipe' } ) - }) - it('does not skip tests if git metadata upload fails', (done) => { - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: `${featuresPath}farewell.feature` - } - }]) - receiver.setGitUploadStatus(404) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) - receiver.assertPayloadReceived(() => { - const error = new Error('should not request skippable') - done(error) - }, ({ url }) => url.endsWith('/api/v2/ci/tests/skippable')) + it('retries flaky tests and sets exit code to 0 as long as one attempt passes', (done) => { + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } + } + }) + // Tests in "cucumber.ci-visibility/features-flaky/flaky.feature" will be considered new + receiver.setKnownTests({}) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + assert.propertyVal(testSession.meta, CUCUMBER_IS_PARALLEL, 'true') + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const testSuites = events + .filter(event => event.type === 'test_suite_end').map(event => event.content) + + tests.forEach(test => { + assert.propertyVal(test.meta, TEST_IS_NEW, 'true') + assert.propertyVal(test.meta, CUCUMBER_IS_PARALLEL, 'true') + }) + + // All test suites pass, even though there are failed tests + testSuites.forEach(testSuite => { + assert.propertyVal(testSuite.meta, TEST_STATUS, 'pass') + }) + + const failedAttempts = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + const passedAttempts = tests.filter(test => test.meta[TEST_STATUS] === 'pass') + + // (1 original run + 3 retries) / 2 + assert.equal(failedAttempts.length, 2) + assert.equal(passedAttempts.length, 2) + }) - receiver.assertPayloadReceived(({ payload }) => { - const eventTypes = payload.events.map(event => event.type) - // because they are not skipped - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + childProcess = exec( + './node_modules/.bin/cucumber-js ci-visibility/features-flaky/*.feature --parallel 2', + { + cwd, + env: envVars, + stdio: 'pipe' + } ) - assert.equal(numSuites, 2) - const testSession = payload.events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') - assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - const testModule = payload.events.find(event => event.type === 'test_module_end').content - assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') - assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - }, ({ url }) => url.endsWith('/api/v2/citestcycle')).then(() => done()).catch(done) - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: envVars, - stdio: 'inherit' - } - ) - }) - it('does not skip tests if test skipping is disabled by the API', (done) => { - receiver.setSettings({ - itr_enabled: true, - code_coverage: true, - tests_skipping: false + childProcess.on('exit', (exitCode) => { + assert.equal(exitCode, 0) + eventsPromise.then(() => { + done() + }).catch(done) + }) }) - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: `${featuresPath}farewell.feature` - } - }]) + it('bails out of EFD if the percentage of new tests is too high', (done) => { + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 0 + } + }) + // tests in cucumber.ci-visibility/features/farewell.feature will be considered new + receiver.setKnownTests( + { + cucumber: { + 'ci-visibility/features/greetings.feature': ['Say greetings', 'Say yeah', 'Say yo', 'Say skip'] + } + } + ) - receiver.assertPayloadReceived(() => { - const error = new Error('should not request skippable') - done(error) - }, ({ url }) => url.endsWith('/api/v2/ci/tests/skippable')) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) - receiver.assertPayloadReceived(({ payload }) => { - const eventTypes = payload.events.map(event => event.type) - // because they are not skipped - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ABORT_REASON, 'faulty') + assert.propertyVal(testSession.meta, CUCUMBER_IS_PARALLEL, 'true') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') + assert.equal(newTests.length, 0) + + const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 0) + }) + + childProcess = exec( + parallelModeCommand, + { + cwd, + env: envVars, + stdio: 'pipe' + } ) - assert.equal(numSuites, 2) - }, ({ url }) => url.endsWith('/api/v2/citestcycle')).then(() => done()).catch(done) - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'inherit' - } - ) - }) - it('does not skip suites if suite is marked as unskippable', (done) => { - receiver.setSettings({ - itr_enabled: true, - code_coverage: true, - tests_skipping: true + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) }) - receiver.setSuitesToSkip([ - { - type: 'suite', - attributes: { - suite: `${featuresPath}farewell.feature` + it('does not retry tests that are skipped', (done) => { + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } } - }, - { - type: 'suite', - attributes: { - suite: `${featuresPath}greetings.feature` + }) + // "cucumber.ci-visibility/features/farewell.feature.Say whatever" will be considered new + // "cucumber.ci-visibility/features/greetings.feature.Say skip" will be considered new + receiver.setKnownTests({ + cucumber: { + 'ci-visibility/features/farewell.feature': ['Say farewell'], + 'ci-visibility/features/greetings.feature': ['Say greetings', 'Say yeah', 'Say yo'] + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + assert.propertyVal(testSession.meta, CUCUMBER_IS_PARALLEL, 'true') + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const skippedNewTest = tests.filter(test => + test.resource === 'ci-visibility/features/greetings.feature.Say skip' + ) + // not retried + assert.equal(skippedNewTest.length, 1) + }) + + childProcess = exec( + parallelModeCommand, + { + cwd, + env: envVars, + stdio: 'pipe' } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + }) + } + }) + + if (version === 'latest') { // flaky test retries only supported from >=8.0.0 + context('flaky test retries', () => { + it('can retry failed tests', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + flaky_test_retries_enabled: true, + early_flake_detection: { + enabled: false } - ]) + }) const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { const events = payloads.flatMap(({ payload }) => payload.events) - const suites = events.filter(event => event.type === 'test_suite_end') - - assert.equal(suites.length, 2) - const testSession = events.find(event => event.type === 'test_session_end').content - const testModule = events.find(event => event.type === 'test_session_end').content + const tests = events.filter(event => event.type === 'test').map(event => event.content) - assert.propertyVal(testSession.meta, TEST_ITR_UNSKIPPABLE, 'true') - assert.propertyVal(testSession.meta, TEST_ITR_FORCED_RUN, 'true') - assert.propertyVal(testModule.meta, TEST_ITR_UNSKIPPABLE, 'true') - assert.propertyVal(testModule.meta, TEST_ITR_FORCED_RUN, 'true') + // 2 failures and 1 passed attempt + assert.equal(tests.length, 3) - const skippedSuite = suites.find( - event => event.content.resource === 'test_suite.ci-visibility/features/farewell.feature' - ).content - const forcedToRunSuite = suites.find( - event => event.content.resource === 'test_suite.ci-visibility/features/greetings.feature' - ).content + const failedTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(failedTests.length, 2) + const passedTests = tests.filter(test => test.meta[TEST_STATUS] === 'pass') + assert.equal(passedTests.length, 1) - assert.propertyVal(skippedSuite.meta, TEST_STATUS, 'skip') - assert.notProperty(skippedSuite.meta, TEST_ITR_UNSKIPPABLE) - assert.notProperty(skippedSuite.meta, TEST_ITR_FORCED_RUN) - - assert.propertyVal(forcedToRunSuite.meta, TEST_STATUS, 'fail') - assert.propertyVal(forcedToRunSuite.meta, TEST_ITR_UNSKIPPABLE, 'true') - assert.propertyVal(forcedToRunSuite.meta, TEST_ITR_FORCED_RUN, 'true') - }, 25000) + // All but the first one are retries + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 2) + }) childProcess = exec( - runTestsWithCoverageCommand, + './node_modules/.bin/cucumber-js ci-visibility/features-retry/*.feature', { cwd, env: envVars, - stdio: 'inherit' + stdio: 'pipe' } ) @@ -620,59 +1395,39 @@ versions.forEach(version => { }).catch(done) }) }) - it('only sets forced to run if suite was going to be skipped by ITR', (done) => { - receiver.setSettings({ - itr_enabled: true, - code_coverage: true, - tests_skipping: true - }) - receiver.setSuitesToSkip([ - { - type: 'suite', - attributes: { - suite: `${featuresPath}farewell.feature` - } + it('is disabled if DD_CIVISIBILITY_FLAKY_RETRY_ENABLED is false', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + flaky_test_retries_enabled: true, + early_flake_detection: { + enabled: false } - ]) + }) const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { const events = payloads.flatMap(({ payload }) => payload.events) - const suites = events.filter(event => event.type === 'test_suite_end') - assert.equal(suites.length, 2) - - const testSession = events.find(event => event.type === 'test_session_end').content - const testModule = events.find(event => event.type === 'test_session_end').content - - assert.propertyVal(testSession.meta, TEST_ITR_UNSKIPPABLE, 'true') - assert.notProperty(testSession.meta, TEST_ITR_FORCED_RUN) - assert.propertyVal(testModule.meta, TEST_ITR_UNSKIPPABLE, 'true') - assert.notProperty(testModule.meta, TEST_ITR_FORCED_RUN) - - const skippedSuite = suites.find( - event => event.content.resource === 'test_suite.ci-visibility/features/farewell.feature' - ) - const failedSuite = suites.find( - event => event.content.resource === 'test_suite.ci-visibility/features/greetings.feature' - ) + const tests = events.filter(event => event.type === 'test').map(event => event.content) - assert.propertyVal(skippedSuite.content.meta, TEST_STATUS, 'skip') - assert.notProperty(skippedSuite.content.meta, TEST_ITR_UNSKIPPABLE) - assert.notProperty(skippedSuite.content.meta, TEST_ITR_FORCED_RUN) + assert.equal(tests.length, 1) - assert.propertyVal(failedSuite.content.meta, TEST_STATUS, 'fail') - assert.propertyVal(failedSuite.content.meta, TEST_ITR_UNSKIPPABLE, 'true') - assert.notProperty(failedSuite.content.meta, TEST_ITR_FORCED_RUN) - }, 25000) + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 0) + }) childProcess = exec( - runTestsWithCoverageCommand, + './node_modules/.bin/cucumber-js ci-visibility/features-retry/*.feature', { cwd, - env: envVars, - stdio: 'inherit' + env: { + ...envVars, + DD_CIVISIBILITY_FLAKY_RETRY_ENABLED: 'false' + }, + stdio: 'pipe' } ) @@ -682,91 +1437,189 @@ versions.forEach(version => { }).catch(done) }) }) - it('sets _dd.ci.itr.tests_skipped to false if the received suite is not skipped', (done) => { - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: `${featuresPath}not-existing.feature` + + it('retries DD_CIVISIBILITY_FLAKY_RETRY_COUNT times', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + flaky_test_retries_enabled: true, + early_flake_detection: { + enabled: false } - }]) + }) + const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { const events = payloads.flatMap(({ payload }) => payload.events) - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') - assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - assert.propertyVal(testSession.metrics, TEST_ITR_SKIPPING_COUNT, 0) - const testModule = events.find(event => event.type === 'test_module_end').content - assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') - assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - assert.propertyVal(testModule.metrics, TEST_ITR_SKIPPING_COUNT, 0) - }, 25000) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + // 2 failures + assert.equal(tests.length, 2) + + const failedTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(failedTests.length, 2) + const passedTests = tests.filter(test => test.meta[TEST_STATUS] === 'pass') + assert.equal(passedTests.length, 0) + + // All but the first one are retries + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 1) + }) childProcess = exec( - runTestsWithCoverageCommand, + './node_modules/.bin/cucumber-js ci-visibility/features-retry/*.feature', { cwd, - env: envVars, - stdio: 'inherit' + env: { + ...envVars, + DD_CIVISIBILITY_FLAKY_RETRY_COUNT: 1 + }, + stdio: 'pipe' } ) + childProcess.on('exit', () => { eventsPromise.then(() => { done() }).catch(done) }) }) - if (!isAgentless) { - context('if the agent is not event platform proxy compatible', () => { - it('does not do any intelligent test runner request', (done) => { - receiver.setInfoResponse({ endpoints: [] }) - - receiver.assertPayloadReceived(() => { - const error = new Error('should not request search_commits') - done(error) - }, ({ url }) => url === '/evp_proxy/v2/api/v2/git/repository/search_commits') - receiver.assertPayloadReceived(() => { - const error = new Error('should not request search_commits') - done(error) - }, ({ url }) => url === '/api/v2/git/repository/search_commits') - receiver.assertPayloadReceived(() => { - const error = new Error('should not request setting') - done(error) - }, ({ url }) => url === '/api/v2/libraries/tests/services/setting') - receiver.assertPayloadReceived(() => { - const error = new Error('should not request setting') - done(error) - }, ({ url }) => url === '/evp_proxy/v2/api/v2/libraries/tests/services/setting') - - receiver.assertPayloadReceived(({ payload }) => { - const testSpans = payload.flatMap(trace => trace) - const resourceNames = testSpans.map(span => span.resource) - - assert.includeMembers(resourceNames, - [ - `${featuresPath}farewell.feature.Say farewell`, - `${featuresPath}greetings.feature.Say greetings`, - `${featuresPath}greetings.feature.Say yeah`, - `${featuresPath}greetings.feature.Say yo`, - `${featuresPath}greetings.feature.Say skip` - ] - ) - }, ({ url }) => url === '/v0.4/traces').then(() => done()).catch(done) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisEvpProxyConfig(receiver.port), - stdio: 'inherit' - } - ) - }) - }) - } }) + } + }) + }) + + it('correctly calculates test code owners when working directory is not repository root', (done) => { + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const test = events.find(event => event.type === 'test').content + const testSuite = events.find(event => event.type === 'test_suite_end').content + // The test is in a subproject + assert.notEqual(test.meta[TEST_SOURCE_FILE], test.meta[TEST_SUITE]) + assert.equal(test.meta[TEST_CODE_OWNERS], JSON.stringify(['@datadog-dd-trace-js'])) + assert.equal(testSuite.meta[TEST_CODE_OWNERS], JSON.stringify(['@datadog-dd-trace-js'])) + }) + + childProcess = exec( + 'node ../../node_modules/.bin/cucumber-js features/*.feature', + { + cwd: `${cwd}/ci-visibility/subproject`, + env: { + ...getCiVisAgentlessConfig(receiver.port) + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('takes into account untested files if "all" is passed to nyc', (done) => { + const linesPctMatchRegex = /Lines\s*:\s*([\d.]+)%/ + let linesPctMatch + let linesPctFromNyc = 0 + let codeCoverageWithUntestedFiles = 0 + let codeCoverageWithoutUntestedFiles = 0 + + let eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + codeCoverageWithUntestedFiles = testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT] + }) + + childProcess = exec( + './node_modules/nyc/bin/nyc.js --all -r=text-summary --nycrc-path ./my-nyc.config.js ' + + 'node ./node_modules/.bin/cucumber-js ci-visibility/features/*.feature', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + NYC_INCLUDE: JSON.stringify( + [ + 'ci-visibility/features/**', + 'ci-visibility/features-esm/**' + ] + ) + }, + stdio: 'inherit' + } + ) + + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + + childProcess.on('exit', () => { + linesPctMatch = testOutput.match(linesPctMatchRegex) + linesPctFromNyc = linesPctMatch ? Number(linesPctMatch[1]) : null + + assert.equal( + linesPctFromNyc, + codeCoverageWithUntestedFiles, + 'nyc --all output does not match the reported coverage' + ) + + // reset test output for next test session + testOutput = '' + // we run the same tests without the all flag + childProcess = exec( + './node_modules/nyc/bin/nyc.js -r=text-summary --nycrc-path ./my-nyc.config.js ' + + 'node ./node_modules/.bin/cucumber-js ci-visibility/features/*.feature', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + NYC_INCLUDE: JSON.stringify( + [ + 'ci-visibility/features/**', + 'ci-visibility/features-esm/**' + ] + ) + }, + stdio: 'inherit' + } + ) + + eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + codeCoverageWithoutUntestedFiles = testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT] + }) + + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + + childProcess.on('exit', () => { + linesPctMatch = testOutput.match(linesPctMatchRegex) + linesPctFromNyc = linesPctMatch ? Number(linesPctMatch[1]) : null + + assert.equal( + linesPctFromNyc, + codeCoverageWithoutUntestedFiles, + 'nyc output does not match the reported coverage (no --all flag)' + ) + + eventsPromise.then(() => { + assert.isAbove(codeCoverageWithoutUntestedFiles, codeCoverageWithUntestedFiles) + done() + }).catch(done) }) }) }) diff --git a/integration-tests/cypress-config.json b/integration-tests/cypress-config.json index 3bd4dc31817..3ad19f9f90a 100644 --- a/integration-tests/cypress-config.json +++ b/integration-tests/cypress-config.json @@ -4,5 +4,6 @@ "pluginsFile": "cypress/plugins-old/index.js", "supportFile": "cypress/support/e2e.js", "integrationFolder": "cypress/e2e", - "defaultCommandTimeout": 100 + "defaultCommandTimeout": 100, + "nodeVersion": "system" } diff --git a/integration-tests/cypress-esm-config.mjs b/integration-tests/cypress-esm-config.mjs index 2cc098144af..92888de62e7 100644 --- a/integration-tests/cypress-esm-config.mjs +++ b/integration-tests/cypress-esm-config.mjs @@ -7,10 +7,34 @@ async function runCypress () { defaultCommandTimeout: 100, e2e: { setupNodeEvents (on, config) { - import('../ci/cypress/plugin.js').then(module => { - module.default(on, config) + if (process.env.CYPRESS_ENABLE_INCOMPATIBLE_PLUGIN) { + import('cypress-fail-fast/plugin').then(module => { + module.default(on, config) + }) + } + if (process.env.CYPRESS_ENABLE_AFTER_RUN_CUSTOM) { + on('after:run', (...args) => { + // do custom stuff + // and call after-run at the end + return import('dd-trace/ci/cypress/after-run').then(module => { + module.default(...args) + }) + }) + } + if (process.env.CYPRESS_ENABLE_AFTER_SPEC_CUSTOM) { + on('after:spec', (...args) => { + // do custom stuff + // and call after-spec at the end + return import('dd-trace/ci/cypress/after-spec').then(module => { + module.default(...args) + }) + }) + } + return import('dd-trace/ci/cypress/plugin').then(module => { + return module.default(on, config) }) - } + }, + specPattern: process.env.SPEC_PATTERN || 'cypress/e2e/**/*.cy.js' }, video: false, screenshotOnRunFailure: false diff --git a/integration-tests/cypress.config.js b/integration-tests/cypress.config.js index cec58078fbe..799ca06df8c 100644 --- a/integration-tests/cypress.config.js +++ b/integration-tests/cypress.config.js @@ -1,9 +1,32 @@ +const ddAfterRun = require('dd-trace/ci/cypress/after-run') +const ddAfterSpec = require('dd-trace/ci/cypress/after-spec') +const cypressFailFast = require('cypress-fail-fast/plugin') +const ddTracePlugin = require('dd-trace/ci/cypress/plugin') + module.exports = { defaultCommandTimeout: 100, e2e: { setupNodeEvents (on, config) { - require('dd-trace/ci/cypress/plugin')(on, config) - } + if (process.env.CYPRESS_ENABLE_INCOMPATIBLE_PLUGIN) { + cypressFailFast(on, config) + } + if (process.env.CYPRESS_ENABLE_AFTER_RUN_CUSTOM) { + on('after:run', (...args) => { + // do custom stuff + // and call after-run at the end + return ddAfterRun(...args) + }) + } + if (process.env.CYPRESS_ENABLE_AFTER_SPEC_CUSTOM) { + on('after:spec', (...args) => { + // do custom stuff + // and call after-spec at the end + return ddAfterSpec(...args) + }) + } + return ddTracePlugin(on, config) + }, + specPattern: process.env.SPEC_PATTERN || 'cypress/e2e/**/*.cy.js' }, video: false, screenshotOnRunFailure: false diff --git a/integration-tests/cypress/cypress.spec.js b/integration-tests/cypress/cypress.spec.js index b781875bb98..afc79b2ebe5 100644 --- a/integration-tests/cypress/cypress.spec.js +++ b/integration-tests/cypress/cypress.spec.js @@ -26,19 +26,30 @@ const { TEST_ITR_SKIPPING_COUNT, TEST_ITR_SKIPPING_TYPE, TEST_ITR_UNSKIPPABLE, - TEST_ITR_FORCED_RUN + TEST_ITR_FORCED_RUN, + TEST_SOURCE_FILE, + TEST_SOURCE_START, + TEST_IS_NEW, + TEST_IS_RETRY, + TEST_EARLY_FLAKE_ENABLED, + TEST_SUITE, + TEST_CODE_OWNERS, + TEST_SESSION_NAME, + TEST_LEVEL_EVENT_TYPES } = require('../../packages/dd-trace/src/plugins/util/test') +const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants') -const semver = require('semver') +const { NODE_MAJOR } = require('../../version') const version = process.env.CYPRESS_VERSION const hookFile = 'dd-trace/loader-hook.mjs' +const NUM_RETRIES_EFD = 3 -const moduleType = [ +const moduleTypes = [ { type: 'commonJS', testCommand: function commandWithSuffic (version) { - const commandSuffix = version === '6.7.0' ? '--config-file cypress-config.json' : '' + const commandSuffix = version === '6.7.0' ? '--config-file cypress-config.json --spec "cypress/e2e/*.cy.js"' : '' return `./node_modules/.bin/cypress run ${commandSuffix}` } }, @@ -46,14 +57,17 @@ const moduleType = [ type: 'esm', testCommand: `node --loader=${hookFile} ./cypress-esm-config.mjs` } -] +].filter(moduleType => !process.env.CYPRESS_MODULE_TYPE || process.env.CYPRESS_MODULE_TYPE === moduleType.type) -moduleType.forEach(({ +moduleTypes.forEach(({ type, testCommand }) => { // cypress only supports esm on versions >= 10.0.0 - if (type === 'esm' && semver.satisfies(version, '<10.0.0')) { + if (type === 'esm' && version === '6.7.0') { + return + } + if (version === '6.7.0' && NODE_MAJOR > 16) { return } describe(`cypress@${version} ${type}`, function () { @@ -66,7 +80,8 @@ moduleType.forEach(({ } before(async () => { - sandbox = await createSandbox([`cypress@${version}`], true) + // cypress-fail-fast is required as an incompatible plugin + sandbox = await createSandbox([`cypress@${version}`, 'cypress-fail-fast@7.1.0'], true) cwd = sandbox.folder webAppPort = await getPort() webAppServer.listen(webAppPort) @@ -78,8 +93,7 @@ moduleType.forEach(({ }) beforeEach(async function () { - const port = await getPort() - receiver = await new FakeCiVisIntake(port).start() + receiver = await new FakeCiVisIntake().start() }) afterEach(async () => { @@ -108,7 +122,8 @@ moduleType.forEach(({ env: { ...restEnvVars, CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, - DD_SITE: '= invalid = url' + DD_SITE: '= invalid = url', + SPEC_PATTERN: 'cypress/e2e/spec.cy.js' }, stdio: 'pipe' } @@ -119,9 +134,23 @@ moduleType.forEach(({ childProcess.stderr.on('data', (chunk) => { testOutput += chunk.toString() }) + + // TODO: remove once we find the source of flakiness + childProcess.stdout.pipe(process.stdout) + childProcess.stderr.pipe(process.stderr) + childProcess.on('exit', () => { assert.notInclude(testOutput, 'TypeError') - assert.include(testOutput, '3 of 4 failed') + // TODO: remove try/catch once we find the source of flakiness + try { + assert.include(testOutput, '1 of 1 failed') + } catch (e) { + // eslint-disable-next-line no-console + console.log('---- Actual test output -----') + // eslint-disable-next-line no-console + console.log(testOutput) + throw e + } done() }) }) @@ -200,6 +229,13 @@ moduleType.forEach(({ it('can run and report tests', (done) => { const receiverPromise = receiver .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const metadataDicts = payloads.flatMap(({ payload }) => payload.metadata) + + metadataDicts.forEach(metadata => { + for (const testLevel of TEST_LEVEL_EVENT_TYPES) { + assert.equal(metadata[testLevel][TEST_SESSION_NAME], 'my-test-session') + } + }) const events = payloads.flatMap(({ payload }) => payload.events) const testSessionEvent = events.find(event => event.type === 'test_session_end') @@ -241,6 +277,7 @@ moduleType.forEach(({ testSuiteEvents.forEach(({ content: { meta, + metrics, test_suite_id: testSuiteId, test_module_id: testModuleId, test_session_id: testSessionId @@ -251,6 +288,9 @@ moduleType.forEach(({ assert.exists(testSuiteId) assert.equal(testModuleId.toString(10), testModuleEventContent.test_module_id.toString(10)) assert.equal(testSessionId.toString(10), testSessionEventContent.test_session_id.toString(10)) + assert.isTrue(meta[TEST_SOURCE_FILE].startsWith('cypress/e2e/')) + assert.equal(metrics[TEST_SOURCE_START], 1) + assert.exists(metrics[DD_HOST_CPU_COUNT]) }) assert.includeMembers(testEvents.map(test => test.content.resource), [ @@ -268,6 +308,7 @@ moduleType.forEach(({ testEvents.forEach(({ content: { meta, + metrics, test_suite_id: testSuiteId, test_module_id: testModuleId, test_session_id: testSessionId @@ -278,6 +319,11 @@ moduleType.forEach(({ assert.exists(testSuiteId) assert.equal(testModuleId.toString(10), testModuleEventContent.test_module_id.toString(10)) assert.equal(testSessionId.toString(10), testSessionEventContent.test_session_id.toString(10)) + assert.equal(meta[TEST_SOURCE_FILE].startsWith('cypress/e2e/'), true) + // Can read DD_TAGS + assert.propertyVal(meta, 'test.customtag', 'customvalue') + assert.propertyVal(meta, 'test.customtag2', 'customvalue2') + assert.exists(metrics[DD_HOST_CPU_COUNT]) }) }, 25000) @@ -292,7 +338,9 @@ moduleType.forEach(({ cwd, env: { ...restEnvVars, - CYPRESS_BASE_URL: `http://localhost:${webAppPort}` + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + DD_TAGS: 'test.customtag:customvalue,test.customtag2:customvalue2', + DD_TEST_SESSION_NAME: 'my-test-session' }, stdio: 'pipe' } @@ -337,7 +385,8 @@ moduleType.forEach(({ cwd, env: { ...restEnvVars, - CYPRESS_BASE_URL: `http://localhost:${webAppPort}` + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: 'cypress/e2e/spec.cy.js' }, stdio: 'pipe' } @@ -369,7 +418,8 @@ moduleType.forEach(({ cwd, env: { ...restEnvVars, - CYPRESS_BASE_URL: `http://localhost:${webAppPort}` + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: 'cypress/e2e/spec.cy.js' }, stdio: 'pipe' } @@ -385,6 +435,7 @@ moduleType.forEach(({ }).catch(done) }) }) + it('does not report code coverage if disabled by the API', (done) => { receiver.setSettings({ code_coverage: false, @@ -414,7 +465,8 @@ moduleType.forEach(({ cwd, env: { ...restEnvVars, - CYPRESS_BASE_URL: `http://localhost:${webAppPort}` + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: 'cypress/e2e/spec.cy.js' }, stdio: 'pipe' } @@ -426,6 +478,7 @@ moduleType.forEach(({ }).catch(done) }) }) + it('can skip tests received by the intelligent test runner API and still reports code coverage', (done) => { receiver.setSuitesToSkip([{ type: 'test', @@ -484,7 +537,8 @@ moduleType.forEach(({ cwd, env: { ...restEnvVars, - CYPRESS_BASE_URL: `http://localhost:${webAppPort}` + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: 'cypress/e2e/{other,spec}.cy.js' }, stdio: 'pipe' } @@ -495,6 +549,7 @@ moduleType.forEach(({ }).catch(done) }) }) + it('does not skip tests if test skipping is disabled by the API', (done) => { receiver.setSettings({ code_coverage: true, @@ -521,6 +576,7 @@ moduleType.forEach(({ event.content.resource === 'cypress/e2e/other.cy.js.context passes' ) assert.exists(notSkippedTest) + assert.equal(notSkippedTest.content.meta[TEST_STATUS], 'pass') }, 25000) const { @@ -534,7 +590,8 @@ moduleType.forEach(({ cwd, env: { ...restEnvVars, - CYPRESS_BASE_URL: `http://localhost:${webAppPort}` + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: 'cypress/e2e/other.cy.js' }, stdio: 'pipe' } @@ -546,6 +603,7 @@ moduleType.forEach(({ }).catch(done) }) }) + it('does not skip tests if suite is marked as unskippable', (done) => { receiver.setSettings({ code_coverage: true, @@ -607,7 +665,8 @@ moduleType.forEach(({ cwd, env: { ...restEnvVars, - CYPRESS_BASE_URL: `http://localhost:${webAppPort}` + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: 'cypress/e2e/{other,spec}.cy.js' }, stdio: 'pipe' } @@ -619,6 +678,7 @@ moduleType.forEach(({ }).catch(done) }) }) + it('only sets forced to run if test was going to be skipped by ITR', (done) => { receiver.setSettings({ code_coverage: true, @@ -675,7 +735,8 @@ moduleType.forEach(({ cwd, env: { ...restEnvVars, - CYPRESS_BASE_URL: `http://localhost:${webAppPort}` + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: 'cypress/e2e/{other,spec}.cy.js' }, stdio: 'pipe' } @@ -687,6 +748,7 @@ moduleType.forEach(({ }).catch(done) }) }) + it('sets _dd.ci.itr.tests_skipped to false if the received test is not skipped', (done) => { receiver.setSuitesToSkip([{ type: 'test', @@ -727,7 +789,8 @@ moduleType.forEach(({ cwd, env: { ...restEnvVars, - CYPRESS_BASE_URL: `http://localhost:${webAppPort}` + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: 'cypress/e2e/spec.cy.js' }, stdio: 'pipe' } @@ -738,6 +801,658 @@ moduleType.forEach(({ }).catch(done) }) }) + + it('reports itr_correlation_id in tests', (done) => { + const itrCorrelationId = '4321' + receiver.setItrCorrelationId(itrCorrelationId) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + tests.forEach(test => { + assert.equal(test.itr_correlation_id, itrCorrelationId) + }) + }, 25000) + + const { + NODE_OPTIONS, + ...restEnvVars + } = getCiVisAgentlessConfig(receiver.port) + + childProcess = exec( + testCommand, + { + cwd, + env: { + ...restEnvVars, + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: 'cypress/e2e/spec.cy.js' + }, + stdio: 'pipe' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + }) + + it('still reports correct format if there is a plugin incompatibility', (done) => { + const { + NODE_OPTIONS, // NODE_OPTIONS dd-trace config does not work with cypress + ...restEnvVars + } = getCiVisEvpProxyConfig(receiver.port) + + const receiverPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testEvents = events.filter(event => event.type === 'test') + const testModuleEvent = events.find(event => event.type === 'test_module_end') + + testEvents.forEach(testEvent => { + assert.exists(testEvent.content.test_suite_id) + assert.exists(testEvent.content.test_module_id) + assert.exists(testEvent.content.test_session_id) + assert.notEqual(testEvent.content.test_suite_id, testModuleEvent.content.test_module_id) + }) + }) + + childProcess = exec( + testCommand, + { + cwd, + env: { + ...restEnvVars, + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + CYPRESS_ENABLE_INCOMPATIBLE_PLUGIN: '1', + SPEC_PATTERN: 'cypress/e2e/spec.cy.js' + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + receiverPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('works if after:run is explicitly used', (done) => { + const receiverPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSessionEvent = events.find(event => event.type === 'test_session_end') + assert.exists(testSessionEvent) + const testModuleEvent = events.find(event => event.type === 'test_module_end') + assert.exists(testModuleEvent) + const testSuiteEvents = events.filter(event => event.type === 'test_suite_end') + assert.equal(testSuiteEvents.length, 4) + const testEvents = events.filter(event => event.type === 'test') + assert.equal(testEvents.length, 9) + }, 30000) + + const { + NODE_OPTIONS, // NODE_OPTIONS dd-trace config does not work with cypress + ...restEnvVars + } = getCiVisEvpProxyConfig(receiver.port) + + childProcess = exec( + testCommand, + { + cwd, + env: { + ...restEnvVars, + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + CYPRESS_ENABLE_AFTER_RUN_CUSTOM: '1' + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + receiverPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('works if after:spec is explicitly used', (done) => { + const receiverPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSessionEvent = events.find(event => event.type === 'test_session_end') + assert.exists(testSessionEvent) + const testModuleEvent = events.find(event => event.type === 'test_module_end') + assert.exists(testModuleEvent) + const testSuiteEvents = events.filter(event => event.type === 'test_suite_end') + assert.equal(testSuiteEvents.length, 4) + const testEvents = events.filter(event => event.type === 'test') + assert.equal(testEvents.length, 9) + }, 30000) + + const { + NODE_OPTIONS, // NODE_OPTIONS dd-trace config does not work with cypress + ...restEnvVars + } = getCiVisEvpProxyConfig(receiver.port) + + childProcess = exec( + testCommand, + { + cwd, + env: { + ...restEnvVars, + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + CYPRESS_ENABLE_AFTER_SPEC_CUSTOM: '1' + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + receiverPromise.then(() => { + done() + }).catch(done) + }) + }) + + context('early flake detection', () => { + it('retries new tests', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } + } + }) + + receiver.setKnownTests({ + cypress: { + 'cypress/e2e/spec.cy.js': [ + // 'context passes', // This test will be considered new + 'other context fails' + ] + } + }) + + const receiverPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + assert.equal(tests.length, 5) + + const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') + assert.equal(newTests.length, NUM_RETRIES_EFD + 1) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, NUM_RETRIES_EFD) + + newTests.forEach(newTest => { + assert.equal(newTest.resource, 'cypress/e2e/spec.cy.js.context passes') + }) + + const knownTest = tests.filter(test => !test.meta[TEST_IS_NEW]) + assert.equal(knownTest.length, 1) + assert.equal(knownTest[0].resource, 'cypress/e2e/spec.cy.js.other context fails') + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + }) + + const { + NODE_OPTIONS, // NODE_OPTIONS dd-trace config does not work with cypress + ...restEnvVars + } = getCiVisEvpProxyConfig(receiver.port) + + const specToRun = 'cypress/e2e/spec.cy.js' + + childProcess = exec( + version === 'latest' ? testCommand : `${testCommand} --spec ${specToRun}`, + { + cwd, + env: { + ...restEnvVars, + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: specToRun + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + receiverPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('is disabled if DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED is false', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } + } + }) + + receiver.setKnownTests({ + cypress: { + 'cypress/e2e/spec.cy.js': [ + // 'context passes', // This test will be considered new + 'other context fails' + ] + } + }) + + const { + NODE_OPTIONS, // NODE_OPTIONS dd-trace config does not work with cypress + ...restEnvVars + } = getCiVisEvpProxyConfig(receiver.port) + + const receiverPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + assert.equal(tests.length, 2) + + const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') + assert.equal(newTests.length, 0) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + }) + + const specToRun = 'cypress/e2e/spec.cy.js' + childProcess = exec( + version === 'latest' ? testCommand : `${testCommand} --spec ${specToRun}`, + { + cwd, + env: { + ...restEnvVars, + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: specToRun, + DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED: 'false' + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + receiverPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('does not retry tests that are skipped', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } + } + }) + + receiver.setKnownTests({}) + const { + NODE_OPTIONS, // NODE_OPTIONS dd-trace config does not work with cypress + ...restEnvVars + } = getCiVisEvpProxyConfig(receiver.port) + + const receiverPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + assert.equal(tests.length, 1) + + const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') + assert.equal(newTests.length, 0) + + assert.equal(tests[0].resource, 'cypress/e2e/skipped-test.js.skipped skipped') + assert.propertyVal(tests[0].meta, TEST_STATUS, 'skip') + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + }) + + const specToRun = 'cypress/e2e/skipped-test.js' + + childProcess = exec( + version === 'latest' ? testCommand : `${testCommand} --spec ${specToRun}`, + { + cwd, + env: { + ...restEnvVars, + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: 'cypress/e2e/skipped-test.js' + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + receiverPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('does not run EFD if the known tests request fails', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } + } + }) + + receiver.setKnownTestsResponseCode(500) + receiver.setKnownTests({}) + + const { + NODE_OPTIONS, // NODE_OPTIONS dd-trace config does not work with cypress + ...restEnvVars + } = getCiVisEvpProxyConfig(receiver.port) + + const receiverPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + assert.equal(tests.length, 2) + + const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') + assert.equal(newTests.length, 0) + }) + + const specToRun = 'cypress/e2e/spec.cy.js' + + childProcess = exec( + version === 'latest' ? testCommand : `${testCommand} --spec ${specToRun}`, + { + cwd, + env: { + ...restEnvVars, + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: specToRun + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + receiverPromise.then(() => { + done() + }).catch(done) + }) + }) + }) + + context('flaky test retries', () => { + it('retries flaky tests', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + flaky_test_retries_enabled: true, + early_flake_detection: { + enabled: false + } + }) + + const receiverPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSuites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) + assert.equal(testSuites.length, 1) + assert.equal(testSuites[0].meta[TEST_STATUS], 'fail') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + assert.equal(tests.length, 10) + + assert.includeMembers(tests.map(test => test.resource), [ + 'cypress/e2e/flaky-test-retries.js.flaky test retry eventually passes', + 'cypress/e2e/flaky-test-retries.js.flaky test retry eventually passes', + // passes at the second retry + 'cypress/e2e/flaky-test-retries.js.flaky test retry eventually passes', + 'cypress/e2e/flaky-test-retries.js.flaky test retry never passes', + 'cypress/e2e/flaky-test-retries.js.flaky test retry never passes', + 'cypress/e2e/flaky-test-retries.js.flaky test retry never passes', + 'cypress/e2e/flaky-test-retries.js.flaky test retry never passes', + 'cypress/e2e/flaky-test-retries.js.flaky test retry never passes', + // never passes + 'cypress/e2e/flaky-test-retries.js.flaky test retry never passes', + // passes on the first try + 'cypress/e2e/flaky-test-retries.js.flaky test retry always passes' + ]) + + const eventuallyPassingTest = tests.filter( + test => test.resource === 'cypress/e2e/flaky-test-retries.js.flaky test retry eventually passes' + ) + assert.equal(eventuallyPassingTest.length, 3) + assert.equal(eventuallyPassingTest.filter(test => test.meta[TEST_STATUS] === 'fail').length, 2) + assert.equal(eventuallyPassingTest.filter(test => test.meta[TEST_STATUS] === 'pass').length, 1) + assert.equal(eventuallyPassingTest.filter(test => test.meta[TEST_IS_RETRY] === 'true').length, 2) + + const neverPassingTest = tests.filter( + test => test.resource === 'cypress/e2e/flaky-test-retries.js.flaky test retry never passes' + ) + assert.equal(neverPassingTest.length, 6) + assert.equal(neverPassingTest.filter(test => test.meta[TEST_STATUS] === 'fail').length, 6) + assert.equal(neverPassingTest.filter(test => test.meta[TEST_STATUS] === 'pass').length, 0) + assert.equal(neverPassingTest.filter(test => test.meta[TEST_IS_RETRY] === 'true').length, 5) + }) + + const { + NODE_OPTIONS, // NODE_OPTIONS dd-trace config does not work with cypress + ...restEnvVars + } = getCiVisEvpProxyConfig(receiver.port) + + const specToRun = 'cypress/e2e/flaky-test-retries.js' + + childProcess = exec( + version === 'latest' ? testCommand : `${testCommand} --spec ${specToRun}`, + { + cwd, + env: { + ...restEnvVars, + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: specToRun + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + receiverPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('is disabled if DD_CIVISIBILITY_FLAKY_RETRY_ENABLED is false', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + flaky_test_retries_enabled: true, + early_flake_detection: { + enabled: false + } + }) + + const receiverPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSuites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) + assert.equal(testSuites.length, 1) + assert.equal(testSuites[0].meta[TEST_STATUS], 'fail') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + assert.equal(tests.length, 3) + + assert.includeMembers(tests.map(test => test.resource), [ + 'cypress/e2e/flaky-test-retries.js.flaky test retry eventually passes', + 'cypress/e2e/flaky-test-retries.js.flaky test retry never passes', + 'cypress/e2e/flaky-test-retries.js.flaky test retry always passes' + ]) + assert.equal(tests.filter(test => test.meta[TEST_IS_RETRY] === 'true').length, 0) + }) + + const { + NODE_OPTIONS, // NODE_OPTIONS dd-trace config does not work with cypress + ...restEnvVars + } = getCiVisEvpProxyConfig(receiver.port) + + const specToRun = 'cypress/e2e/flaky-test-retries.js' + + childProcess = exec( + version === 'latest' ? testCommand : `${testCommand} --spec ${specToRun}`, + { + cwd, + env: { + ...restEnvVars, + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + DD_CIVISIBILITY_FLAKY_RETRY_ENABLED: 'false', + SPEC_PATTERN: specToRun + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + receiverPromise.then(() => done()).catch(done) + }) + }) + + it('retries DD_CIVISIBILITY_FLAKY_RETRY_COUNT times', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + flaky_test_retries_enabled: true, + early_flake_detection: { + enabled: false + } + }) + + const receiverPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSuites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) + assert.equal(testSuites.length, 1) + assert.equal(testSuites[0].meta[TEST_STATUS], 'fail') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + assert.equal(tests.length, 5) + + assert.includeMembers(tests.map(test => test.resource), [ + 'cypress/e2e/flaky-test-retries.js.flaky test retry eventually passes', + 'cypress/e2e/flaky-test-retries.js.flaky test retry eventually passes', + 'cypress/e2e/flaky-test-retries.js.flaky test retry never passes', + 'cypress/e2e/flaky-test-retries.js.flaky test retry never passes', + 'cypress/e2e/flaky-test-retries.js.flaky test retry always passes' + ]) + + assert.equal(tests.filter(test => test.meta[TEST_IS_RETRY] === 'true').length, 2) + }) + + const { + NODE_OPTIONS, // NODE_OPTIONS dd-trace config does not work with cypress + ...restEnvVars + } = getCiVisEvpProxyConfig(receiver.port) + + const specToRun = 'cypress/e2e/flaky-test-retries.js' + + childProcess = exec( + version === 'latest' ? testCommand : `${testCommand} --spec ${specToRun}`, + { + cwd, + env: { + ...restEnvVars, + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + DD_CIVISIBILITY_FLAKY_RETRY_COUNT: 1, + SPEC_PATTERN: specToRun + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + receiverPromise.then(() => { + done() + }).catch(done) + }) + }) + }) + + it('correctly calculates test code owners when working directory is not repository root', (done) => { + let command + + if (type === 'commonJS') { + const commandSuffix = version === '6.7.0' + ? '--config-file cypress-config.json --spec "cypress/e2e/*.cy.js"' + : '' + command = `../../node_modules/.bin/cypress run ${commandSuffix}` + } else { + command = `node --loader=${hookFile} ../../cypress-esm-config.mjs` + } + + const { + NODE_OPTIONS, // NODE_OPTIONS dd-trace config does not work with cypress + ...restEnvVars + } = getCiVisAgentlessConfig(receiver.port) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const test = events.find(event => event.type === 'test').content + const testSuite = events.find(event => event.type === 'test_suite_end').content + // The test is in a subproject + assert.notEqual(test.meta[TEST_SOURCE_FILE], test.meta[TEST_SUITE]) + assert.equal(test.meta[TEST_CODE_OWNERS], JSON.stringify(['@datadog-dd-trace-js'])) + assert.equal(testSuite.meta[TEST_CODE_OWNERS], JSON.stringify(['@datadog-dd-trace-js'])) + }, 25000) + + childProcess = exec( + command, + { + cwd: `${cwd}/ci-visibility/subproject`, + env: { + ...restEnvVars, + CYPRESS_BASE_URL: `http://localhost:${webAppPort}` + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) }) }) }) diff --git a/integration-tests/cypress/e2e/flaky-test-retries.js b/integration-tests/cypress/e2e/flaky-test-retries.js new file mode 100644 index 00000000000..f919683581c --- /dev/null +++ b/integration-tests/cypress/e2e/flaky-test-retries.js @@ -0,0 +1,19 @@ +/* eslint-disable */ +describe('flaky test retry', () => { + let numAttempt = 0 + it('eventually passes', () => { + cy.visit('/') + .get('.hello-world') + .should('have.text', numAttempt++ === 2 ? 'Hello World' : 'Hello Warld') + }) + it('never passes', () => { + cy.visit('/') + .get('.hello-world') + .should('have.text', 'Hello Warld') + }) + it('always passes', () => { + cy.visit('/') + .get('.hello-world') + .should('have.text', 'Hello World') + }) +}) diff --git a/integration-tests/cypress/e2e/skipped-test.js b/integration-tests/cypress/e2e/skipped-test.js new file mode 100644 index 00000000000..9ffadd09259 --- /dev/null +++ b/integration-tests/cypress/e2e/skipped-test.js @@ -0,0 +1,8 @@ +/* eslint-disable */ +describe('skipped', () => { + it.skip('skipped', () => { + cy.visit('/') + .get('.hello-world') + .should('have.text', 'Hello World') + }) +}) diff --git a/integration-tests/cypress/plugins-old/index.js b/integration-tests/cypress/plugins-old/index.js index 90ac1da11d4..66de6be80fe 100644 --- a/integration-tests/cypress/plugins-old/index.js +++ b/integration-tests/cypress/plugins-old/index.js @@ -1,3 +1,26 @@ +const ddAfterRun = require('dd-trace/ci/cypress/after-run') +const ddAfterSpec = require('dd-trace/ci/cypress/after-spec') + module.exports = (on, config) => { - require('dd-trace/ci/cypress/plugin')(on, config) + if (process.env.CYPRESS_ENABLE_INCOMPATIBLE_PLUGIN) { + require('cypress-fail-fast/plugin')(on, config) + } + if (process.env.SPEC_PATTERN) { + config.testFiles = process.env.SPEC_PATTERN.replace('cypress/e2e/', '') + } + if (process.env.CYPRESS_ENABLE_AFTER_RUN_CUSTOM) { + on('after:run', (...args) => { + // do custom stuff + // and call after-run at the end + return ddAfterRun(...args) + }) + } + if (process.env.CYPRESS_ENABLE_AFTER_SPEC_CUSTOM) { + on('after:spec', (...args) => { + // do custom stuff + // and call after-spec at the end + return ddAfterSpec(...args) + }) + } + return require('dd-trace/ci/cypress/plugin')(on, config) } diff --git a/integration-tests/cypress/support/e2e.js b/integration-tests/cypress/support/e2e.js index 19b738f9476..db40f9db820 100644 --- a/integration-tests/cypress/support/e2e.js +++ b/integration-tests/cypress/support/e2e.js @@ -1 +1,5 @@ -import 'dd-trace/ci/cypress/support' +// eslint-disable-next-line +if (Cypress.env('ENABLE_INCOMPATIBLE_PLUGIN')) { + require('cypress-fail-fast') +} +require('dd-trace/ci/cypress/support') diff --git a/integration-tests/debugger/index.spec.js b/integration-tests/debugger/index.spec.js new file mode 100644 index 00000000000..8670ba82b47 --- /dev/null +++ b/integration-tests/debugger/index.spec.js @@ -0,0 +1,654 @@ +'use strict' + +const path = require('path') +const { randomUUID } = require('crypto') +const os = require('os') + +const getPort = require('get-port') +const Axios = require('axios') +const { assert } = require('chai') +const { assertObjectContains, assertUUID, createSandbox, FakeAgent, spawnProc } = require('../helpers') +const { ACKNOWLEDGED, ERROR } = require('../../packages/dd-trace/src/appsec/remote_config/apply_states') +const { version } = require('../../package.json') + +const probeFile = 'debugger/target-app/index.js' +const probeLineNo = 14 +const pollInterval = 1 + +describe('Dynamic Instrumentation', function () { + let axios, sandbox, cwd, appPort, appFile, agent, proc, rcConfig + + before(async function () { + sandbox = await createSandbox(['fastify']) + cwd = sandbox.folder + appFile = path.join(cwd, ...probeFile.split('/')) + }) + + after(async function () { + await sandbox.remove() + }) + + beforeEach(async function () { + rcConfig = generateRemoteConfig() + appPort = await getPort() + agent = await new FakeAgent().start() + proc = await spawnProc(appFile, { + cwd, + env: { + APP_PORT: appPort, + DD_DYNAMIC_INSTRUMENTATION_ENABLED: true, + DD_TRACE_AGENT_PORT: agent.port, + DD_TRACE_DEBUG: process.env.DD_TRACE_DEBUG, // inherit to make debugging the sandbox easier + DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS: pollInterval + } + }) + axios = Axios.create({ + baseURL: `http://localhost:${appPort}` + }) + }) + + afterEach(async function () { + proc.kill() + await agent.stop() + }) + + it('base case: target app should work as expected if no test probe has been added', async function () { + const response = await axios.get('/foo') + assert.strictEqual(response.status, 200) + assert.deepStrictEqual(response.data, { hello: 'foo' }) + }) + + describe('diagnostics messages', function () { + it('should send expected diagnostics messages if probe is received and triggered', function (done) { + let receivedAckUpdate = false + const probeId = rcConfig.config.id + const expectedPayloads = [{ + ddsource: 'dd_debugger', + service: 'node', + debugger: { diagnostics: { probeId, version: 0, status: 'RECEIVED' } } + }, { + ddsource: 'dd_debugger', + service: 'node', + debugger: { diagnostics: { probeId, version: 0, status: 'INSTALLED' } } + }, { + ddsource: 'dd_debugger', + service: 'node', + debugger: { diagnostics: { probeId, version: 0, status: 'EMITTING' } } + }] + + agent.on('remote-config-ack-update', (id, version, state, error) => { + assert.strictEqual(id, rcConfig.id) + assert.strictEqual(version, 1) + assert.strictEqual(state, ACKNOWLEDGED) + assert.notOk(error) // falsy check since error will be an empty string, but that's an implementation detail + + receivedAckUpdate = true + endIfDone() + }) + + agent.on('debugger-diagnostics', ({ payload }) => { + const expected = expectedPayloads.shift() + assertObjectContains(payload, expected) + assertUUID(payload.debugger.diagnostics.runtimeId) + + if (payload.debugger.diagnostics.status === 'INSTALLED') { + axios.get('/foo') + .then((response) => { + assert.strictEqual(response.status, 200) + assert.deepStrictEqual(response.data, { hello: 'foo' }) + }) + .catch(done) + } else { + endIfDone() + } + }) + + agent.addRemoteConfig(rcConfig) + + function endIfDone () { + if (receivedAckUpdate && expectedPayloads.length === 0) done() + } + }) + + it('should send expected diagnostics messages if probe is first received and then updated', function (done) { + let receivedAckUpdates = 0 + const probeId = rcConfig.config.id + const expectedPayloads = [{ + ddsource: 'dd_debugger', + service: 'node', + debugger: { diagnostics: { probeId, version: 0, status: 'RECEIVED' } } + }, { + ddsource: 'dd_debugger', + service: 'node', + debugger: { diagnostics: { probeId, version: 0, status: 'INSTALLED' } } + }, { + ddsource: 'dd_debugger', + service: 'node', + debugger: { diagnostics: { probeId, version: 1, status: 'RECEIVED' } } + }, { + ddsource: 'dd_debugger', + service: 'node', + debugger: { diagnostics: { probeId, version: 1, status: 'INSTALLED' } } + }] + const triggers = [ + () => { + rcConfig.config.version++ + agent.updateRemoteConfig(rcConfig.id, rcConfig.config) + }, + () => {} + ] + + agent.on('remote-config-ack-update', (id, version, state, error) => { + assert.strictEqual(id, rcConfig.id) + assert.strictEqual(version, ++receivedAckUpdates) + assert.strictEqual(state, ACKNOWLEDGED) + assert.notOk(error) // falsy check since error will be an empty string, but that's an implementation detail + + endIfDone() + }) + + agent.on('debugger-diagnostics', ({ payload }) => { + const expected = expectedPayloads.shift() + assertObjectContains(payload, expected) + assertUUID(payload.debugger.diagnostics.runtimeId) + if (payload.debugger.diagnostics.status === 'INSTALLED') triggers.shift()() + endIfDone() + }) + + agent.addRemoteConfig(rcConfig) + + function endIfDone () { + if (receivedAckUpdates === 2 && expectedPayloads.length === 0) done() + } + }) + + it('should send expected diagnostics messages if probe is first received and then deleted', function (done) { + let receivedAckUpdate = false + let payloadsProcessed = false + const probeId = rcConfig.config.id + const expectedPayloads = [{ + ddsource: 'dd_debugger', + service: 'node', + debugger: { diagnostics: { probeId, version: 0, status: 'RECEIVED' } } + }, { + ddsource: 'dd_debugger', + service: 'node', + debugger: { diagnostics: { probeId, version: 0, status: 'INSTALLED' } } + }] + + agent.on('remote-config-ack-update', (id, version, state, error) => { + assert.strictEqual(id, rcConfig.id) + assert.strictEqual(version, 1) + assert.strictEqual(state, ACKNOWLEDGED) + assert.notOk(error) // falsy check since error will be an empty string, but that's an implementation detail + + receivedAckUpdate = true + endIfDone() + }) + + agent.on('debugger-diagnostics', ({ payload }) => { + const expected = expectedPayloads.shift() + assertObjectContains(payload, expected) + assertUUID(payload.debugger.diagnostics.runtimeId) + + if (payload.debugger.diagnostics.status === 'INSTALLED') { + agent.removeRemoteConfig(rcConfig.id) + // Wait a little to see if we get any follow-up `debugger-diagnostics` messages + setTimeout(() => { + payloadsProcessed = true + endIfDone() + }, pollInterval * 2 * 1000) // wait twice as long as the RC poll interval + } + }) + + agent.addRemoteConfig(rcConfig) + + function endIfDone () { + if (receivedAckUpdate && payloadsProcessed) done() + } + }) + + const unsupporedOrInvalidProbes = [[ + 'should send expected error diagnostics messages if probe doesn\'t conform to expected schema', + 'bad config!!!', + { status: 'ERROR' } + ], [ + 'should send expected error diagnostics messages if probe type isn\'t supported', + generateProbeConfig({ type: 'INVALID_PROBE' }) + ], [ + 'should send expected error diagnostics messages if it isn\'t a line-probe', + generateProbeConfig({ where: { foo: 'bar' } }) // TODO: Use valid schema for method probe instead + ]] + + for (const [title, config, customErrorDiagnosticsObj] of unsupporedOrInvalidProbes) { + it(title, function (done) { + let receivedAckUpdate = false + + agent.on('remote-config-ack-update', (id, version, state, error) => { + assert.strictEqual(id, `logProbe_${config.id}`) + assert.strictEqual(version, 1) + assert.strictEqual(state, ERROR) + assert.strictEqual(error.slice(0, 6), 'Error:') + + receivedAckUpdate = true + endIfDone() + }) + + const probeId = config.id + const expectedPayloads = [{ + ddsource: 'dd_debugger', + service: 'node', + debugger: { diagnostics: { status: 'RECEIVED' } } + }, { + ddsource: 'dd_debugger', + service: 'node', + debugger: { diagnostics: customErrorDiagnosticsObj ?? { probeId, version: 0, status: 'ERROR' } } + }] + + agent.on('debugger-diagnostics', ({ payload }) => { + const expected = expectedPayloads.shift() + assertObjectContains(payload, expected) + const { diagnostics } = payload.debugger + assertUUID(diagnostics.runtimeId) + + if (diagnostics.status === 'ERROR') { + assert.property(diagnostics, 'exception') + assert.hasAllKeys(diagnostics.exception, ['message', 'stacktrace']) + assert.typeOf(diagnostics.exception.message, 'string') + assert.typeOf(diagnostics.exception.stacktrace, 'string') + } + + endIfDone() + }) + + agent.addRemoteConfig({ + product: 'LIVE_DEBUGGING', + id: `logProbe_${config.id}`, + config + }) + + function endIfDone () { + if (receivedAckUpdate && expectedPayloads.length === 0) done() + } + }) + } + }) + + describe('input messages', function () { + it('should capture and send expected payload when a log line probe is triggered', function (done) { + agent.on('debugger-diagnostics', ({ payload }) => { + if (payload.debugger.diagnostics.status === 'INSTALLED') { + axios.get('/foo') + } + }) + + agent.on('debugger-input', ({ payload }) => { + const expected = { + ddsource: 'dd_debugger', + hostname: os.hostname(), + service: 'node', + message: 'Hello World!', + logger: { + name: 'debugger/target-app/index.js', + method: 'handler', + version, + thread_name: 'MainThread' + }, + 'debugger.snapshot': { + probe: { + id: rcConfig.config.id, + version: 0, + location: { file: probeFile, lines: [String(probeLineNo)] } + }, + language: 'javascript' + } + } + + assertObjectContains(payload, expected) + assert.match(payload.logger.thread_id, /^pid:\d+$/) + assertUUID(payload['debugger.snapshot'].id) + assert.isNumber(payload['debugger.snapshot'].timestamp) + assert.isTrue(payload['debugger.snapshot'].timestamp > Date.now() - 1000 * 60) + assert.isTrue(payload['debugger.snapshot'].timestamp <= Date.now()) + + assert.isArray(payload['debugger.snapshot'].stack) + assert.isAbove(payload['debugger.snapshot'].stack.length, 0) + for (const frame of payload['debugger.snapshot'].stack) { + assert.isObject(frame) + assert.hasAllKeys(frame, ['fileName', 'function', 'lineNumber', 'columnNumber']) + assert.isString(frame.fileName) + assert.isString(frame.function) + assert.isAbove(frame.lineNumber, 0) + assert.isAbove(frame.columnNumber, 0) + } + const topFrame = payload['debugger.snapshot'].stack[0] + assert.match(topFrame.fileName, new RegExp(`${appFile}$`)) // path seems to be prefeixed with `/private` on Mac + assert.strictEqual(topFrame.function, 'handler') + assert.strictEqual(topFrame.lineNumber, probeLineNo) + assert.strictEqual(topFrame.columnNumber, 3) + + done() + }) + + agent.addRemoteConfig(rcConfig) + }) + + it('should respond with updated message if probe message is updated', function (done) { + const expectedMessages = ['Hello World!', 'Hello Updated World!'] + const triggers = [ + async () => { + await axios.get('/foo') + rcConfig.config.version++ + rcConfig.config.template = 'Hello Updated World!' + agent.updateRemoteConfig(rcConfig.id, rcConfig.config) + }, + async () => { + await axios.get('/foo') + } + ] + + agent.on('debugger-diagnostics', ({ payload }) => { + if (payload.debugger.diagnostics.status === 'INSTALLED') triggers.shift()().catch(done) + }) + + agent.on('debugger-input', ({ payload }) => { + assert.strictEqual(payload.message, expectedMessages.shift()) + if (expectedMessages.length === 0) done() + }) + + agent.addRemoteConfig(rcConfig) + }) + + it('should not trigger if probe is deleted', function (done) { + agent.on('debugger-diagnostics', async ({ payload }) => { + try { + if (payload.debugger.diagnostics.status === 'INSTALLED') { + agent.once('remote-confg-responded', async () => { + try { + await axios.get('/foo') + // We want to wait enough time to see if the client triggers on the breakpoint so that the test can fail + // if it does, but not so long that the test times out. + // TODO: Is there some signal we can use instead of a timer? + setTimeout(done, pollInterval * 2 * 1000) // wait twice as long as the RC poll interval + } catch (err) { + // Nessecary hack: Any errors thrown inside of an async function is invisible to Mocha unless the outer + // `it` callback is also `async` (which we can't do in this case since we rely on the `done` callback). + done(err) + } + }) + + agent.removeRemoteConfig(rcConfig.id) + } + } catch (err) { + // Nessecary hack: Any errors thrown inside of an async function is invisible to Mocha unless the outer `it` + // callback is also `async` (which we can't do in this case since we rely on the `done` callback). + done(err) + } + }) + + agent.on('debugger-input', () => { + assert.fail('should not capture anything when the probe is deleted') + }) + + agent.addRemoteConfig(rcConfig) + }) + + describe('with snapshot', () => { + beforeEach(() => { + // Trigger the breakpoint once probe is successfully installed + agent.on('debugger-diagnostics', ({ payload }) => { + if (payload.debugger.diagnostics.status === 'INSTALLED') { + axios.get('/foo') + } + }) + }) + + it('should capture a snapshot', (done) => { + agent.on('debugger-input', ({ payload: { 'debugger.snapshot': { captures } } }) => { + assert.deepEqual(Object.keys(captures), ['lines']) + assert.deepEqual(Object.keys(captures.lines), [String(probeLineNo)]) + + const { locals } = captures.lines[probeLineNo] + const { request, fastify, getSomeData } = locals + delete locals.request + delete locals.fastify + delete locals.getSomeData + + // from block scope + assert.deepEqual(locals, { + nil: { type: 'null', isNull: true }, + undef: { type: 'undefined' }, + bool: { type: 'boolean', value: 'true' }, + num: { type: 'number', value: '42' }, + bigint: { type: 'bigint', value: '42' }, + str: { type: 'string', value: 'foo' }, + lstr: { + type: 'string', + // eslint-disable-next-line max-len + value: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor i', + truncated: true, + size: 445 + }, + sym: { type: 'symbol', value: 'Symbol(foo)' }, + regex: { type: 'RegExp', value: '/bar/i' }, + arr: { + type: 'Array', + elements: [ + { type: 'number', value: '1' }, + { type: 'number', value: '2' }, + { type: 'number', value: '3' }, + { type: 'number', value: '4' }, + { type: 'number', value: '5' } + ] + }, + obj: { + type: 'Object', + fields: { + foo: { + type: 'Object', + fields: { + baz: { type: 'number', value: '42' }, + nil: { type: 'null', isNull: true }, + undef: { type: 'undefined' }, + deep: { + type: 'Object', + fields: { nested: { type: 'Object', notCapturedReason: 'depth' } } + } + } + }, + bar: { type: 'boolean', value: 'true' } + } + }, + emptyObj: { type: 'Object', fields: {} }, + fn: { + type: 'Function', + fields: { + length: { type: 'number', value: '0' }, + name: { type: 'string', value: 'fn' } + } + }, + p: { + type: 'Promise', + fields: { + '[[PromiseState]]': { type: 'string', value: 'fulfilled' }, + '[[PromiseResult]]': { type: 'undefined' } + } + } + }) + + // from local scope + // There's no reason to test the `request` object 100%, instead just check its fingerprint + assert.deepEqual(Object.keys(request), ['type', 'fields']) + assert.equal(request.type, 'Request') + assert.deepEqual(request.fields.id, { type: 'string', value: 'req-1' }) + assert.deepEqual(request.fields.params, { + type: 'NullObject', fields: { name: { type: 'string', value: 'foo' } } + }) + assert.deepEqual(request.fields.query, { type: 'Object', fields: {} }) + assert.deepEqual(request.fields.body, { type: 'undefined' }) + + // from closure scope + // There's no reason to test the `fastify` object 100%, instead just check its fingerprint + assert.deepEqual(Object.keys(fastify), ['type', 'fields']) + assert.equal(fastify.type, 'Object') + + assert.deepEqual(getSomeData, { + type: 'Function', + fields: { + length: { type: 'number', value: '0' }, + name: { type: 'string', value: 'getSomeData' } + } + }) + + done() + }) + + agent.addRemoteConfig(generateRemoteConfig({ captureSnapshot: true })) + }) + + it('should respect maxReferenceDepth', (done) => { + agent.on('debugger-input', ({ payload: { 'debugger.snapshot': { captures } } }) => { + const { locals } = captures.lines[probeLineNo] + delete locals.request + delete locals.fastify + delete locals.getSomeData + + assert.deepEqual(locals, { + nil: { type: 'null', isNull: true }, + undef: { type: 'undefined' }, + bool: { type: 'boolean', value: 'true' }, + num: { type: 'number', value: '42' }, + bigint: { type: 'bigint', value: '42' }, + str: { type: 'string', value: 'foo' }, + lstr: { + type: 'string', + // eslint-disable-next-line max-len + value: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor i', + truncated: true, + size: 445 + }, + sym: { type: 'symbol', value: 'Symbol(foo)' }, + regex: { type: 'RegExp', value: '/bar/i' }, + arr: { type: 'Array', notCapturedReason: 'depth' }, + obj: { type: 'Object', notCapturedReason: 'depth' }, + emptyObj: { type: 'Object', notCapturedReason: 'depth' }, + fn: { type: 'Function', notCapturedReason: 'depth' }, + p: { type: 'Promise', notCapturedReason: 'depth' } + }) + + done() + }) + + agent.addRemoteConfig(generateRemoteConfig({ captureSnapshot: true, capture: { maxReferenceDepth: 0 } })) + }) + + it('should respect maxLength', (done) => { + agent.on('debugger-input', ({ payload: { 'debugger.snapshot': { captures } } }) => { + const { locals } = captures.lines[probeLineNo] + + assert.deepEqual(locals.lstr, { + type: 'string', + value: 'Lorem ipsu', + truncated: true, + size: 445 + }) + + done() + }) + + agent.addRemoteConfig(generateRemoteConfig({ captureSnapshot: true, capture: { maxLength: 10 } })) + }) + + it('should respect maxCollectionSize', (done) => { + agent.on('debugger-input', ({ payload: { 'debugger.snapshot': { captures } } }) => { + const { locals } = captures.lines[probeLineNo] + + assert.deepEqual(locals.arr, { + type: 'Array', + elements: [ + { type: 'number', value: '1' }, + { type: 'number', value: '2' }, + { type: 'number', value: '3' } + ], + notCapturedReason: 'collectionSize', + size: 5 + }) + + done() + }) + + agent.addRemoteConfig(generateRemoteConfig({ captureSnapshot: true, capture: { maxCollectionSize: 3 } })) + }) + }) + }) + + describe('race conditions', () => { + it('should remove the last breakpoint completely before trying to add a new one', (done) => { + const rcConfig2 = generateRemoteConfig() + + agent.on('debugger-diagnostics', ({ payload: { debugger: { diagnostics: { status, probeId } } } }) => { + if (status !== 'INSTALLED') return + + if (probeId === rcConfig.config.id) { + // First INSTALLED payload: Try to trigger the race condition. + agent.removeRemoteConfig(rcConfig.id) + agent.addRemoteConfig(rcConfig2) + } else { + // Second INSTALLED payload: Perform an HTTP request to see if we successfully handled the race condition. + let finished = false + + // If the race condition occurred, the debugger will have been detached from the main thread and the new + // probe will never trigger. If that's the case, the following timer will fire: + const timer = setTimeout(() => { + done(new Error('Race condition occurred!')) + }, 1000) + + // If we successfully handled the race condition, the probe will trigger, we'll get a probe result and the + // following event listener will be called: + agent.once('debugger-input', () => { + clearTimeout(timer) + finished = true + done() + }) + + // Perform HTTP request to try and trigger the probe + axios.get('/foo').catch((err) => { + // If the request hasn't fully completed by the time the tests ends and the target app is destroyed, Axios + // will complain with a "socket hang up" error. Hence this sanity check before calling `done(err)`. If we + // later add more tests below this one, this shouuldn't be an issue. + if (!finished) done(err) + }) + } + }) + + agent.addRemoteConfig(rcConfig) + }) + }) +}) + +function generateRemoteConfig (overrides = {}) { + overrides.id = overrides.id || randomUUID() + return { + product: 'LIVE_DEBUGGING', + id: `logProbe_${overrides.id}`, + config: generateProbeConfig(overrides) + } +} + +function generateProbeConfig (overrides = {}) { + overrides.capture = { maxReferenceDepth: 3, ...overrides.capture } + overrides.sampling = { snapshotsPerSecond: 5000, ...overrides.sampling } + return { + id: randomUUID(), + version: 0, + type: 'LOG_PROBE', + language: 'javascript', + where: { sourceFile: probeFile, lines: [String(probeLineNo)] }, + tags: [], + template: 'Hello World!', + segments: [{ str: 'Hello World!' }], + captureSnapshot: false, + evaluateAt: 'EXIT', + ...overrides + } +} diff --git a/integration-tests/debugger/target-app/index.js b/integration-tests/debugger/target-app/index.js new file mode 100644 index 00000000000..75b8f551a7a --- /dev/null +++ b/integration-tests/debugger/target-app/index.js @@ -0,0 +1,53 @@ +'use strict' + +require('dd-trace/init') +const Fastify = require('fastify') + +const fastify = Fastify() + +// Since line probes have hardcoded line numbers, we want to try and keep the line numbers from changing within the +// `handler` function below when making changes to this file. This is achieved by calling `getSomeData` and keeping all +// variable names on the same line as much as possible. +fastify.get('/:name', function handler (request) { + // eslint-disable-next-line no-unused-vars + const { nil, undef, bool, num, bigint, str, lstr, sym, regex, arr, obj, emptyObj, fn, p } = getSomeData() + return { hello: request.params.name } +}) + +// WARNING: Breakpoints present above this line - Any changes to the lines above might influence tests! + +fastify.listen({ port: process.env.APP_PORT }, (err) => { + if (err) { + fastify.log.error(err) + process.exit(1) + } + process.send({ port: process.env.APP_PORT }) +}) + +function getSomeData () { + return { + nil: null, + undef: undefined, + bool: true, + num: 42, + bigint: 42n, + str: 'foo', + // eslint-disable-next-line max-len + lstr: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.', + sym: Symbol('foo'), + regex: /bar/i, + arr: [1, 2, 3, 4, 5], + obj: { + foo: { + baz: 42, + nil: null, + undef: undefined, + deep: { nested: { obj: { that: { goes: { on: { forever: true } } } } } } + }, + bar: true + }, + emptyObj: {}, + fn: () => {}, + p: Promise.resolve() + } +} diff --git a/integration-tests/esbuild/aws-sdk.js b/integration-tests/esbuild/aws-sdk.js index 9f8a63ab8ae..c89f570689b 100644 --- a/integration-tests/esbuild/aws-sdk.js +++ b/integration-tests/esbuild/aws-sdk.js @@ -2,4 +2,4 @@ require('../../').init() // dd-trace const aws = require('aws-sdk') -void aws.util.inherit +global.test = aws.util.inherit diff --git a/integration-tests/esbuild/basic-test.js b/integration-tests/esbuild/basic-test.js index 20f53708b44..5e95234eddf 100755 --- a/integration-tests/esbuild/basic-test.js +++ b/integration-tests/esbuild/basic-test.js @@ -1,17 +1,12 @@ #!/usr/bin/env node -// TODO: add support for Node.js v14.17+ and v16.0+ -if (Number(process.versions.node.split('.')[0]) < 16) { - console.error(`Skip esbuild test for node@${process.version}`) // eslint-disable-line no-console - process.exit(0) -} - const tracer = require('../../').init() // dd-trace const assert = require('assert') const express = require('express') const http = require('http') require('knex') // has dead code paths for multiple instrumented packages +require('@apollo/server') const app = express() const PORT = 31415 diff --git a/integration-tests/esbuild/build-and-test-aws-sdk.js b/integration-tests/esbuild/build-and-test-aws-sdk.js index c9d46a5da78..a270d6948fc 100755 --- a/integration-tests/esbuild/build-and-test-aws-sdk.js +++ b/integration-tests/esbuild/build-and-test-aws-sdk.js @@ -13,8 +13,8 @@ esbuild.build({ outfile: SCRIPT, plugins: [ddPlugin], platform: 'node', - target: ['node16'], - external: [ ] + target: ['node18'], + external: [] }).then(() => { const { status, stdout, stderr } = spawnSync('node', [SCRIPT]) if (stdout.length) { diff --git a/integration-tests/esbuild/build-and-test-skip-external.js b/integration-tests/esbuild/build-and-test-skip-external.js index 51da8597ff4..b7a35d6026b 100755 --- a/integration-tests/esbuild/build-and-test-skip-external.js +++ b/integration-tests/esbuild/build-and-test-skip-external.js @@ -11,7 +11,7 @@ esbuild.build({ outfile: 'skip-external-out.js', plugins: [ddPlugin], platform: 'node', - target: ['node16'], + target: ['node18'], external: [ 'knex' ] diff --git a/integration-tests/esbuild/build.js b/integration-tests/esbuild/build.js index fa862f279ad..60ba653548f 100755 --- a/integration-tests/esbuild/build.js +++ b/integration-tests/esbuild/build.js @@ -9,7 +9,7 @@ esbuild.build({ outfile: 'out.js', plugins: [ddPlugin], platform: 'node', - target: ['node16'], + target: ['node18'], external: [ // dead code paths introduced by knex 'pg', diff --git a/integration-tests/esbuild/package.json b/integration-tests/esbuild/package.json index d96723cc631..63e8caa8372 100644 --- a/integration-tests/esbuild/package.json +++ b/integration-tests/esbuild/package.json @@ -18,8 +18,9 @@ "author": "Thomas Hunter II ", "license": "ISC", "dependencies": { + "@apollo/server": "^4.11.0", "aws-sdk": "^2.1446.0", - "axios": "^0.21.2", + "axios": "^1.6.7", "esbuild": "0.16.12", "express": "^4.16.2", "knex": "^2.4.2" diff --git a/integration-tests/graphql.spec.js b/integration-tests/graphql.spec.js index 86e846ab460..4a66f163a83 100644 --- a/integration-tests/graphql.spec.js +++ b/integration-tests/graphql.spec.js @@ -15,7 +15,7 @@ describe('graphql', () => { let sandbox, cwd, agent, webFile, proc, appPort before(async function () { - sandbox = await createSandbox([`@apollo/server`, 'graphql', 'koalas']) + sandbox = await createSandbox(['@apollo/server', 'graphql', 'koalas']) cwd = sandbox.folder webFile = path.join(cwd, 'graphql/index.js') appPort = await getPort() @@ -79,7 +79,6 @@ describe('graphql', () => { { id: 'test-rule-id-1', name: 'test-rule-name-1', - on_match: ['block'], tags: { category: 'attack_attempt', @@ -92,8 +91,8 @@ describe('graphql', () => { operator_value: '', parameters: [ { - address: 'graphql.server.all_resolvers', - key_path: ['images', '0', 'category'], + address: 'graphql.server.resolver', + key_path: ['images', 'category'], value: 'testattack', highlight: ['testattack'] } diff --git a/integration-tests/graphql/graphql-rules.json b/integration-tests/graphql/graphql-rules.json index e258dda5226..1073c3d05a2 100644 --- a/integration-tests/graphql/graphql-rules.json +++ b/integration-tests/graphql/graphql-rules.json @@ -17,6 +17,9 @@ "inputs": [ { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "list": [ @@ -27,7 +30,7 @@ } ], "transformers": ["lowercase"], - "on_match": ["block"] + "on_match": [] } ] } diff --git a/integration-tests/helpers.js b/integration-tests/helpers.js deleted file mode 100644 index 4be08654fb3..00000000000 --- a/integration-tests/helpers.js +++ /dev/null @@ -1,310 +0,0 @@ -'use strict' - -const { promisify } = require('util') -const express = require('express') -const bodyParser = require('body-parser') -const msgpack = require('msgpack-lite') -const codec = msgpack.createCodec({ int64: true }) -const EventEmitter = require('events') -const childProcess = require('child_process') -const { fork } = childProcess -const exec = promisify(childProcess.exec) -const http = require('http') -const fs = require('fs') -const mkdir = promisify(fs.mkdir) -const os = require('os') -const path = require('path') -const rimraf = promisify(require('rimraf')) -const id = require('../packages/dd-trace/src/id') -const upload = require('multer')() - -const hookFile = 'dd-trace/loader-hook.mjs' - -class FakeAgent extends EventEmitter { - constructor (port = 0) { - super() - this.port = port - } - - async start () { - const app = express() - app.use(bodyParser.raw({ limit: Infinity, type: 'application/msgpack' })) - app.use(bodyParser.json({ limit: Infinity, type: 'application/json' })) - app.put('/v0.4/traces', (req, res) => { - if (req.body.length === 0) return res.status(200).send() - res.status(200).send({ rate_by_service: { 'service:,env:': 1 } }) - this.emit('message', { - headers: req.headers, - payload: msgpack.decode(req.body, { codec }) - }) - }) - app.post('/profiling/v1/input', upload.any(), (req, res) => { - res.status(200).send() - this.emit('message', { - headers: req.headers, - payload: req.body, - files: req.files - }) - }) - app.post('/telemetry/proxy/api/v2/apmtelemetry', (req, res) => { - res.status(200).send() - this.emit('telemetry', { - headers: req.headers, - payload: req.body - }) - }) - - return new Promise((resolve, reject) => { - const timeoutObj = setTimeout(() => { - reject(new Error('agent timed out starting up')) - }, 10000) - this.server = http.createServer(app) - this.server.on('error', reject) - this.server.listen(this.port, () => { - this.port = this.server.address().port - clearTimeout(timeoutObj) - resolve(this) - }) - }) - } - - stop () { - return new Promise((resolve) => { - this.server.on('close', resolve) - this.server.close() - }) - } - - // **resolveAtFirstSuccess** - specific use case for Next.js (or any other future libraries) - // where multiple payloads are generated, and only one is expected to have the proper span (ie next.request), - // but it't not guaranteed to be the last one (so, expectedMessageCount would not be helpful). - // It can still fail if it takes longer than `timeout` duration or if none pass the assertions (timeout still called) - assertMessageReceived (fn, timeout, expectedMessageCount = 1, resolveAtFirstSuccess) { - timeout = timeout || 5000 - let resultResolve - let resultReject - let msgCount = 0 - const errors = [] - - const timeoutObj = setTimeout(() => { - resultReject([...errors, new Error('timeout')]) - }, timeout) - - const resultPromise = new Promise((resolve, reject) => { - resultResolve = () => { - clearTimeout(timeoutObj) - resolve() - } - resultReject = (e) => { - clearTimeout(timeoutObj) - reject(e) - } - }) - - const messageHandler = msg => { - try { - msgCount += 1 - fn(msg) - if (resolveAtFirstSuccess || msgCount === expectedMessageCount) { - resultResolve() - this.removeListener('message', messageHandler) - } - } catch (e) { - errors.push(e) - } - } - this.on('message', messageHandler) - - return resultPromise - } - - assertTelemetryReceived (fn, timeout, requestType, expectedMessageCount = 1) { - timeout = timeout || 5000 - let resultResolve - let resultReject - let msgCount = 0 - const errors = [] - - const timeoutObj = setTimeout(() => { - resultReject([...errors, new Error('timeout')]) - }, timeout) - - const resultPromise = new Promise((resolve, reject) => { - resultResolve = () => { - clearTimeout(timeoutObj) - resolve() - } - resultReject = (e) => { - clearTimeout(timeoutObj) - reject(e) - } - }) - - const messageHandler = msg => { - if (msg.payload.request_type !== requestType) return - msgCount += 1 - try { - fn(msg) - if (msgCount === expectedMessageCount) { - resultResolve() - } - } catch (e) { - errors.push(e) - } - if (msgCount === expectedMessageCount) { - this.removeListener('telemetry', messageHandler) - } - } - this.on('telemetry', messageHandler) - - return resultPromise - } -} - -function spawnProc (filename, options = {}, stdioHandler) { - const proc = fork(filename, { ...options, stdio: 'pipe' }) - return new Promise((resolve, reject) => { - proc - .on('message', ({ port }) => { - proc.url = `http://localhost:${port}` - resolve(proc) - }) - .on('error', reject) - .on('exit', code => { - if (code !== 0) { - reject(new Error(`Process exited with status code ${code}.`)) - } - resolve() - }) - - proc.stdout.on('data', data => { - if (stdioHandler) { - stdioHandler(data) - } - // eslint-disable-next-line no-console - console.log(data.toString()) - }) - - proc.stderr.on('data', data => { - // eslint-disable-next-line no-console - console.error(data.toString()) - }) - }) -} - -async function createSandbox (dependencies = [], isGitRepo = false, - integrationTestsPaths = ['./integration-tests/*'], followUpCommand) { - /* To execute integration tests without a sandbox uncomment the next line - * and do `yarn link && yarn link dd-trace` */ - // return { folder: path.join(process.cwd(), 'integration-tests'), remove: async () => {} } - const folder = path.join(os.tmpdir(), id().toString()) - const out = path.join(folder, 'dd-trace.tgz') - const allDependencies = [`file:${out}`].concat(dependencies) - - // We might use NODE_OPTIONS to init the tracer. We don't want this to affect this operations - const { NODE_OPTIONS, ...restOfEnv } = process.env - - await mkdir(folder) - await exec(`yarn pack --filename ${out}`) // TODO: cache this - await exec(`yarn add ${allDependencies.join(' ')}`, { cwd: folder, env: restOfEnv }) - - for (const path of integrationTestsPaths) { - await exec(`cp -R ${path} ${folder}`) - await exec(`sync ${folder}`) - } - - if (followUpCommand) { - await exec(followUpCommand, { cwd: folder, env: restOfEnv }) - } - - if (isGitRepo) { - await exec('git init', { cwd: folder }) - await exec('echo "node_modules/" > .gitignore', { cwd: folder }) - await exec('git config user.email "john@doe.com"', { cwd: folder }) - await exec('git config user.name "John Doe"', { cwd: folder }) - await exec('git config commit.gpgsign false', { cwd: folder }) - await exec( - 'git add -A && git commit -m "first commit" --no-verify && git remote add origin git@git.com:datadog/example.git', - { cwd: folder } - ) - } - - return { - folder, - remove: async () => rimraf(folder) - } -} - -async function curl (url, useHttp2 = false) { - if (typeof url === 'object') { - if (url.then) { - return curl(await url) - } - url = url.url - } - - return new Promise((resolve, reject) => { - http.get(url, res => { - const bufs = [] - res.on('data', d => bufs.push(d)) - res.on('end', () => { - res.body = Buffer.concat(bufs).toString('utf8') - resolve(res) - }) - res.on('error', reject) - }).on('error', reject) - }) -} - -async function curlAndAssertMessage (agent, procOrUrl, fn, timeout, expectedMessageCount, resolveAtFirstSuccess) { - const resultPromise = agent.assertMessageReceived(fn, timeout, expectedMessageCount, resolveAtFirstSuccess) - await curl(procOrUrl) - return resultPromise -} - -function getCiVisAgentlessConfig (port) { - return { - ...process.env, - DD_API_KEY: '1', - DD_CIVISIBILITY_AGENTLESS_ENABLED: 1, - DD_CIVISIBILITY_AGENTLESS_URL: `http://127.0.0.1:${port}`, - NODE_OPTIONS: '-r dd-trace/ci/init' - } -} - -function getCiVisEvpProxyConfig (port) { - return { - ...process.env, - DD_TRACE_AGENT_PORT: port, - NODE_OPTIONS: '-r dd-trace/ci/init', - DD_CIVISIBILITY_AGENTLESS_ENABLED: '0' - } -} - -function checkSpansForServiceName (spans, name) { - return spans.some((span) => span.some((nestedSpan) => nestedSpan.name === name)) -} - -async function spawnPluginIntegrationTestProc (cwd, serverFile, agentPort, stdioHandler, additionalEnvArgs = {}) { - let env = { - NODE_OPTIONS: `--loader=${hookFile}`, - DD_TRACE_AGENT_PORT: agentPort - } - env = { ...env, ...additionalEnvArgs } - return spawnProc(path.join(cwd, serverFile), { - cwd, - env - }, stdioHandler) -} - -module.exports = { - FakeAgent, - spawnProc, - createSandbox, - curl, - curlAndAssertMessage, - getCiVisAgentlessConfig, - getCiVisEvpProxyConfig, - checkSpansForServiceName, - spawnPluginIntegrationTestProc -} diff --git a/integration-tests/helpers/fake-agent.js b/integration-tests/helpers/fake-agent.js new file mode 100644 index 00000000000..70aff2ecfa8 --- /dev/null +++ b/integration-tests/helpers/fake-agent.js @@ -0,0 +1,324 @@ +'use strict' + +const { createHash } = require('crypto') +const EventEmitter = require('events') +const http = require('http') +const express = require('express') +const bodyParser = require('body-parser') +const msgpack = require('msgpack-lite') +const codec = msgpack.createCodec({ int64: true }) +const upload = require('multer')() + +module.exports = class FakeAgent extends EventEmitter { + constructor (port = 0) { + super() + this.port = port + this.resetRemoteConfig() + } + + async start () { + return new Promise((resolve, reject) => { + const timeoutObj = setTimeout(() => { + reject(new Error('agent timed out starting up')) + }, 10000) + this.server = http.createServer(buildExpressServer(this)) + this.server.on('error', reject) + this.server.listen(this.port, () => { + this.port = this.server.address().port + clearTimeout(timeoutObj) + resolve(this) + }) + }) + } + + stop () { + return new Promise((resolve) => { + this.server.on('close', resolve) + this.server.close() + }) + } + + /** + * Add a config object to be returned by the fake Remote Config endpoint. + * @param {Object} config - Object containing the Remote Config "file" and metadata + * @param {number} [config.orgId=2] - The Datadog organization ID + * @param {string} config.product - The Remote Config product name + * @param {string} config.id - The Remote Config config ID + * @param {string} [config.name] - The Remote Config "name". Defaults to the sha256 hash of `config.id` + * @param {Object} config.config - The Remote Config "file" object + */ + addRemoteConfig (config) { + config = { ...config } + config.orgId = config.orgId || 2 + config.name = config.name || createHash('sha256').update(config.id).digest('hex') + config.config = JSON.stringify(config.config) + config.path = `datadog/${config.orgId}/${config.product}/${config.id}/${config.name}` + config.fileHash = createHash('sha256').update(config.config).digest('hex') + config.meta = { + custom: { v: 1 }, + hashes: { sha256: config.fileHash }, + length: config.config.length + } + + this._rcFiles[config.id] = config + this._rcTargetsVersion++ + } + + /** + * Update an existing config object + * @param {string} id - The Remote Config config ID + * @param {Object} config - The Remote Config "file" object + */ + updateRemoteConfig (id, config) { + config = JSON.stringify(config) + config = Object.assign( + this._rcFiles[id], + { + config, + fileHash: createHash('sha256').update(config).digest('hex') + } + ) + config.meta.custom.v++ + config.meta.hashes.sha256 = config.fileHash + config.meta.length = config.config.length + this._rcTargetsVersion++ + } + + /** + * Remove a specific config object + * @param {string} id - The ID of the config object that should be removed + */ + removeRemoteConfig (id) { + delete this._rcFiles[id] + this._rcTargetsVersion++ + } + + /** + * Reset any existing Remote Config state. Usefull in `before` and `beforeEach` blocks. + */ + resetRemoteConfig () { + this._rcFiles = {} + this._rcTargetsVersion = 0 + this._rcSeenStates = new Set() + } + + // **resolveAtFirstSuccess** - specific use case for Next.js (or any other future libraries) + // where multiple payloads are generated, and only one is expected to have the proper span (ie next.request), + // but it't not guaranteed to be the last one (so, expectedMessageCount would not be helpful). + // It can still fail if it takes longer than `timeout` duration or if none pass the assertions (timeout still called) + assertMessageReceived (fn, timeout, expectedMessageCount = 1, resolveAtFirstSuccess) { + timeout = timeout || 30000 + let resultResolve + let resultReject + let msgCount = 0 + const errors = [] + + const timeoutObj = setTimeout(() => { + const errorsMsg = errors.length === 0 ? '' : `, additionally:\n${errors.map(e => e.stack).join('\n')}\n===\n` + resultReject(new Error(`timeout${errorsMsg}`, { cause: { errors } })) + }, timeout) + + const resultPromise = new Promise((resolve, reject) => { + resultResolve = () => { + clearTimeout(timeoutObj) + resolve() + } + resultReject = (e) => { + clearTimeout(timeoutObj) + reject(e) + } + }) + + const messageHandler = msg => { + try { + msgCount += 1 + fn(msg) + if (resolveAtFirstSuccess || msgCount === expectedMessageCount) { + resultResolve() + this.removeListener('message', messageHandler) + } + } catch (e) { + errors.push(e) + } + } + this.on('message', messageHandler) + + return resultPromise + } + + assertTelemetryReceived (fn, timeout, requestType, expectedMessageCount = 1) { + timeout = timeout || 30000 + let resultResolve + let resultReject + let msgCount = 0 + const errors = [] + + const timeoutObj = setTimeout(() => { + const errorsMsg = errors.length === 0 ? '' : `, additionally:\n${errors.map(e => e.stack).join('\n')}\n===\n` + resultReject(new Error(`timeout${errorsMsg}`, { cause: { errors } })) + }, timeout) + + const resultPromise = new Promise((resolve, reject) => { + resultResolve = () => { + clearTimeout(timeoutObj) + resolve() + } + resultReject = (e) => { + clearTimeout(timeoutObj) + reject(e) + } + }) + + const messageHandler = msg => { + if (msg.payload.request_type !== requestType) return + msgCount += 1 + try { + fn(msg) + if (msgCount === expectedMessageCount) { + resultResolve() + } + } catch (e) { + errors.push(e) + } + if (msgCount === expectedMessageCount) { + this.removeListener('telemetry', messageHandler) + } + } + this.on('telemetry', messageHandler) + + return resultPromise + } +} + +function buildExpressServer (agent) { + const app = express() + + app.use(bodyParser.raw({ limit: Infinity, type: 'application/msgpack' })) + app.use(bodyParser.json({ limit: Infinity, type: 'application/json' })) + + app.put('/v0.4/traces', (req, res) => { + if (req.body.length === 0) return res.status(200).send() + res.status(200).send({ rate_by_service: { 'service:,env:': 1 } }) + agent.emit('message', { + headers: req.headers, + payload: msgpack.decode(req.body, { codec }) + }) + }) + + app.post('/v0.7/config', (req, res) => { + const { + client: { products, state }, + cached_target_files: cachedTargetFiles + } = req.body + + if (state.has_error) { + // Print the error sent by the client in case it's useful in debugging tests + console.error(state.error) // eslint-disable-line no-console + } + + for (const cs of state.config_states) { + const uniqueState = `${cs.id}-${cs.version}-${cs.apply_state}` + if (!agent._rcSeenStates.has(uniqueState)) { + agent._rcSeenStates.add(uniqueState) + agent.emit('remote-config-ack-update', cs.id, cs.version, cs.apply_state, cs.apply_error) + } + + if (cs.apply_error) { + // Print the error sent by the client in case it's useful in debugging tests + console.error(cs.apply_error) // eslint-disable-line no-console + } + } + + res.on('close', () => { + agent.emit('remote-confg-responded') + }) + + if (agent._rcTargetsVersion === state.targets_version) { + // If the state hasn't changed since the last time the client asked, just return an empty result + res.json({}) + return + } + + if (Object.keys(agent._rcFiles).length === 0) { + // All config files have been removed, but the client has not yet been informed. + // Return this custom result to let the client know. + res.json({ client_configs: [] }) + return + } + + // The actual targets object is much more complicated, + // but the Node.js tracer currently only cares about the following properties. + const targets = { + signed: { + custom: { opaque_backend_state: 'foo' }, + targets: {}, + version: agent._rcTargetsVersion + } + } + const targetFiles = [] + const clientConfigs = [] + + const files = Object.values(agent._rcFiles).filter(({ product }) => products.includes(product)) + + for (const { path, fileHash, meta, config } of files) { + clientConfigs.push(path) + targets.signed.targets[path] = meta + + // skip files already cached by the client so we don't send them more than once + if (cachedTargetFiles.some((cached) => + path === cached.path && + fileHash === cached.hashes.find((e) => e.algorithm === 'sha256').hash + )) continue + + targetFiles.push({ path, raw: base64(config) }) + } + + // The real response object also contains a `roots` property which has been omitted here since it's not currently + // used by the Node.js tracer. + res.json({ + targets: clientConfigs.length === 0 ? undefined : base64(targets), + target_files: targetFiles, + client_configs: clientConfigs + }) + }) + + app.post('/debugger/v1/input', (req, res) => { + res.status(200).send() + agent.emit('debugger-input', { + headers: req.headers, + payload: req.body + }) + }) + + app.post('/debugger/v1/diagnostics', upload.any(), (req, res) => { + res.status(200).send() + agent.emit('debugger-diagnostics', { + headers: req.headers, + payload: JSON.parse(req.files[0].buffer.toString()) + }) + }) + + app.post('/profiling/v1/input', upload.any(), (req, res) => { + res.status(200).send() + agent.emit('message', { + headers: req.headers, + payload: req.body, + files: req.files + }) + }) + + app.post('/telemetry/proxy/api/v2/apmtelemetry', (req, res) => { + res.status(200).send() + agent.emit('telemetry', { + headers: req.headers, + payload: req.body + }) + }) + + return app +} + +function base64 (strOrObj) { + const str = typeof strOrObj === 'string' ? strOrObj : JSON.stringify(strOrObj) + return Buffer.from(str).toString('base64') +} diff --git a/integration-tests/helpers/index.js b/integration-tests/helpers/index.js new file mode 100644 index 00000000000..09cc6c5bee4 --- /dev/null +++ b/integration-tests/helpers/index.js @@ -0,0 +1,376 @@ +'use strict' + +const { promisify } = require('util') +const childProcess = require('child_process') +const { fork, spawn } = childProcess +const exec = promisify(childProcess.exec) +const http = require('http') +const fs = require('fs') +const os = require('os') +const path = require('path') +const assert = require('assert') +const rimraf = promisify(require('rimraf')) +const FakeAgent = require('./fake-agent') +const id = require('../../packages/dd-trace/src/id') + +const hookFile = 'dd-trace/loader-hook.mjs' + +async function runAndCheckOutput (filename, cwd, expectedOut) { + const proc = spawn('node', [filename], { cwd, stdio: 'pipe' }) + const pid = proc.pid + let out = await new Promise((resolve, reject) => { + proc.on('error', reject) + let out = Buffer.alloc(0) + proc.stdout.on('data', data => { + out = Buffer.concat([out, data]) + }) + proc.stderr.pipe(process.stdout) + proc.on('exit', () => resolve(out.toString('utf8'))) + setTimeout(() => { + if (proc.exitCode === null) proc.kill() + }, 1000) // TODO this introduces flakiness. find a better way to end the process. + }) + if (typeof expectedOut === 'function') { + expectedOut(out) + } else { + if (process.env.DD_TRACE_DEBUG) { + // Debug adds this, which we don't care about in these tests + out = out.replace('Flushing 0 metrics via HTTP\n', '') + } + assert.strictEqual(out, expectedOut) + } + return pid +} + +// This is set by the useSandbox function +let sandbox + +// This _must_ be used with the useSandbox function +async function runAndCheckWithTelemetry (filename, expectedOut, ...expectedTelemetryPoints) { + const cwd = sandbox.folder + const cleanup = telemetryForwarder(expectedTelemetryPoints) + const pid = await runAndCheckOutput(filename, cwd, expectedOut) + const msgs = await cleanup() + if (expectedTelemetryPoints.length === 0) { + // assert no telemetry sent + try { + assert.deepStrictEqual(msgs.length, 0) + } catch (e) { + // This console.log is useful for debugging telemetry. Plz don't remove. + // eslint-disable-next-line no-console + console.error('Expected no telemetry, but got:\n', msgs.map(msg => JSON.stringify(msg[1].points)).join('\n')) + throw e + } + return + } + let points = [] + for (const [telemetryType, data] of msgs) { + assert.strictEqual(telemetryType, 'library_entrypoint') + assert.deepStrictEqual(data.metadata, meta(pid)) + points = points.concat(data.points) + } + let expectedPoints = getPoints(...expectedTelemetryPoints) + // We now have to sort both the expected and actual telemetry points. + // This is because data can come in in any order. + // We'll just contatenate all the data together for each point and sort them. + points = points.map(p => p.name + '\t' + p.tags.join(',')).sort().join('\n') + expectedPoints = expectedPoints.map(p => p.name + '\t' + p.tags.join(',')).sort().join('\n') + assert.strictEqual(points, expectedPoints) + + function getPoints (...args) { + const expectedPoints = [] + let currentPoint = {} + for (const arg of args) { + if (!currentPoint.name) { + currentPoint.name = 'library_entrypoint.' + arg + } else { + currentPoint.tags = arg.split(',') + expectedPoints.push(currentPoint) + currentPoint = {} + } + } + return expectedPoints + } + + function meta (pid) { + return { + language_name: 'nodejs', + language_version: process.versions.node, + runtime_name: 'nodejs', + runtime_version: process.versions.node, + tracer_version: require('../../package.json').version, + pid: Number(pid) + } + } +} + +function spawnProc (filename, options = {}, stdioHandler, stderrHandler) { + const proc = fork(filename, { ...options, stdio: 'pipe' }) + return new Promise((resolve, reject) => { + proc + .on('message', ({ port }) => { + proc.url = `http://localhost:${port}` + resolve(proc) + }) + .on('error', reject) + .on('exit', code => { + if (code !== 0) { + reject(new Error(`Process exited with status code ${code}.`)) + } + resolve() + }) + + proc.stdout.on('data', data => { + if (stdioHandler) { + stdioHandler(data) + } + // eslint-disable-next-line no-console + if (!options.silent) console.log(data.toString()) + }) + + proc.stderr.on('data', data => { + if (stderrHandler) { + stderrHandler(data) + } + // eslint-disable-next-line no-console + if (!options.silent) console.error(data.toString()) + }) + }) +} + +async function createSandbox (dependencies = [], isGitRepo = false, + integrationTestsPaths = ['./integration-tests/*'], followUpCommand) { + /* To execute integration tests without a sandbox uncomment the next line + * and do `yarn link && yarn link dd-trace` */ + // return { folder: path.join(process.cwd(), 'integration-tests'), remove: async () => {} } + const folder = path.join(os.tmpdir(), id().toString()) + const out = path.join(folder, 'dd-trace.tgz') + const allDependencies = [`file:${out}`].concat(dependencies) + + // We might use NODE_OPTIONS to init the tracer. We don't want this to affect this operations + const { NODE_OPTIONS, ...restOfEnv } = process.env + + fs.mkdirSync(folder) + const addCommand = `yarn add ${allDependencies.join(' ')} --ignore-engines` + const addOptions = { cwd: folder, env: restOfEnv } + await exec(`yarn pack --filename ${out}`, { env: restOfEnv }) // TODO: cache this + + try { + await exec(addCommand, addOptions) + } catch (e) { // retry in case of server error from registry + await exec(addCommand, addOptions) + } + + for (const path of integrationTestsPaths) { + if (process.platform === 'win32') { + await exec(`Copy-Item -Recurse -Path "${path}" -Destination "${folder}"`, { shell: 'powershell.exe' }) + } else { + await exec(`cp -R ${path} ${folder}`) + } + } + if (process.platform === 'win32') { + // On Windows, we can only sync entire filesystem volume caches. + await exec(`Write-VolumeCache ${folder[0]}`, { shell: 'powershell.exe' }) + } else { + await exec(`sync ${folder}`) + } + + if (followUpCommand) { + await exec(followUpCommand, { cwd: folder, env: restOfEnv }) + } + + if (isGitRepo) { + await exec('git init', { cwd: folder }) + fs.writeFileSync(path.join(folder, '.gitignore'), 'node_modules/', { flush: true }) + await exec('git config user.email "john@doe.com"', { cwd: folder }) + await exec('git config user.name "John Doe"', { cwd: folder }) + await exec('git config commit.gpgsign false', { cwd: folder }) + await exec( + 'git add -A && git commit -m "first commit" --no-verify && git remote add origin git@git.com:datadog/example.git', + { cwd: folder } + ) + } + + return { + folder, + remove: async () => rimraf(folder) + } +} + +function telemetryForwarder (expectedTelemetryPoints) { + process.env.DD_TELEMETRY_FORWARDER_PATH = + path.join(__dirname, '..', 'telemetry-forwarder.sh') + process.env.FORWARDER_OUT = path.join(__dirname, `forwarder-${Date.now()}.out`) + + let retries = 0 + + const tryAgain = async function () { + retries += 1 + await new Promise(resolve => setTimeout(resolve, 100)) + return cleanup() + } + + const cleanup = function () { + let msgs + try { + msgs = fs.readFileSync(process.env.FORWARDER_OUT, 'utf8').trim().split('\n') + } catch (e) { + if (expectedTelemetryPoints.length && e.code === 'ENOENT' && retries < 10) { + return tryAgain() + } + return [] + } + for (let i = 0; i < msgs.length; i++) { + const [telemetryType, data] = msgs[i].split('\t') + if (!data && retries < 10) { + return tryAgain() + } + let parsed + try { + parsed = JSON.parse(data) + } catch (e) { + if (!data && retries < 10) { + return tryAgain() + } + throw new SyntaxError(`error parsing data: ${e.message}\n${data}`) + } + msgs[i] = [telemetryType, parsed] + } + fs.unlinkSync(process.env.FORWARDER_OUT) + delete process.env.FORWARDER_OUT + delete process.env.DD_TELEMETRY_FORWARDER_PATH + return msgs + } + + return cleanup +} + +async function curl (url, useHttp2 = false) { + if (url !== null && typeof url === 'object') { + if (url.then) { + return curl(await url) + } + url = url.url + } + + return new Promise((resolve, reject) => { + http.get(url, res => { + const bufs = [] + res.on('data', d => bufs.push(d)) + res.on('end', () => { + res.body = Buffer.concat(bufs).toString('utf8') + resolve(res) + }) + res.on('error', reject) + }).on('error', reject) + }) +} + +async function curlAndAssertMessage (agent, procOrUrl, fn, timeout, expectedMessageCount, resolveAtFirstSuccess) { + const resultPromise = agent.assertMessageReceived(fn, timeout, expectedMessageCount, resolveAtFirstSuccess) + await curl(procOrUrl) + return resultPromise +} + +function getCiVisAgentlessConfig (port) { + // We remove GITHUB_WORKSPACE so the repository root is not assigned to dd-trace-js + const { GITHUB_WORKSPACE, ...rest } = process.env + return { + ...rest, + DD_API_KEY: '1', + DD_CIVISIBILITY_AGENTLESS_ENABLED: 1, + DD_CIVISIBILITY_AGENTLESS_URL: `http://127.0.0.1:${port}`, + NODE_OPTIONS: '-r dd-trace/ci/init', + DD_INSTRUMENTATION_TELEMETRY_ENABLED: 'false' + } +} + +function getCiVisEvpProxyConfig (port) { + // We remove GITHUB_WORKSPACE so the repository root is not assigned to dd-trace-js + const { GITHUB_WORKSPACE, ...rest } = process.env + return { + ...rest, + DD_TRACE_AGENT_PORT: port, + NODE_OPTIONS: '-r dd-trace/ci/init', + DD_CIVISIBILITY_AGENTLESS_ENABLED: '0', + DD_INSTRUMENTATION_TELEMETRY_ENABLED: 'false' + } +} + +function checkSpansForServiceName (spans, name) { + return spans.some((span) => span.some((nestedSpan) => nestedSpan.name === name)) +} + +async function spawnPluginIntegrationTestProc (cwd, serverFile, agentPort, stdioHandler, additionalEnvArgs = {}) { + let env = { + NODE_OPTIONS: `--loader=${hookFile}`, + DD_TRACE_AGENT_PORT: agentPort + } + env = { ...env, ...additionalEnvArgs } + return spawnProc(path.join(cwd, serverFile), { + cwd, + env + }, stdioHandler) +} + +function useEnv (env) { + before(() => { + Object.assign(process.env, env) + }) + after(() => { + for (const key of Object.keys(env)) { + delete process.env[key] + } + }) +} + +function useSandbox (...args) { + before(async () => { + sandbox = await createSandbox(...args) + }) + after(() => { + const oldSandbox = sandbox + sandbox = undefined + return oldSandbox.remove() + }) +} + +function sandboxCwd () { + return sandbox.folder +} + +function assertObjectContains (actual, expected) { + for (const [key, val] of Object.entries(expected)) { + if (val !== null && typeof val === 'object') { + assert.ok(key in actual) + assert.notStrictEqual(actual[key], null) + assert.strictEqual(typeof actual[key], 'object') + assertObjectContains(actual[key], val) + } else { + assert.strictEqual(actual[key], expected[key]) + } + } +} + +function assertUUID (actual, msg = 'not a valid UUID') { + assert.match(actual, /^[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}$/, msg) +} + +module.exports = { + FakeAgent, + hookFile, + assertObjectContains, + assertUUID, + spawnProc, + runAndCheckWithTelemetry, + createSandbox, + curl, + curlAndAssertMessage, + getCiVisAgentlessConfig, + getCiVisEvpProxyConfig, + checkSpansForServiceName, + spawnPluginIntegrationTestProc, + useEnv, + useSandbox, + sandboxCwd +} diff --git a/integration-tests/init.spec.js b/integration-tests/init.spec.js new file mode 100644 index 00000000000..571179276e1 --- /dev/null +++ b/integration-tests/init.spec.js @@ -0,0 +1,185 @@ +const semver = require('semver') +const { + runAndCheckWithTelemetry: testFile, + useEnv, + useSandbox, + sandboxCwd +} = require('./helpers') +const path = require('path') +const fs = require('fs') +const { DD_MAJOR } = require('../version') + +const DD_INJECTION_ENABLED = 'tracing' +const DD_INJECT_FORCE = 'true' +const DD_TRACE_DEBUG = 'true' + +const telemetryAbort = ['abort', 'reason:incompatible_runtime', 'abort.runtime', ''] +const telemetryForced = ['complete', 'injection_forced:true'] +const telemetryGood = ['complete', 'injection_forced:false'] + +const { engines } = require('../package.json') +const supportedRange = engines.node +const currentVersionIsSupported = semver.satisfies(process.versions.node, supportedRange) + +// These are on by default in release tests, so we'll turn them off for +// more fine-grained control of these variables in these tests. +delete process.env.DD_INJECTION_ENABLED +delete process.env.DD_INJECT_FORCE + +function testInjectionScenarios (arg, filename, esmWorks = false) { + if (!currentVersionIsSupported) return + const doTest = (file, ...args) => testFile(file, ...args) + context('preferring app-dir dd-trace', () => { + context('when dd-trace is not in the app dir', () => { + const NODE_OPTIONS = `--no-warnings --${arg} ${path.join(__dirname, '..', filename)}` + useEnv({ NODE_OPTIONS }) + + context('without DD_INJECTION_ENABLED', () => { + it('should initialize the tracer', () => doTest('init/trace.js', 'true\n')) + it('should initialize instrumentation', () => doTest('init/instrument.js', 'true\n')) + it(`should ${esmWorks ? '' : 'not '}initialize ESM instrumentation`, () => + doTest('init/instrument.mjs', `${esmWorks}\n`)) + }) + context('with DD_INJECTION_ENABLED', () => { + useEnv({ DD_INJECTION_ENABLED }) + + it('should not initialize the tracer', () => doTest('init/trace.js', 'false\n')) + it('should not initialize instrumentation', () => doTest('init/instrument.js', 'false\n')) + it('should not initialize ESM instrumentation', () => doTest('init/instrument.mjs', 'false\n')) + }) + }) + context('when dd-trace in the app dir', () => { + const NODE_OPTIONS = `--no-warnings --${arg} dd-trace/${filename}` + useEnv({ NODE_OPTIONS }) + + context('without DD_INJECTION_ENABLED', () => { + it('should initialize the tracer', () => doTest('init/trace.js', 'true\n')) + it('should initialize instrumentation', () => doTest('init/instrument.js', 'true\n')) + it(`should ${esmWorks ? '' : 'not '}initialize ESM instrumentation`, () => + doTest('init/instrument.mjs', `${esmWorks}\n`)) + }) + context('with DD_INJECTION_ENABLED', () => { + useEnv({ DD_INJECTION_ENABLED }) + + it('should initialize the tracer', () => doTest('init/trace.js', 'true\n', ...telemetryGood)) + it('should initialize instrumentation', () => doTest('init/instrument.js', 'true\n', ...telemetryGood)) + it(`should ${esmWorks ? '' : 'not '}initialize ESM instrumentation`, () => + doTest('init/instrument.mjs', `${esmWorks}\n`, ...telemetryGood)) + }) + }) + }) +} + +function testRuntimeVersionChecks (arg, filename) { + context('runtime version check', () => { + const NODE_OPTIONS = `--${arg} dd-trace/${filename}` + const doTest = (...args) => testFile('init/trace.js', ...args) + const doTestForced = async (...args) => { + Object.assign(process.env, { DD_INJECT_FORCE }) + try { + await testFile('init/trace.js', ...args) + } finally { + delete process.env.DD_INJECT_FORCE + } + } + + if (!currentVersionIsSupported) { + context('when node version is less than engines field', () => { + useEnv({ NODE_OPTIONS }) + + it('should initialize the tracer, if no DD_INJECTION_ENABLED', () => + doTest('true\n')) + context('with DD_INJECTION_ENABLED', () => { + useEnv({ DD_INJECTION_ENABLED }) + + context('without debug', () => { + it('should not initialize the tracer', () => doTest('false\n', ...telemetryAbort)) + it('should initialize the tracer, if DD_INJECT_FORCE', () => doTestForced('true\n', ...telemetryForced)) + }) + context('with debug', () => { + useEnv({ DD_TRACE_DEBUG }) + + it('should not initialize the tracer', () => + doTest(`Aborting application instrumentation due to incompatible_runtime. +Found incompatible runtime nodejs ${process.versions.node}, Supported runtimes: nodejs \ +>=${DD_MAJOR === 4 ? '16' : '18'}. +false +`, ...telemetryAbort)) + it('should initialize the tracer, if DD_INJECT_FORCE', () => + doTestForced(`Aborting application instrumentation due to incompatible_runtime. +Found incompatible runtime nodejs ${process.versions.node}, Supported runtimes: nodejs \ +>=${DD_MAJOR === 4 ? '16' : '18'}. +DD_INJECT_FORCE enabled, allowing unsupported runtimes and continuing. +Application instrumentation bootstrapping complete +true +`, ...telemetryForced)) + }) + }) + }) + } else { + context('when node version is more than engines field', () => { + useEnv({ NODE_OPTIONS }) + + it('should initialize the tracer, if no DD_INJECTION_ENABLED', () => doTest('true\n')) + context('with DD_INJECTION_ENABLED', () => { + useEnv({ DD_INJECTION_ENABLED }) + + context('without debug', () => { + it('should initialize the tracer', () => doTest('true\n', ...telemetryGood)) + it('should initialize the tracer, if DD_INJECT_FORCE', () => + doTestForced('true\n', ...telemetryGood)) + }) + context('with debug', () => { + useEnv({ DD_TRACE_DEBUG }) + + it('should initialize the tracer', () => + doTest('Application instrumentation bootstrapping complete\ntrue\n', ...telemetryGood)) + it('should initialize the tracer, if DD_INJECT_FORCE', () => + doTestForced('Application instrumentation bootstrapping complete\ntrue\n', ...telemetryGood)) + }) + }) + }) + } + }) +} + +function stubTracerIfNeeded () { + if (!currentVersionIsSupported) { + before(() => { + // Stub out the tracer in the sandbox, since it will not likely load properly. + // We're only doing this on versions we don't support, since the forcing + // action results in undefined behavior in the tracer. + fs.writeFileSync( + path.join(sandboxCwd(), 'node_modules/dd-trace/index.js'), + 'exports.init = () => { Object.assign(global, { _ddtrace: true }) }' + ) + }) + } +} + +describe('init.js', () => { + useSandbox() + stubTracerIfNeeded() + + testInjectionScenarios('require', 'init.js', false) + testRuntimeVersionChecks('require', 'init.js') +}) + +// ESM is not supportable prior to Node.js 12 +if (semver.satisfies(process.versions.node, '>=12')) { + describe('initialize.mjs', () => { + useSandbox() + stubTracerIfNeeded() + + context('as --loader', () => { + testInjectionScenarios('loader', 'initialize.mjs', true) + testRuntimeVersionChecks('loader', 'initialize.mjs') + }) + if (Number(process.versions.node.split('.')[0]) >= 18) { + context('as --import', () => { + testInjectionScenarios('import', 'initialize.mjs', true) + testRuntimeVersionChecks('loader', 'initialize.mjs') + }) + } + }) +} diff --git a/integration-tests/init/instrument.js b/integration-tests/init/instrument.js new file mode 100644 index 00000000000..55e5d28f450 --- /dev/null +++ b/integration-tests/init/instrument.js @@ -0,0 +1,21 @@ +const http = require('http') +const dc = require('dc-polyfill') + +let gotEvent = false +dc.subscribe('apm:http:client:request:start', (event) => { + gotEvent = true +}) + +const server = http.createServer((req, res) => { + res.end('Hello World') +}).listen(0, () => { + http.get(`http://localhost:${server.address().port}`, (res) => { + res.on('data', () => {}) + res.on('end', () => { + server.close() + // eslint-disable-next-line no-console + console.log(gotEvent) + process.exit() + }) + }) +}) diff --git a/integration-tests/init/instrument.mjs b/integration-tests/init/instrument.mjs new file mode 100644 index 00000000000..bddaf6ef13a --- /dev/null +++ b/integration-tests/init/instrument.mjs @@ -0,0 +1,21 @@ +import http from 'http' +import dc from 'dc-polyfill' + +let gotEvent = false +dc.subscribe('apm:http:client:request:start', (event) => { + gotEvent = true +}) + +const server = http.createServer((req, res) => { + res.end('Hello World') +}).listen(0, () => { + http.get(`http://localhost:${server.address().port}`, (res) => { + res.on('data', () => {}) + res.on('end', () => { + server.close() + // eslint-disable-next-line no-console + console.log(gotEvent) + process.exit() + }) + }) +}) diff --git a/integration-tests/init/trace.js b/integration-tests/init/trace.js new file mode 100644 index 00000000000..9c9c5d731d9 --- /dev/null +++ b/integration-tests/init/trace.js @@ -0,0 +1,3 @@ +// eslint-disable-next-line no-console +console.log(!!global._ddtrace) +process.exit() diff --git a/integration-tests/jest/jest.spec.js b/integration-tests/jest/jest.spec.js new file mode 100644 index 00000000000..789019100da --- /dev/null +++ b/integration-tests/jest/jest.spec.js @@ -0,0 +1,2360 @@ +'use strict' + +const { fork, exec } = require('child_process') +const path = require('path') + +const { assert } = require('chai') + +const { + createSandbox, + getCiVisAgentlessConfig, + getCiVisEvpProxyConfig +} = require('../helpers') +const { FakeCiVisIntake } = require('../ci-visibility-intake') +const { + TEST_CODE_COVERAGE_ENABLED, + TEST_ITR_SKIPPING_ENABLED, + TEST_ITR_TESTS_SKIPPED, + TEST_CODE_COVERAGE_LINES_PCT, + TEST_SUITE, + TEST_STATUS, + TEST_SKIPPED_BY_ITR, + TEST_ITR_SKIPPING_TYPE, + TEST_ITR_SKIPPING_COUNT, + TEST_ITR_UNSKIPPABLE, + TEST_ITR_FORCED_RUN, + TEST_SOURCE_FILE, + TEST_IS_NEW, + TEST_IS_RETRY, + TEST_EARLY_FLAKE_ENABLED, + TEST_NAME, + JEST_DISPLAY_NAME, + TEST_EARLY_FLAKE_ABORT_REASON, + TEST_SOURCE_START, + TEST_CODE_OWNERS, + TEST_SESSION_NAME, + TEST_LEVEL_EVENT_TYPES +} = require('../../packages/dd-trace/src/plugins/util/test') +const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') +const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants') + +const testFile = 'ci-visibility/run-jest.js' +const expectedStdout = 'Test Suites: 2 passed' +const expectedCoverageFiles = [ + 'ci-visibility/test/sum.js', + 'ci-visibility/test/ci-visibility-test.js', + 'ci-visibility/test/ci-visibility-test-2.js' +] +const runTestsWithCoverageCommand = 'node ./ci-visibility/run-jest.js' + +// TODO: add ESM tests +describe('jest CommonJS', () => { + let receiver + let childProcess + let sandbox + let cwd + let startupTestFile + let testOutput = '' + + before(async function () { + sandbox = await createSandbox(['jest', 'chai@v4', 'jest-jasmine2', 'jest-environment-jsdom'], true) + cwd = sandbox.folder + startupTestFile = path.join(cwd, testFile) + }) + + after(async function () { + await sandbox.remove() + }) + + beforeEach(async function () { + receiver = await new FakeCiVisIntake().start() + }) + + afterEach(async () => { + childProcess.kill() + testOutput = '' + await receiver.stop() + }) + + it('can run tests and report tests with the APM protocol (old agents)', (done) => { + receiver.setInfoResponse({ endpoints: [] }) + receiver.payloadReceived(({ url }) => url === '/v0.4/traces').then(({ payload }) => { + const testSpans = payload.flatMap(trace => trace) + const resourceNames = testSpans.map(span => span.resource) + + assert.includeMembers(resourceNames, + [ + 'ci-visibility/test/ci-visibility-test.js.ci visibility can report tests', + 'ci-visibility/test/ci-visibility-test-2.js.ci visibility 2 can report tests 2' + ] + ) + + const areAllTestSpans = testSpans.every(span => span.name === 'jest.test') + assert.isTrue(areAllTestSpans) + + assert.include(testOutput, expectedStdout) + + // Can read DD_TAGS + testSpans.forEach(testSpan => { + assert.propertyVal(testSpan.meta, 'test.customtag', 'customvalue') + assert.propertyVal(testSpan.meta, 'test.customtag2', 'customvalue2') + }) + + testSpans.forEach(testSpan => { + assert.equal(testSpan.meta[TEST_SOURCE_FILE].startsWith('ci-visibility/test/ci-visibility-test'), true) + assert.exists(testSpan.metrics[TEST_SOURCE_START]) + }) + + done() + }) + + childProcess = fork(startupTestFile, { + cwd, + env: { + DD_TRACE_AGENT_PORT: receiver.port, + NODE_OPTIONS: '-r dd-trace/ci/init', + DD_TAGS: 'test.customtag:customvalue,test.customtag2:customvalue2' + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + }) + + const nonLegacyReportingOptions = ['agentless', 'evp proxy'] + + nonLegacyReportingOptions.forEach((reportingOption) => { + it(`can run and report tests with ${reportingOption}`, (done) => { + const envVars = reportingOption === 'agentless' + ? getCiVisAgentlessConfig(receiver.port) + : getCiVisEvpProxyConfig(receiver.port) + if (reportingOption === 'evp proxy') { + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + } + receiver.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('citestcycle'), (payloads) => { + const metadataDicts = payloads.flatMap(({ payload }) => payload.metadata) + + metadataDicts.forEach(metadata => { + for (const testLevel of TEST_LEVEL_EVENT_TYPES) { + assert.equal(metadata[testLevel][TEST_SESSION_NAME], 'my-test-session') + } + }) + + const events = payloads.flatMap(({ payload }) => payload.events) + const sessionEventContent = events.find(event => event.type === 'test_session_end').content + const moduleEventContent = events.find(event => event.type === 'test_module_end').content + const suites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const resourceNames = tests.map(span => span.resource) + + assert.includeMembers(resourceNames, + [ + 'ci-visibility/test/ci-visibility-test.js.ci visibility can report tests', + 'ci-visibility/test/ci-visibility-test-2.js.ci visibility 2 can report tests 2' + ] + ) + assert.equal(suites.length, 2) + assert.exists(sessionEventContent) + assert.exists(moduleEventContent) + + assert.include(testOutput, expectedStdout) + + tests.forEach(testEvent => { + assert.equal(testEvent.meta[TEST_SOURCE_FILE].startsWith('ci-visibility/test/ci-visibility-test'), true) + assert.exists(testEvent.metrics[TEST_SOURCE_START]) + // Can read DD_TAGS + assert.propertyVal(testEvent.meta, 'test.customtag', 'customvalue') + assert.propertyVal(testEvent.meta, 'test.customtag2', 'customvalue2') + assert.exists(testEvent.metrics[DD_HOST_CPU_COUNT]) + }) + + suites.forEach(testSuite => { + assert.isTrue(testSuite.meta[TEST_SOURCE_FILE].startsWith('ci-visibility/test/ci-visibility-test')) + assert.equal(testSuite.metrics[TEST_SOURCE_START], 1) + assert.exists(testSuite.metrics[DD_HOST_CPU_COUNT]) + }) + + done() + }) + + childProcess = fork(startupTestFile, { + cwd, + env: { + ...envVars, + DD_TAGS: 'test.customtag:customvalue,test.customtag2:customvalue2', + DD_TEST_SESSION_NAME: 'my-test-session' + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + }) + }) + + const envVarSettings = ['DD_TRACING_ENABLED', 'DD_TRACE_ENABLED'] + + envVarSettings.forEach(envVar => { + context(`when ${envVar}=false`, () => { + it('does not report spans but still runs tests', (done) => { + receiver.assertMessageReceived(() => { + done(new Error('Should not create spans')) + }).catch(() => {}) + + childProcess = fork(startupTestFile, { + cwd, + env: { + DD_TRACE_AGENT_PORT: receiver.port, + NODE_OPTIONS: '-r dd-trace/ci/init', + [envVar]: 'false' + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('message', () => { + assert.include(testOutput, expectedStdout) + done() + }) + }) + }) + }) + + context('when no ci visibility init is used', () => { + it('does not crash', (done) => { + childProcess = fork(startupTestFile, { + cwd, + env: { + DD_TRACE_AGENT_PORT: receiver.port, + NODE_OPTIONS: '-r dd-trace/init' + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('message', () => { + assert.notInclude(testOutput, 'TypeError') + assert.notInclude(testOutput, 'Uncaught error outside test suite') + assert.include(testOutput, expectedStdout) + done() + }) + }) + }) + + it('correctly calculates test code owners when working directory is not repository root', (done) => { + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const test = events.find(event => event.type === 'test').content + const testSuite = events.find(event => event.type === 'test_suite_end').content + // The test is in a subproject + assert.notEqual(test.meta[TEST_SOURCE_FILE], test.meta[TEST_SUITE]) + assert.equal(test.meta[TEST_CODE_OWNERS], JSON.stringify(['@datadog-dd-trace-js'])) + assert.equal(testSuite.meta[TEST_CODE_OWNERS], JSON.stringify(['@datadog-dd-trace-js'])) + }) + + childProcess = exec( + 'node ./node_modules/jest/bin/jest --config config-jest.js --rootDir ci-visibility/subproject', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + PROJECTS: JSON.stringify([{ + testMatch: ['**/subproject-test*'] + }]) + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('works when sharding', (done) => { + receiver.payloadReceived(({ url }) => url === '/api/v2/citestcycle').then(events => { + const testSuiteEvents = events.payload.events.filter(event => event.type === 'test_suite_end') + assert.equal(testSuiteEvents.length, 3) + const testSuites = testSuiteEvents.map(span => span.content.meta[TEST_SUITE]) + + assert.includeMembers(testSuites, + [ + 'ci-visibility/sharding-test/sharding-test-5.js', + 'ci-visibility/sharding-test/sharding-test-4.js', + 'ci-visibility/sharding-test/sharding-test-1.js' + ] + ) + + const testSession = events.payload.events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') + + // We run the second shard + receiver.setSuitesToSkip([ + { + type: 'suite', + attributes: { + suite: 'ci-visibility/sharding-test/sharding-test-2.js' + } + }, + { + type: 'suite', + attributes: { + suite: 'ci-visibility/sharding-test/sharding-test-3.js' + } + } + ]) + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: 'sharding-test/sharding-test', + TEST_SHARD: '2/2' + }, + stdio: 'inherit' + } + ) + + receiver.payloadReceived(({ url }) => url === '/api/v2/citestcycle').then(secondShardEvents => { + const testSuiteEvents = secondShardEvents.payload.events.filter(event => event.type === 'test_suite_end') + + // The suites for this shard are to be skipped + assert.equal(testSuiteEvents.length, 2) + + testSuiteEvents.forEach(testSuite => { + assert.propertyVal(testSuite.content.meta, TEST_STATUS, 'skip') + assert.propertyVal(testSuite.content.meta, TEST_SKIPPED_BY_ITR, 'true') + }) + + const testSession = secondShardEvents + .payload + .events + .find(event => event.type === 'test_session_end').content + + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_TYPE, 'suite') + assert.propertyVal(testSession.metrics, TEST_ITR_SKIPPING_COUNT, 2) + + done() + }) + }) + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: 'sharding-test/sharding-test', + TEST_SHARD: '1/2' + }, + stdio: 'inherit' + } + ) + }) + + it('does not crash when jest is badly initialized', (done) => { + childProcess = fork('ci-visibility/run-jest-bad-init.js', { + cwd, + env: { + DD_TRACE_AGENT_PORT: receiver.port + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('message', () => { + assert.notInclude(testOutput, 'TypeError') + assert.include(testOutput, expectedStdout) + done() + }) + }) + + it('does not crash when jest uses jest-jasmine2', (done) => { + childProcess = fork(testFile, { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + OLD_RUNNER: 1, + NODE_OPTIONS: '-r dd-trace/ci/init', + RUN_IN_PARALLEL: true + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('message', () => { + assert.notInclude(testOutput, 'TypeError') + done() + }) + }) + + context('when jest is using workers to run tests in parallel', () => { + it('reports tests when using the old agents', (done) => { + receiver.setInfoResponse({ endpoints: [] }) + childProcess = fork(testFile, { + cwd, + env: { + DD_TRACE_AGENT_PORT: receiver.port, + NODE_OPTIONS: '-r dd-trace/ci/init', + RUN_IN_PARALLEL: true + }, + stdio: 'pipe' + }) + + receiver.gatherPayloads(({ url }) => url === '/v0.4/traces', 5000).then(tracesRequests => { + const testSpans = tracesRequests.flatMap(trace => trace.payload).flatMap(request => request) + assert.equal(testSpans.length, 2) + const spanTypes = testSpans.map(span => span.type) + assert.includeMembers(spanTypes, ['test']) + assert.notInclude(spanTypes, ['test_session_end', 'test_suite_end', 'test_module_end']) + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v2'] }) + done() + }).catch(done) + }) + + it('reports tests when using agentless', (done) => { + childProcess = fork(testFile, { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + RUN_IN_PARALLEL: true, + DD_TEST_SESSION_NAME: 'my-test-session' + }, + stdio: 'pipe' + }) + + receiver.gatherPayloads(({ url }) => url === '/api/v2/citestcycle', 5000).then(eventsRequests => { + const metadataDicts = eventsRequests.flatMap(({ payload }) => payload.metadata) + + // it propagates test session name to the test and test suite events in parallel mode + metadataDicts.forEach(metadata => { + for (const testLevel of TEST_LEVEL_EVENT_TYPES) { + assert.equal(metadata[testLevel][TEST_SESSION_NAME], 'my-test-session') + } + }) + + const events = eventsRequests.map(({ payload }) => payload) + .flatMap(({ events }) => events) + const eventTypes = events.map(event => event.type) + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + + done() + }).catch(done) + }) + + it('reports tests when using evp proxy', (done) => { + childProcess = fork(testFile, { + cwd, + env: { + ...getCiVisEvpProxyConfig(receiver.port), + RUN_IN_PARALLEL: true + }, + stdio: 'pipe' + }) + + receiver.gatherPayloads(({ url }) => url === '/evp_proxy/v2/api/v2/citestcycle', 5000) + .then(eventsRequests => { + const eventTypes = eventsRequests.map(({ payload }) => payload) + .flatMap(({ events }) => events) + .map(event => event.type) + + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + done() + }).catch(done) + }) + }) + + it('reports timeout error message', (done) => { + childProcess = fork(testFile, { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + NODE_OPTIONS: '-r dd-trace/ci/init', + RUN_IN_PARALLEL: true, + TESTS_TO_RUN: 'timeout-test/timeout-test.js' + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('message', () => { + assert.include(testOutput, 'Exceeded timeout of 100 ms for a test') + done() + }) + }) + + it('reports parsing errors in the test file', (done) => { + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const suites = events.filter(event => event.type === 'test_suite_end') + assert.equal(suites.length, 2) + + const resourceNames = suites.map(suite => suite.content.resource) + + assert.includeMembers(resourceNames, [ + 'test_suite.ci-visibility/test-parsing-error/parsing-error-2.js', + 'test_suite.ci-visibility/test-parsing-error/parsing-error.js' + ]) + suites.forEach(suite => { + assert.equal(suite.content.meta[TEST_STATUS], 'fail') + assert.include(suite.content.meta[ERROR_MESSAGE], 'chao') + }) + }) + childProcess = fork(testFile, { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: 'test-parsing-error/parsing-error' + }, + stdio: 'pipe' + }) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('does not report total code coverage % if user has not configured coverage manually', (done) => { + receiver.setSettings({ + itr_enabled: true, + code_coverage: true, + tests_skipping: false + }) + + receiver.assertPayloadReceived(({ payload }) => { + const testSession = payload.events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.metrics, TEST_CODE_COVERAGE_LINES_PCT) + }, ({ url }) => url === '/api/v2/citestcycle').then(() => done()).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + DISABLE_CODE_COVERAGE: '1' + }, + stdio: 'inherit' + } + ) + }) + + it('reports total code coverage % even when ITR is disabled', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false + }) + + receiver.assertPayloadReceived(({ payload }) => { + const testSession = payload.events.find(event => event.type === 'test_session_end').content + assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) + }, ({ url }) => url === '/api/v2/citestcycle').then(() => done()).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + }) + + it('works with --forceExit and logs a warning', (done) => { + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + assert.include(testOutput, "Jest's '--forceExit' flag has been passed") + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end') + const testModule = events.find(event => event.type === 'test_module_end') + const testSuites = events.filter(event => event.type === 'test_suite_end') + const tests = events.filter(event => event.type === 'test') + + assert.exists(testSession) + assert.exists(testModule) + assert.equal(testSuites.length, 2) + assert.equal(tests.length, 2) + }) + // Needs to run with the CLI if we want --forceExit to work + childProcess = exec( + 'node ./node_modules/jest/bin/jest --config config-jest.js --forceExit', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + DD_TRACE_DEBUG: '1', + DD_TRACE_LOG_LEVEL: 'warn' + }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + }) + + it('does not hang if server is not available and logs an error', (done) => { + // Very slow intake + receiver.setWaitingTime(30000) + // Needs to run with the CLI if we want --forceExit to work + childProcess = exec( + 'node ./node_modules/jest/bin/jest --config config-jest.js --forceExit', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + DD_TRACE_DEBUG: '1', + DD_TRACE_LOG_LEVEL: 'warn' + }, + stdio: 'inherit' + } + ) + const EXPECTED_FORCE_EXIT_LOG_MESSAGE = "Jest's '--forceExit' flag has been passed" + const EXPECTED_TIMEOUT_LOG_MESSAGE = 'Timeout waiting for the tracer to flush' + childProcess.on('exit', () => { + assert.include( + testOutput, + EXPECTED_FORCE_EXIT_LOG_MESSAGE, + `"${EXPECTED_FORCE_EXIT_LOG_MESSAGE}" log message is not in test output: ${testOutput}` + ) + assert.include( + testOutput, + EXPECTED_TIMEOUT_LOG_MESSAGE, + `"${EXPECTED_TIMEOUT_LOG_MESSAGE}" log message is not in the test output: ${testOutput}` + ) + done() + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + }) + + it('grabs the jest displayName config and sets tag in tests and suites', (done) => { + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + assert.equal(tests.length, 4) // two per display name + const nodeTests = tests.filter(test => test.meta[JEST_DISPLAY_NAME] === 'node') + assert.equal(nodeTests.length, 2) + + const standardTests = tests.filter(test => test.meta[JEST_DISPLAY_NAME] === 'standard') + assert.equal(standardTests.length, 2) + + const suites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) + assert.equal(suites.length, 4) + + const nodeSuites = suites.filter(suite => suite.meta[JEST_DISPLAY_NAME] === 'node') + assert.equal(nodeSuites.length, 2) + + const standardSuites = suites.filter(suite => suite.meta[JEST_DISPLAY_NAME] === 'standard') + assert.equal(standardSuites.length, 2) + }) + childProcess = exec( + 'node ./node_modules/jest/bin/jest --config config-jest-multiproject.js', + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('reports errors in test sessions', (done) => { + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_STATUS, 'fail') + const errorMessage = 'Failed test suites: 1. Failed tests: 1' + assert.include(testSession.meta[ERROR_MESSAGE], errorMessage) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: 'test/fail-test' + }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('does not init if DD_API_KEY is not set', (done) => { + receiver.assertMessageReceived(() => { + done(new Error('Should not create spans')) + }).catch(() => {}) + + childProcess = fork(startupTestFile, { + cwd, + env: { + DD_CIVISIBILITY_AGENTLESS_ENABLED: 1, + NODE_OPTIONS: '-r dd-trace/ci/init' + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('message', () => { + assert.include(testOutput, expectedStdout) + assert.include(testOutput, 'DD_CIVISIBILITY_AGENTLESS_ENABLED is set, ' + + 'but neither DD_API_KEY nor DATADOG_API_KEY are set in your environment, ' + + 'so dd-trace will not be initialized.' + ) + done() + }) + }) + + it('can report git metadata', (done) => { + const searchCommitsRequestPromise = receiver.payloadReceived( + ({ url }) => url === '/api/v2/git/repository/search_commits' + ) + const packfileRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/git/repository/packfile') + const eventsRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcycle') + + Promise.all([ + searchCommitsRequestPromise, + packfileRequestPromise, + eventsRequestPromise + ]).then(([searchCommitRequest, packfileRequest, eventsRequest]) => { + assert.propertyVal(searchCommitRequest.headers, 'dd-api-key', '1') + assert.propertyVal(packfileRequest.headers, 'dd-api-key', '1') + + const eventTypes = eventsRequest.payload.events.map(event => event.type) + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + ) + assert.equal(numSuites, 2) + + done() + }).catch(done) + + childProcess = fork(startupTestFile, { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'pipe' + }) + }) + + context('intelligent test runner', () => { + context('if the agent is not event platform proxy compatible', () => { + it('does not do any intelligent test runner request', (done) => { + receiver.setInfoResponse({ endpoints: [] }) + + receiver.assertPayloadReceived(() => { + const error = new Error('should not request search_commits') + done(error) + }, ({ url }) => url === '/evp_proxy/v2/api/v2/git/repository/search_commits').catch(() => {}) + receiver.assertPayloadReceived(() => { + const error = new Error('should not request search_commits') + done(error) + }, ({ url }) => url === '/api/v2/git/repository/search_commits').catch(() => {}) + receiver.assertPayloadReceived(() => { + const error = new Error('should not request setting') + done(error) + }, ({ url }) => url === '/api/v2/libraries/tests/services/setting').catch(() => {}) + receiver.assertPayloadReceived(() => { + const error = new Error('should not request setting') + done(error) + }, ({ url }) => url === '/evp_proxy/v2/api/v2/libraries/tests/services/setting').catch(() => {}) + + receiver.assertPayloadReceived(({ payload }) => { + const testSpans = payload.flatMap(trace => trace) + const resourceNames = testSpans.map(span => span.resource) + + assert.includeMembers(resourceNames, + [ + 'ci-visibility/test/ci-visibility-test.js.ci visibility can report tests', + 'ci-visibility/test/ci-visibility-test-2.js.ci visibility 2 can report tests 2' + ] + ) + }, ({ url }) => url === '/v0.4/traces').then(() => done()).catch(done) + + childProcess = fork(startupTestFile, { + cwd, + env: getCiVisEvpProxyConfig(receiver.port), + stdio: 'pipe' + }) + }) + }) + it('can report code coverage', (done) => { + const libraryConfigRequestPromise = receiver.payloadReceived( + ({ url }) => url === '/api/v2/libraries/tests/services/setting' + ) + const codeCovRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcov') + const eventsRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcycle') + + Promise.all([ + libraryConfigRequestPromise, + codeCovRequestPromise, + eventsRequestPromise + ]).then(([libraryConfigRequest, codeCovRequest, eventsRequest]) => { + assert.propertyVal(libraryConfigRequest.headers, 'dd-api-key', '1') + + const [coveragePayload] = codeCovRequest.payload + assert.propertyVal(codeCovRequest.headers, 'dd-api-key', '1') + + assert.propertyVal(coveragePayload, 'name', 'coverage1') + assert.propertyVal(coveragePayload, 'filename', 'coverage1.msgpack') + assert.propertyVal(coveragePayload, 'type', 'application/msgpack') + assert.include(coveragePayload.content, { + version: 2 + }) + const allCoverageFiles = codeCovRequest.payload + .flatMap(coverage => coverage.content.coverages) + .flatMap(file => file.files) + .map(file => file.filename) + + assert.includeMembers(allCoverageFiles, expectedCoverageFiles) + assert.exists(coveragePayload.content.coverages[0].test_session_id) + assert.exists(coveragePayload.content.coverages[0].test_suite_id) + + const testSession = eventsRequest.payload.events.find(event => event.type === 'test_session_end').content + assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) + + const eventTypes = eventsRequest.payload.events.map(event => event.type) + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + ) + assert.equal(numSuites, 2) + }).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'pipe' + } + ) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('exit', () => { + done() + }) + }) + + it('does not report code coverage if disabled by the API', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false + }) + + receiver.assertPayloadReceived(() => { + const error = new Error('it should not report code coverage') + done(error) + }, ({ url }) => url === '/api/v2/citestcov').catch(() => {}) + + receiver.assertPayloadReceived(({ headers, payload }) => { + assert.propertyVal(headers, 'dd-api-key', '1') + const eventTypes = payload.events.map(event => event.type) + assert.includeMembers(eventTypes, ['test', 'test_session_end', 'test_module_end', 'test_suite_end']) + const testSession = payload.events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'false') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'false') + assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) + const testModule = payload.events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'false') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'false') + }, ({ url }) => url === '/api/v2/citestcycle').then(() => done()).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + }) + + it('can skip suites received by the intelligent test runner API and still reports code coverage', (done) => { + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: 'ci-visibility/test/ci-visibility-test.js' + } + }]) + + const skippableRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/ci/tests/skippable') + const coverageRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcov') + const eventsRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcycle') + + Promise.all([ + skippableRequestPromise, + coverageRequestPromise, + eventsRequestPromise + ]).then(([skippableRequest, coverageRequest, eventsRequest]) => { + assert.propertyVal(skippableRequest.headers, 'dd-api-key', '1') + const [coveragePayload] = coverageRequest.payload + assert.propertyVal(coverageRequest.headers, 'dd-api-key', '1') + assert.propertyVal(coveragePayload, 'name', 'coverage1') + assert.propertyVal(coveragePayload, 'filename', 'coverage1.msgpack') + assert.propertyVal(coveragePayload, 'type', 'application/msgpack') + + assert.propertyVal(eventsRequest.headers, 'dd-api-key', '1') + const eventTypes = eventsRequest.payload.events.map(event => event.type) + const skippedSuite = eventsRequest.payload.events.find(event => + event.content.resource === 'test_suite.ci-visibility/test/ci-visibility-test.js' + ).content + assert.propertyVal(skippedSuite.meta, TEST_STATUS, 'skip') + assert.propertyVal(skippedSuite.meta, TEST_SKIPPED_BY_ITR, 'true') + + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + ) + assert.equal(numSuites, 2) + const testSession = eventsRequest.payload.events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'true') + assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_TYPE, 'suite') + assert.propertyVal(testSession.metrics, TEST_ITR_SKIPPING_COUNT, 1) + const testModule = eventsRequest.payload.events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'true') + assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_TYPE, 'suite') + assert.propertyVal(testModule.metrics, TEST_ITR_SKIPPING_COUNT, 1) + done() + }).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + }) + + it('marks the test session as skipped if every suite is skipped', (done) => { + receiver.setSuitesToSkip( + [ + { + type: 'suite', + attributes: { + suite: 'ci-visibility/test/ci-visibility-test.js' + } + }, + { + type: 'suite', + attributes: { + suite: 'ci-visibility/test/ci-visibility-test-2.js' + } + } + ] + ) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_STATUS, 'skip') + }) + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('does not skip tests if git metadata upload fails', (done) => { + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: 'ci-visibility/test/ci-visibility-test.js' + } + }]) + + receiver.setGitUploadStatus(404) + + receiver.assertPayloadReceived(() => { + const error = new Error('should not request skippable') + done(error) + }, ({ url }) => url === '/api/v2/ci/tests/skippable').catch(() => {}) + + receiver.assertPayloadReceived(({ headers, payload }) => { + assert.propertyVal(headers, 'dd-api-key', '1') + const eventTypes = payload.events.map(event => event.type) + // because they are not skipped + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + ) + assert.equal(numSuites, 2) + const testSession = payload.events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + const testModule = payload.events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + }, ({ url }) => url === '/api/v2/citestcycle').then(() => done()).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + }) + + it('does not skip tests if test skipping is disabled by the API', (done) => { + receiver.setSettings({ + itr_enabled: true, + code_coverage: true, + tests_skipping: false + }) + + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: 'ci-visibility/test/ci-visibility-test.js' + } + }]) + + receiver.assertPayloadReceived(() => { + const error = new Error('should not request skippable') + done(error) + }, ({ url }) => url === '/api/v2/ci/tests/skippable').catch(() => {}) + + receiver.assertPayloadReceived(({ headers, payload }) => { + assert.propertyVal(headers, 'dd-api-key', '1') + const eventTypes = payload.events.map(event => event.type) + // because they are not skipped + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + ) + assert.equal(numSuites, 2) + }, ({ url }) => url === '/api/v2/citestcycle').then(() => done()).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + }) + + it('does not skip suites if suite is marked as unskippable', (done) => { + receiver.setSuitesToSkip([ + { + type: 'suite', + attributes: { + suite: 'ci-visibility/unskippable-test/test-to-skip.js' + } + }, + { + type: 'suite', + attributes: { + suite: 'ci-visibility/unskippable-test/test-unskippable.js' + } + } + ]) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const suites = events.filter(event => event.type === 'test_suite_end') + + assert.equal(suites.length, 3) + + const testSession = events.find(event => event.type === 'test_session_end').content + const testModule = events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testSession.meta, TEST_ITR_FORCED_RUN, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_UNSKIPPABLE, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_FORCED_RUN, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_UNSKIPPABLE, 'true') + + const passedSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-to-run.js' + ) + const skippedSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-to-skip.js' + ) + const forcedToRunSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-unskippable.js' + ) + // It does not mark as unskippable if there is no docblock + assert.propertyVal(passedSuite.content.meta, TEST_STATUS, 'pass') + assert.notProperty(passedSuite.content.meta, TEST_ITR_UNSKIPPABLE) + assert.notProperty(passedSuite.content.meta, TEST_ITR_FORCED_RUN) + + assert.propertyVal(skippedSuite.content.meta, TEST_STATUS, 'skip') + assert.notProperty(skippedSuite.content.meta, TEST_ITR_UNSKIPPABLE) + assert.notProperty(skippedSuite.content.meta, TEST_ITR_FORCED_RUN) + + assert.propertyVal(forcedToRunSuite.content.meta, TEST_STATUS, 'pass') + assert.propertyVal(forcedToRunSuite.content.meta, TEST_ITR_UNSKIPPABLE, 'true') + assert.propertyVal(forcedToRunSuite.content.meta, TEST_ITR_FORCED_RUN, 'true') + }, 25000) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: 'unskippable-test/test-' + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('only sets forced to run if suite was going to be skipped by ITR', (done) => { + receiver.setSuitesToSkip([ + { + type: 'suite', + attributes: { + suite: 'ci-visibility/unskippable-test/test-to-skip.js' + } + } + ]) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const suites = events.filter(event => event.type === 'test_suite_end') + + assert.equal(suites.length, 3) + + const testSession = events.find(event => event.type === 'test_session_end').content + const testModule = events.find(event => event.type === 'test_module_end').content + assert.notProperty(testSession.meta, TEST_ITR_FORCED_RUN) + assert.propertyVal(testSession.meta, TEST_ITR_UNSKIPPABLE, 'true') + assert.notProperty(testModule.meta, TEST_ITR_FORCED_RUN) + assert.propertyVal(testModule.meta, TEST_ITR_UNSKIPPABLE, 'true') + + const passedSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-to-run.js' + ) + const skippedSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-to-skip.js' + ).content + const nonSkippedSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-unskippable.js' + ).content + + // It does not mark as unskippable if there is no docblock + assert.propertyVal(passedSuite.content.meta, TEST_STATUS, 'pass') + assert.notProperty(passedSuite.content.meta, TEST_ITR_UNSKIPPABLE) + assert.notProperty(passedSuite.content.meta, TEST_ITR_FORCED_RUN) + + assert.propertyVal(skippedSuite.meta, TEST_STATUS, 'skip') + + assert.propertyVal(nonSkippedSuite.meta, TEST_STATUS, 'pass') + assert.propertyVal(nonSkippedSuite.meta, TEST_ITR_UNSKIPPABLE, 'true') + // it was not forced to run because it wasn't going to be skipped + assert.notProperty(nonSkippedSuite.meta, TEST_ITR_FORCED_RUN) + }, 25000) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: 'unskippable-test/test-' + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('sets _dd.ci.itr.tests_skipped to false if the received suite is not skipped', (done) => { + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: 'ci-visibility/test/not-existing-test.js' + } + }]) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + const testModule = events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + }, 25000) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('reports itr_correlation_id in test suites', (done) => { + const itrCorrelationId = '4321' + receiver.setItrCorrelationId(itrCorrelationId) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSuites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) + testSuites.forEach(testSuite => { + assert.equal(testSuite.itr_correlation_id, itrCorrelationId) + }) + }, 25000) + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('can skip when using a custom test sequencer', (done) => { + receiver.setSettings({ + itr_enabled: true, + tests_skipping: true + }) + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: 'ci-visibility/test/ci-visibility-test.js' + } + }]) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testEvents = events.filter(event => event.type === 'test') + // no tests end up running (suite is skipped) + assert.equal(testEvents.length, 0) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'true') + + const skippedSuite = events.find(event => + event.content.resource === 'test_suite.ci-visibility/test/ci-visibility-test.js' + ).content + assert.propertyVal(skippedSuite.meta, TEST_STATUS, 'skip') + assert.propertyVal(skippedSuite.meta, TEST_SKIPPED_BY_ITR, 'true') + }) + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + CUSTOM_TEST_SEQUENCER: './ci-visibility/jest-custom-test-sequencer.js', + TEST_SHARD: '2/2' + }, + stdio: 'inherit' + } + ) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + + childProcess.on('exit', () => { + assert.include(testOutput, 'Running shard with a custom sequencer') + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('works with multi project setup and test skipping', (done) => { + receiver.setSettings({ + itr_enabled: true, + code_coverage: true, + tests_skipping: true + }) + + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: 'ci-visibility/test/ci-visibility-test.js' + } + }]) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + // suites for both projects in the multi-project config are reported as skipped + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSuites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) + + const skippedSuites = testSuites.filter( + suite => suite.resource === 'test_suite.ci-visibility/test/ci-visibility-test.js' + ) + assert.equal(skippedSuites.length, 2) + + skippedSuites.forEach(skippedSuite => { + assert.equal(skippedSuite.meta[TEST_STATUS], 'skip') + assert.equal(skippedSuite.meta[TEST_SKIPPED_BY_ITR], 'true') + }) + }) + + childProcess = exec( + 'node ./node_modules/jest/bin/jest --config config-jest-multiproject.js', + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('calculates executable lines even if there have been skipped suites', (done) => { + receiver.setSettings({ + itr_enabled: true, + code_coverage: true, + tests_skipping: true + }) + + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: 'ci-visibility/test-total-code-coverage/test-skipped.js' + } + }]) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + + // Before https://github.com/DataDog/dd-trace-js/pull/4336, this would've been 100% + // The reason is that skipping jest's `addUntestedFiles`, we would not see unexecuted lines. + // In this cause, these would be from the `unused-dependency.js` file. + // It is 50% now because we only cover 1 out of 2 files (`used-dependency.js`). + assert.propertyVal(testSession.metrics, TEST_CODE_COVERAGE_LINES_PCT, 50) + }) + + childProcess = exec( + runTestsWithCoverageCommand, // Requirement: the user must've opted in to code coverage + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: 'ci-visibility/test-total-code-coverage/test-', + COLLECT_COVERAGE_FROM: '**/test-total-code-coverage/**' + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(done).catch(done) + }) + }) + }) + + context('early flake detection', () => { + it('retries new tests', (done) => { + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + // Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new + receiver.setKnownTests({ + jest: { + 'ci-visibility/test/ci-visibility-test.js': ['ci visibility can report tests'] + } + }) + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 100 + } + }) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + // TODO: maybe check in stdout for the "Retried by Datadog" + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + // no other tests are considered new + const oldTests = tests.filter(test => + test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test.js' + ) + oldTests.forEach(test => { + assert.notProperty(test.meta, TEST_IS_NEW) + }) + assert.equal(oldTests.length, 1) + + const newTests = tests.filter(test => + test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test-2.js' + ) + newTests.forEach(test => { + assert.propertyVal(test.meta, TEST_IS_NEW, 'true') + }) + const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + // all but one has been retried + assert.equal( + newTests.length - 1, + retriedTests.length + ) + assert.equal(retriedTests.length, NUM_RETRIES_EFD) + // Test name does not change + newTests.forEach(test => { + assert.equal(test.meta[TEST_NAME], 'ci visibility 2 can report tests 2') + }) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { ...getCiVisEvpProxyConfig(receiver.port), TESTS_TO_RUN: 'test/ci-visibility-test' }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('handles parameterized tests as a single unit', (done) => { + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + // Tests from ci-visibility/test-early-flake-detection/test-parameterized.js will be considered new + receiver.setKnownTests({ + jest: { + 'ci-visibility/test-early-flake-detection/test.js': ['ci visibility can report tests'] + } + }) + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': 3 + }, + faulty_session_threshold: 100 + } + }) + + const parameterizedTestFile = 'test-parameterized.js' + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + + const newTests = tests.filter(test => + test.meta[TEST_SUITE] === `ci-visibility/test-early-flake-detection/${parameterizedTestFile}` + ) + newTests.forEach(test => { + assert.propertyVal(test.meta, TEST_IS_NEW, 'true') + }) + // Each parameter is repeated independently + const testsForFirstParameter = tests.filter(test => test.resource === + `ci-visibility/test-early-flake-detection/${parameterizedTestFile}.parameterized test parameter 1` + ) + + const testsForSecondParameter = tests.filter(test => test.resource === + `ci-visibility/test-early-flake-detection/${parameterizedTestFile}.parameterized test parameter 2` + ) + + assert.equal(testsForFirstParameter.length, testsForSecondParameter.length) + + // all but one have been retried + assert.equal( + testsForFirstParameter.length - 1, + testsForFirstParameter.filter(test => test.meta[TEST_IS_RETRY] === 'true').length + ) + + assert.equal( + testsForSecondParameter.length - 1, + testsForSecondParameter.filter(test => test.meta[TEST_IS_RETRY] === 'true').length + ) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { ...getCiVisEvpProxyConfig(receiver.port), TESTS_TO_RUN: 'test-early-flake-detection/test' }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('is disabled if DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED is false', (done) => { + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + // Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new + receiver.setKnownTests({ + jest: { + 'ci-visibility/test/ci-visibility-test.js': ['ci visibility can report tests'] + } + }) + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': 3 + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const newTests = tests.filter(test => + test.meta[TEST_IS_NEW] === 'true' + ) + // new tests are not detected + assert.equal(newTests.length, 0) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisEvpProxyConfig(receiver.port), + TESTS_TO_RUN: 'test/ci-visibility-test', + DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED: 'false' + }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('retries flaky tests', (done) => { + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + // Tests from ci-visibility/test/occasionally-failing-test will be considered new + receiver.setKnownTests({}) + + const NUM_RETRIES_EFD = 5 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + // all but one has been retried + assert.equal( + tests.length - 1, + retriedTests.length + ) + assert.equal(retriedTests.length, NUM_RETRIES_EFD) + // Out of NUM_RETRIES_EFD + 1 total runs, half will be passing and half will be failing, + // based on the global counter in the test file + const passingTests = tests.filter(test => test.meta[TEST_STATUS] === 'pass') + const failingTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(passingTests.length, (NUM_RETRIES_EFD + 1) / 2) + assert.equal(failingTests.length, (NUM_RETRIES_EFD + 1) / 2) + // Test name does not change + retriedTests.forEach(test => { + assert.equal(test.meta[TEST_NAME], 'fail occasionally fails') + }) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisEvpProxyConfig(receiver.port), + TESTS_TO_RUN: 'test-early-flake-detection/occasionally-failing-test' + }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + // TODO: check exit code: if a new, retried test fails, the exit code should remain 0 + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('does not retry new tests that are skipped', (done) => { + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + // Tests from ci-visibility/test/skipped-and-todo-test will be considered new + receiver.setKnownTests({}) + + const NUM_RETRIES_EFD = 5 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const newSkippedTests = tests.filter( + test => test.meta[TEST_NAME] === 'ci visibility skip will not be retried' + ) + assert.equal(newSkippedTests.length, 1) + assert.notProperty(newSkippedTests[0].meta, TEST_IS_RETRY) + + const newTodoTests = tests.filter( + test => test.meta[TEST_NAME] === 'ci visibility todo will not be retried' + ) + assert.equal(newTodoTests.length, 1) + assert.notProperty(newTodoTests[0].meta, TEST_IS_RETRY) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisEvpProxyConfig(receiver.port), + TESTS_TO_RUN: 'test-early-flake-detection/skipped-and-todo-test' + }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('handles spaces in test names', (done) => { + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': 3 + }, + faulty_session_threshold: 100 + } + }) + // Tests from ci-visibility/test/skipped-and-todo-test will be considered new + receiver.setKnownTests({ + jest: { + 'ci-visibility/test-early-flake-detection/weird-test-names.js': [ + 'no describe can do stuff', + 'describe trailing space ' + ] + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + assert.equal(tests.length, 2) + + const resourceNames = tests.map(test => test.resource) + + assert.includeMembers(resourceNames, + [ + 'ci-visibility/test-early-flake-detection/weird-test-names.js.no describe can do stuff', + 'ci-visibility/test-early-flake-detection/weird-test-names.js.describe trailing space ' + ] + ) + + const newTests = tests.filter( + test => test.meta[TEST_IS_NEW] === 'true' + ) + // no new tests + assert.equal(newTests.length, 0) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisEvpProxyConfig(receiver.port), + TESTS_TO_RUN: 'test-early-flake-detection/weird-test-names' + }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('does not run EFD if the known tests request fails', (done) => { + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + receiver.setKnownTestsResponseCode(500) + + const NUM_RETRIES_EFD = 5 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + assert.equal(tests.length, 2) + const newTests = tests.filter( + test => test.meta[TEST_IS_NEW] === 'true' + ) + assert.equal(newTests.length, 0) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisEvpProxyConfig(receiver.port), + TESTS_TO_RUN: 'test/ci-visibility-test' + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => done()).catch(done) + }) + }) + + it('retries flaky tests and sets exit code to 0 as long as one attempt passes', (done) => { + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + // Tests from ci-visibility/test/occasionally-failing-test will be considered new + receiver.setKnownTests({}) + + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + // all but one has been retried + assert.equal( + tests.length - 1, + retriedTests.length + ) + assert.equal(retriedTests.length, NUM_RETRIES_EFD) + // Out of NUM_RETRIES_EFD + 1 total runs, half will be passing and half will be failing, + // based on the global counter in the test file + const passingTests = tests.filter(test => test.meta[TEST_STATUS] === 'pass') + const failingTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(passingTests.length, (NUM_RETRIES_EFD + 1) / 2) + assert.equal(failingTests.length, (NUM_RETRIES_EFD + 1) / 2) + // Test name does not change + retriedTests.forEach(test => { + assert.equal(test.meta[TEST_NAME], 'fail occasionally fails') + }) + }) + + childProcess = exec( + 'node ./node_modules/jest/bin/jest --config config-jest.js', + { + cwd, + env: { + ...getCiVisEvpProxyConfig(receiver.port), + TESTS_TO_RUN: '**/ci-visibility/test-early-flake-detection/occasionally-failing-test*' + }, + stdio: 'inherit' + } + ) + + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + + childProcess.on('exit', (exitCode) => { + assert.include(testOutput, '2 failed, 2 passed') + assert.equal(exitCode, 0) + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('does not run early flake detection on snapshot tests', (done) => { + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + // Tests from ci-visibility/test-early-flake-detection/jest-snapshot.js will be considered new + // but we don't retry them because they have snapshots + receiver.setKnownTests({}) + + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + assert.equal(tests.length, 1) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + + assert.equal(retriedTests.length, 0) + + // we still detect that it's new + const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') + assert.equal(newTests.length, 1) + }) + + childProcess = exec(runTestsWithCoverageCommand, { + cwd, + env: { + ...getCiVisEvpProxyConfig(receiver.port), + TESTS_TO_RUN: 'ci-visibility/test-early-flake-detection/jest-snapshot', + CI: '1' // needs to be run as CI so snapshots are not written + }, + stdio: 'inherit' + }) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('bails out of EFD if the percentage of new tests is too high', (done) => { + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + // Tests from ci-visibility/test/ci-visibility-test* will be considered new + receiver.setKnownTests({}) + + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 1 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ABORT_REASON, 'faulty') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + assert.equal(tests.length, 2) + + const newTests = tests.filter( + test => test.meta[TEST_IS_NEW] === 'true' + ) + // no new tests + assert.equal(newTests.length, 0) + }) + + childProcess = exec(runTestsWithCoverageCommand, { + cwd, + env: { + ...getCiVisEvpProxyConfig(receiver.port), + TESTS_TO_RUN: 'test/ci-visibility-test' + }, + stdio: 'inherit' + }) + + childProcess.on('exit', () => { + eventsPromise.then(() => done()).catch(done) + }) + }) + + it('works with jsdom', (done) => { + // Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new + receiver.setKnownTests({ + jest: { + 'ci-visibility/test/ci-visibility-test.js': ['ci visibility can report tests'] + } + }) + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + // no other tests are considered new + const oldTests = tests.filter(test => + test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test.js' + ) + oldTests.forEach(test => { + assert.notProperty(test.meta, TEST_IS_NEW) + }) + assert.equal(oldTests.length, 1) + + const newTests = tests.filter(test => + test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test-2.js' + ) + newTests.forEach(test => { + assert.propertyVal(test.meta, TEST_IS_NEW, 'true') + }) + const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + // all but one has been retried + assert.equal( + newTests.length - 1, + retriedTests.length + ) + assert.equal(retriedTests.length, NUM_RETRIES_EFD) + // Test name does not change + newTests.forEach(test => { + assert.equal(test.meta[TEST_NAME], 'ci visibility 2 can report tests 2') + }) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), // use agentless for this test, just for variety + TESTS_TO_RUN: 'test/ci-visibility-test', + ENABLE_JSDOM: true, + DD_TRACE_DEBUG: 1, + DD_TRACE_LOG_LEVEL: 'warn' + }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + }) + + context('flaky test retries', () => { + it('retries failed tests automatically', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + flaky_test_retries_enabled: true, + early_flake_detection: { + enabled: false + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + assert.equal(tests.length, 10) + assert.includeMembers(tests.map(test => test.resource), [ + // does not retry + 'ci-visibility/jest-flaky/flaky-passes.js.test-flaky-test-retries will not retry passed tests', + 'ci-visibility/jest-flaky/flaky-passes.js.test-flaky-test-retries can retry flaky tests', + 'ci-visibility/jest-flaky/flaky-passes.js.test-flaky-test-retries can retry flaky tests', + // retries twice and passes + 'ci-visibility/jest-flaky/flaky-passes.js.test-flaky-test-retries can retry flaky tests', + 'ci-visibility/jest-flaky/flaky-fails.js.test-flaky-test-retries can retry failed tests', + 'ci-visibility/jest-flaky/flaky-fails.js.test-flaky-test-retries can retry failed tests', + 'ci-visibility/jest-flaky/flaky-fails.js.test-flaky-test-retries can retry failed tests', + 'ci-visibility/jest-flaky/flaky-fails.js.test-flaky-test-retries can retry failed tests', + 'ci-visibility/jest-flaky/flaky-fails.js.test-flaky-test-retries can retry failed tests', + // retries up to 5 times and still fails + 'ci-visibility/jest-flaky/flaky-fails.js.test-flaky-test-retries can retry failed tests' + ]) + + const eventuallyPassingTest = tests.filter( + test => test.resource === + 'ci-visibility/jest-flaky/flaky-passes.js.test-flaky-test-retries can retry flaky tests' + ) + assert.equal(eventuallyPassingTest.length, 3) + assert.equal(eventuallyPassingTest.filter(test => test.meta[TEST_STATUS] === 'fail').length, 2) + assert.equal(eventuallyPassingTest.filter(test => test.meta[TEST_STATUS] === 'pass').length, 1) + assert.equal(eventuallyPassingTest.filter(test => test.meta[TEST_IS_RETRY] === 'true').length, 2) + + const neverPassingTest = tests.filter( + test => test.resource === + 'ci-visibility/jest-flaky/flaky-fails.js.test-flaky-test-retries can retry failed tests' + ) + assert.equal(neverPassingTest.length, 6) + assert.equal(neverPassingTest.filter(test => test.meta[TEST_STATUS] === 'fail').length, 6) + assert.equal(neverPassingTest.filter(test => test.meta[TEST_STATUS] === 'pass').length, 0) + assert.equal(neverPassingTest.filter(test => test.meta[TEST_IS_RETRY] === 'true').length, 5) + + const testSuites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) + + const passingSuite = testSuites.find( + suite => suite.resource === 'test_suite.ci-visibility/jest-flaky/flaky-passes.js' + ) + assert.equal(passingSuite.meta[TEST_STATUS], 'pass') + + const failedSuite = testSuites.find( + suite => suite.resource === 'test_suite.ci-visibility/jest-flaky/flaky-fails.js' + ) + assert.equal(failedSuite.meta[TEST_STATUS], 'fail') + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisEvpProxyConfig(receiver.port), + TESTS_TO_RUN: 'jest-flaky/flaky-' + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('is disabled if DD_CIVISIBILITY_FLAKY_RETRY_ENABLED is false', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + flaky_test_retries_enabled: true, + early_flake_detection: { + enabled: false + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + assert.equal(tests.length, 3) + assert.includeMembers(tests.map(test => test.resource), [ + // does not retry anything + 'ci-visibility/jest-flaky/flaky-passes.js.test-flaky-test-retries will not retry passed tests', + 'ci-visibility/jest-flaky/flaky-passes.js.test-flaky-test-retries can retry flaky tests', + 'ci-visibility/jest-flaky/flaky-fails.js.test-flaky-test-retries can retry failed tests' + ]) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + + assert.equal(retriedTests.length, 0) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisEvpProxyConfig(receiver.port), + TESTS_TO_RUN: 'jest-flaky/flaky-', + DD_CIVISIBILITY_FLAKY_RETRY_ENABLED: 'false' + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('retries DD_CIVISIBILITY_FLAKY_RETRY_COUNT times', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + flaky_test_retries_enabled: true, + early_flake_detection: { + enabled: false + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + assert.equal(tests.length, 5) + // only one retry + assert.includeMembers(tests.map(test => test.resource), [ + 'ci-visibility/jest-flaky/flaky-passes.js.test-flaky-test-retries will not retry passed tests', + 'ci-visibility/jest-flaky/flaky-passes.js.test-flaky-test-retries can retry flaky tests', + 'ci-visibility/jest-flaky/flaky-passes.js.test-flaky-test-retries can retry flaky tests', + 'ci-visibility/jest-flaky/flaky-fails.js.test-flaky-test-retries can retry failed tests', + 'ci-visibility/jest-flaky/flaky-fails.js.test-flaky-test-retries can retry failed tests' + ]) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisEvpProxyConfig(receiver.port), + TESTS_TO_RUN: 'jest-flaky/flaky-', + DD_CIVISIBILITY_FLAKY_RETRY_COUNT: 1 + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => done()).catch(done) + }) + }) + }) +}) diff --git a/integration-tests/memory-leak/index.js b/integration-tests/memory-leak/index.js new file mode 100644 index 00000000000..01d4c2c439e --- /dev/null +++ b/integration-tests/memory-leak/index.js @@ -0,0 +1,15 @@ +const tracer = require('../../') +tracer.init() + +const http = require('http') + +http.createServer((req, res) => { + const delay = Math.random() < 0.01 // 1% + ? 61 * 1000 // over 1 minute + : Math.random() * 1000 // random 0 - 1s + + setTimeout(() => { + res.write('Hello World!') + res.end() + }, delay) +}).listen(8080) diff --git a/integration-tests/mocha/mocha.spec.js b/integration-tests/mocha/mocha.spec.js new file mode 100644 index 00000000000..3fa11871204 --- /dev/null +++ b/integration-tests/mocha/mocha.spec.js @@ -0,0 +1,2108 @@ +'use strict' + +const { fork, exec } = require('child_process') +const path = require('path') + +const { assert } = require('chai') + +const { + createSandbox, + getCiVisAgentlessConfig, + getCiVisEvpProxyConfig +} = require('../helpers') +const { FakeCiVisIntake } = require('../ci-visibility-intake') +const { + TEST_CODE_COVERAGE_ENABLED, + TEST_ITR_SKIPPING_ENABLED, + TEST_ITR_TESTS_SKIPPED, + TEST_CODE_COVERAGE_LINES_PCT, + TEST_SUITE, + TEST_STATUS, + TEST_SKIPPED_BY_ITR, + TEST_ITR_SKIPPING_TYPE, + TEST_ITR_SKIPPING_COUNT, + TEST_ITR_UNSKIPPABLE, + TEST_ITR_FORCED_RUN, + TEST_SOURCE_FILE, + TEST_IS_NEW, + TEST_IS_RETRY, + TEST_EARLY_FLAKE_ENABLED, + TEST_NAME, + TEST_COMMAND, + TEST_MODULE, + MOCHA_IS_PARALLEL, + TEST_SOURCE_START, + TEST_CODE_OWNERS, + TEST_SESSION_NAME, + TEST_LEVEL_EVENT_TYPES, + TEST_EARLY_FLAKE_ABORT_REASON +} = require('../../packages/dd-trace/src/plugins/util/test') +const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') +const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants') + +const runTestsWithCoverageCommand = './node_modules/nyc/bin/nyc.js -r=text-summary node ./ci-visibility/run-mocha.js' +const testFile = 'ci-visibility/run-mocha.js' +const expectedStdout = '2 passing' +const extraStdout = 'end event: can add event listeners to mocha' + +describe('mocha CommonJS', function () { + let receiver + let childProcess + let sandbox + let cwd + let startupTestFile + let testOutput = '' + + before(async function () { + sandbox = await createSandbox(['mocha', 'chai@v4', 'nyc', 'mocha-each', 'workerpool'], true) + cwd = sandbox.folder + startupTestFile = path.join(cwd, testFile) + }) + + after(async function () { + await sandbox.remove() + }) + + beforeEach(async function () { + receiver = await new FakeCiVisIntake().start() + }) + + afterEach(async () => { + childProcess.kill() + testOutput = '' + await receiver.stop() + }) + + it('can run tests and report tests with the APM protocol (old agents)', (done) => { + receiver.setInfoResponse({ endpoints: [] }) + receiver.payloadReceived(({ url }) => url === '/v0.4/traces').then(({ payload }) => { + const testSpans = payload.flatMap(trace => trace) + const resourceNames = testSpans.map(span => span.resource) + + assert.includeMembers(resourceNames, + [ + 'ci-visibility/test/ci-visibility-test.js.ci visibility can report tests', + 'ci-visibility/test/ci-visibility-test-2.js.ci visibility 2 can report tests 2' + ] + ) + + const areAllTestSpans = testSpans.every(span => span.name === 'mocha.test') + assert.isTrue(areAllTestSpans) + + assert.include(testOutput, expectedStdout) + + if (extraStdout) { + assert.include(testOutput, extraStdout) + } + // Can read DD_TAGS + testSpans.forEach(testSpan => { + assert.propertyVal(testSpan.meta, 'test.customtag', 'customvalue') + assert.propertyVal(testSpan.meta, 'test.customtag2', 'customvalue2') + }) + + testSpans.forEach(testSpan => { + assert.equal(testSpan.meta[TEST_SOURCE_FILE].startsWith('ci-visibility/test/ci-visibility-test'), true) + assert.exists(testSpan.metrics[TEST_SOURCE_START]) + }) + + done() + }) + + childProcess = fork(startupTestFile, { + cwd, + env: { + DD_TRACE_AGENT_PORT: receiver.port, + NODE_OPTIONS: '-r dd-trace/ci/init', + DD_TAGS: 'test.customtag:customvalue,test.customtag2:customvalue2' + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + }) + + const nonLegacyReportingOptions = ['agentless', 'evp proxy'] + + nonLegacyReportingOptions.forEach((reportingOption) => { + it(`can run and report tests with ${reportingOption}`, (done) => { + const envVars = reportingOption === 'agentless' + ? getCiVisAgentlessConfig(receiver.port) + : getCiVisEvpProxyConfig(receiver.port) + if (reportingOption === 'evp proxy') { + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + } + receiver.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('citestcycle'), (payloads) => { + const metadataDicts = payloads.flatMap(({ payload }) => payload.metadata) + + metadataDicts.forEach(metadata => { + for (const testLevel of TEST_LEVEL_EVENT_TYPES) { + assert.equal(metadata[testLevel][TEST_SESSION_NAME], 'my-test-session') + } + }) + + const events = payloads.flatMap(({ payload }) => payload.events) + const sessionEventContent = events.find(event => event.type === 'test_session_end').content + const moduleEventContent = events.find(event => event.type === 'test_module_end').content + const suites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const resourceNames = tests.map(span => span.resource) + + assert.includeMembers(resourceNames, + [ + 'ci-visibility/test/ci-visibility-test.js.ci visibility can report tests', + 'ci-visibility/test/ci-visibility-test-2.js.ci visibility 2 can report tests 2' + ] + ) + assert.equal(suites.length, 2) + assert.exists(sessionEventContent) + assert.exists(moduleEventContent) + + assert.include(testOutput, expectedStdout) + assert.include(testOutput, extraStdout) + + tests.forEach(testEvent => { + assert.equal(testEvent.meta[TEST_SOURCE_FILE].startsWith('ci-visibility/test/ci-visibility-test'), true) + assert.exists(testEvent.metrics[TEST_SOURCE_START]) + // Can read DD_TAGS + assert.propertyVal(testEvent.meta, 'test.customtag', 'customvalue') + assert.propertyVal(testEvent.meta, 'test.customtag2', 'customvalue2') + assert.exists(testEvent.metrics[DD_HOST_CPU_COUNT]) + }) + + suites.forEach(testSuite => { + assert.isTrue(testSuite.meta[TEST_SOURCE_FILE].startsWith('ci-visibility/test/ci-visibility-test')) + assert.equal(testSuite.metrics[TEST_SOURCE_START], 1) + assert.exists(testSuite.metrics[DD_HOST_CPU_COUNT]) + }) + + done() + }) + + childProcess = fork(startupTestFile, { + cwd, + env: { + ...envVars, + DD_TAGS: 'test.customtag:customvalue,test.customtag2:customvalue2', + DD_TEST_SESSION_NAME: 'my-test-session' + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + }) + }) + + const envVarSettings = ['DD_TRACING_ENABLED', 'DD_TRACE_ENABLED'] + + envVarSettings.forEach(envVar => { + context(`when ${envVar}=false`, () => { + it('does not report spans but still runs tests', (done) => { + receiver.assertMessageReceived(() => { + done(new Error('Should not create spans')) + }).catch(() => {}) + + childProcess = fork(startupTestFile, { + cwd, + env: { + DD_TRACE_AGENT_PORT: receiver.port, + NODE_OPTIONS: '-r dd-trace/ci/init', + [envVar]: 'false' + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('message', () => { + assert.include(testOutput, expectedStdout) + done() + }) + }) + }) + }) + + context('when no ci visibility init is used', () => { + it('does not crash', (done) => { + childProcess = fork(startupTestFile, { + cwd, + env: { + DD_TRACE_AGENT_PORT: receiver.port, + NODE_OPTIONS: '-r dd-trace/init' + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('message', () => { + assert.notInclude(testOutput, 'TypeError') + assert.notInclude(testOutput, 'Uncaught error outside test suite') + assert.include(testOutput, expectedStdout) + done() + }) + }) + }) + + it('correctly calculates test code owners when working directory is not repository root', (done) => { + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const test = events.find(event => event.type === 'test').content + const testSuite = events.find(event => event.type === 'test_suite_end').content + // The test is in a subproject + assert.notEqual(test.meta[TEST_SOURCE_FILE], test.meta[TEST_SUITE]) + assert.equal(test.meta[TEST_CODE_OWNERS], JSON.stringify(['@datadog-dd-trace-js'])) + assert.equal(testSuite.meta[TEST_CODE_OWNERS], JSON.stringify(['@datadog-dd-trace-js'])) + }) + + childProcess = exec( + 'node ../../node_modules/mocha/bin/mocha subproject-test.js', + { + cwd: `${cwd}/ci-visibility/subproject`, + env: { + ...getCiVisAgentlessConfig(receiver.port) + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('does not change mocha config if CI Visibility fails to init', (done) => { + receiver.assertPayloadReceived(() => { + const error = new Error('it should not report tests') + done(error) + }, ({ url }) => url === '/api/v2/citestcycle', 3000).catch(() => {}) + + const { DD_CIVISIBILITY_AGENTLESS_URL, ...restEnvVars } = getCiVisAgentlessConfig(receiver.port) + + // `runMocha` is only executed when using the CLI, which is where we modify mocha config + // if CI Visibility is init + childProcess = exec('mocha ./ci-visibility/test/ci-visibility-test.js', { + cwd, + env: { + ...restEnvVars, + DD_TRACE_DEBUG: 1, + DD_TRACE_LOG_LEVEL: 'error', + DD_SITE: '= invalid = url' + }, + stdio: 'pipe' + }) + + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('exit', () => { + assert.include(testOutput, 'Invalid URL') + assert.include(testOutput, '1 passing') // we only run one file here + done() + }) + }) + + it('works with parallel mode', (done) => { + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const metadataDicts = payloads.flatMap(({ payload }) => payload.metadata) + + metadataDicts.forEach(metadata => { + for (const testLevel of TEST_LEVEL_EVENT_TYPES) { + assert.equal(metadata[testLevel][TEST_SESSION_NAME], 'my-test-session') + } + }) + + const events = payloads.flatMap(({ payload }) => payload.events) + const sessionEventContent = events.find(event => event.type === 'test_session_end').content + const moduleEventContent = events.find(event => event.type === 'test_module_end').content + const suites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + assert.equal(sessionEventContent.meta[MOCHA_IS_PARALLEL], 'true') + assert.equal( + sessionEventContent.test_session_id.toString(10), + moduleEventContent.test_session_id.toString(10) + ) + suites.forEach(({ + meta, + test_suite_id: testSuiteId, + test_module_id: testModuleId, + test_session_id: testSessionId + }) => { + assert.exists(meta[TEST_COMMAND]) + assert.exists(meta[TEST_MODULE]) + assert.exists(testSuiteId) + assert.equal(testModuleId.toString(10), moduleEventContent.test_module_id.toString(10)) + assert.equal(testSessionId.toString(10), moduleEventContent.test_session_id.toString(10)) + }) + + tests.forEach(({ + meta, + metrics, + test_suite_id: testSuiteId, + test_module_id: testModuleId, + test_session_id: testSessionId + }) => { + assert.exists(meta[TEST_COMMAND]) + assert.exists(meta[TEST_MODULE]) + assert.exists(testSuiteId) + assert.equal(testModuleId.toString(10), moduleEventContent.test_module_id.toString(10)) + assert.equal(testSessionId.toString(10), moduleEventContent.test_session_id.toString(10)) + assert.propertyVal(meta, MOCHA_IS_PARALLEL, 'true') + assert.exists(metrics[TEST_SOURCE_START]) + }) + }) + + childProcess = fork(testFile, { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + RUN_IN_PARALLEL: true, + DD_TRACE_DEBUG: 1, + DD_TRACE_LOG_LEVEL: 'warn', + DD_TEST_SESSION_NAME: 'my-test-session' + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('message', () => { + eventsPromise.then(() => { + assert.notInclude(testOutput, 'TypeError') + assert.notInclude( + testOutput, 'Unable to initialize CI Visibility because Mocha is running in parallel mode.' + ) + done() + }).catch(done) + }) + }) + + it('works with parallel mode when run with the cli', (done) => { + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const sessionEventContent = events.find(event => event.type === 'test_session_end').content + const suites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + assert.equal(sessionEventContent.meta[MOCHA_IS_PARALLEL], 'true') + assert.equal(suites.length, 2) + assert.equal(tests.length, 2) + }) + + childProcess = exec('mocha --parallel --jobs 2 ./ci-visibility/test/ci-visibility-test*', { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('exit', () => { + eventsPromise.then(() => { + assert.notInclude(testOutput, 'TypeError') + assert.notInclude( + testOutput, 'Unable to initialize CI Visibility because Mocha is running in parallel mode.' + ) + done() + }).catch(done) + }) + }) + + it('does not blow up when workerpool is used outside of a test', (done) => { + childProcess = exec('node ./ci-visibility/run-workerpool.js', { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('exit', (code) => { + assert.include(testOutput, 'result 7') + assert.equal(code, 0) + done() + }) + }) + + it('reports errors in test sessions', (done) => { + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_STATUS, 'fail') + const errorMessage = 'Failed tests: 1' + assert.include(testSession.meta[ERROR_MESSAGE], errorMessage) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './test/fail-test.js' + ]) + }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('does not init if DD_API_KEY is not set', (done) => { + receiver.assertMessageReceived(() => { + done(new Error('Should not create spans')) + }).catch(() => {}) + + childProcess = fork(startupTestFile, { + cwd, + env: { + DD_CIVISIBILITY_AGENTLESS_ENABLED: 1, + NODE_OPTIONS: '-r dd-trace/ci/init' + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('message', () => { + assert.include(testOutput, expectedStdout) + assert.include(testOutput, 'DD_CIVISIBILITY_AGENTLESS_ENABLED is set, ' + + 'but neither DD_API_KEY nor DATADOG_API_KEY are set in your environment, ' + + 'so dd-trace will not be initialized.' + ) + done() + }) + }) + + it('can report git metadata', (done) => { + const searchCommitsRequestPromise = receiver.payloadReceived( + ({ url }) => url === '/api/v2/git/repository/search_commits' + ) + const packfileRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/git/repository/packfile') + const eventsRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcycle') + + Promise.all([ + searchCommitsRequestPromise, + packfileRequestPromise, + eventsRequestPromise + ]).then(([searchCommitRequest, packfileRequest, eventsRequest]) => { + assert.propertyVal(searchCommitRequest.headers, 'dd-api-key', '1') + assert.propertyVal(packfileRequest.headers, 'dd-api-key', '1') + + const eventTypes = eventsRequest.payload.events.map(event => event.type) + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + ) + assert.equal(numSuites, 2) + + done() + }).catch(done) + + childProcess = fork(startupTestFile, { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'pipe' + }) + }) + + context('intelligent test runner', () => { + context('if the agent is not event platform proxy compatible', () => { + it('does not do any intelligent test runner request', (done) => { + receiver.setInfoResponse({ endpoints: [] }) + + receiver.assertPayloadReceived(() => { + const error = new Error('should not request search_commits') + done(error) + }, ({ url }) => url === '/evp_proxy/v2/api/v2/git/repository/search_commits').catch(() => {}) + receiver.assertPayloadReceived(() => { + const error = new Error('should not request search_commits') + done(error) + }, ({ url }) => url === '/api/v2/git/repository/search_commits').catch(() => {}) + receiver.assertPayloadReceived(() => { + const error = new Error('should not request setting') + done(error) + }, ({ url }) => url === '/api/v2/libraries/tests/services/setting').catch(() => {}) + receiver.assertPayloadReceived(() => { + const error = new Error('should not request setting') + done(error) + }, ({ url }) => url === '/evp_proxy/v2/api/v2/libraries/tests/services/setting').catch(() => {}) + + receiver.assertPayloadReceived(({ payload }) => { + const testSpans = payload.flatMap(trace => trace) + const resourceNames = testSpans.map(span => span.resource) + + assert.includeMembers(resourceNames, + [ + 'ci-visibility/test/ci-visibility-test.js.ci visibility can report tests', + 'ci-visibility/test/ci-visibility-test-2.js.ci visibility 2 can report tests 2' + ] + ) + }, ({ url }) => url === '/v0.4/traces').then(() => done()).catch(done) + + childProcess = fork(startupTestFile, { + cwd, + env: getCiVisEvpProxyConfig(receiver.port), + stdio: 'pipe' + }) + }) + }) + it('can report code coverage', (done) => { + let testOutput + const libraryConfigRequestPromise = receiver.payloadReceived( + ({ url }) => url === '/api/v2/libraries/tests/services/setting' + ) + const codeCovRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcov') + const eventsRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcycle') + + Promise.all([ + libraryConfigRequestPromise, + codeCovRequestPromise, + eventsRequestPromise + ]).then(([libraryConfigRequest, codeCovRequest, eventsRequest]) => { + assert.propertyVal(libraryConfigRequest.headers, 'dd-api-key', '1') + + const [coveragePayload] = codeCovRequest.payload + assert.propertyVal(codeCovRequest.headers, 'dd-api-key', '1') + + assert.propertyVal(coveragePayload, 'name', 'coverage1') + assert.propertyVal(coveragePayload, 'filename', 'coverage1.msgpack') + assert.propertyVal(coveragePayload, 'type', 'application/msgpack') + assert.include(coveragePayload.content, { + version: 2 + }) + const allCoverageFiles = codeCovRequest.payload + .flatMap(coverage => coverage.content.coverages) + .flatMap(file => file.files) + .map(file => file.filename) + + assert.includeMembers(allCoverageFiles, + [ + 'ci-visibility/test/sum.js', + 'ci-visibility/test/ci-visibility-test.js', + 'ci-visibility/test/ci-visibility-test-2.js' + ] + ) + assert.exists(coveragePayload.content.coverages[0].test_session_id) + assert.exists(coveragePayload.content.coverages[0].test_suite_id) + + const testSession = eventsRequest.payload.events.find(event => event.type === 'test_session_end').content + assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) + + const eventTypes = eventsRequest.payload.events.map(event => event.type) + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + ) + assert.equal(numSuites, 2) + }).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'pipe' + } + ) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('exit', () => { + // coverage report + assert.include(testOutput, 'Lines ') + done() + }) + }) + + it('does not report code coverage if disabled by the API', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false + }) + + receiver.assertPayloadReceived(() => { + const error = new Error('it should not report code coverage') + done(error) + }, ({ url }) => url === '/api/v2/citestcov').catch(() => {}) + + receiver.assertPayloadReceived(({ headers, payload }) => { + assert.propertyVal(headers, 'dd-api-key', '1') + const eventTypes = payload.events.map(event => event.type) + assert.includeMembers(eventTypes, ['test', 'test_session_end', 'test_module_end', 'test_suite_end']) + const testSession = payload.events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'false') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'false') + assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) + const testModule = payload.events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'false') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'false') + }, ({ url }) => url === '/api/v2/citestcycle').then(() => done()).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + }) + + it('can skip suites received by the intelligent test runner API and still reports code coverage', (done) => { + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: 'ci-visibility/test/ci-visibility-test.js' + } + }]) + + const skippableRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/ci/tests/skippable') + const coverageRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcov') + const eventsRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcycle') + + Promise.all([ + skippableRequestPromise, + coverageRequestPromise, + eventsRequestPromise + ]).then(([skippableRequest, coverageRequest, eventsRequest]) => { + assert.propertyVal(skippableRequest.headers, 'dd-api-key', '1') + const [coveragePayload] = coverageRequest.payload + assert.propertyVal(coverageRequest.headers, 'dd-api-key', '1') + assert.propertyVal(coveragePayload, 'name', 'coverage1') + assert.propertyVal(coveragePayload, 'filename', 'coverage1.msgpack') + assert.propertyVal(coveragePayload, 'type', 'application/msgpack') + + assert.propertyVal(eventsRequest.headers, 'dd-api-key', '1') + const eventTypes = eventsRequest.payload.events.map(event => event.type) + const skippedSuite = eventsRequest.payload.events.find(event => + event.content.resource === 'test_suite.ci-visibility/test/ci-visibility-test.js' + ).content + assert.propertyVal(skippedSuite.meta, TEST_STATUS, 'skip') + assert.propertyVal(skippedSuite.meta, TEST_SKIPPED_BY_ITR, 'true') + + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + ) + assert.equal(numSuites, 2) + const testSession = eventsRequest.payload.events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'true') + assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_TYPE, 'suite') + assert.propertyVal(testSession.metrics, TEST_ITR_SKIPPING_COUNT, 1) + const testModule = eventsRequest.payload.events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'true') + assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_TYPE, 'suite') + assert.propertyVal(testModule.metrics, TEST_ITR_SKIPPING_COUNT, 1) + done() + }).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + }) + + it('marks the test session as skipped if every suite is skipped', (done) => { + receiver.setSuitesToSkip( + [ + { + type: 'suite', + attributes: { + suite: 'ci-visibility/test/ci-visibility-test.js' + } + }, + { + type: 'suite', + attributes: { + suite: 'ci-visibility/test/ci-visibility-test-2.js' + } + } + ] + ) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_STATUS, 'skip') + }) + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('does not skip tests if git metadata upload fails', (done) => { + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: 'ci-visibility/test/ci-visibility-test.js' + } + }]) + + receiver.setGitUploadStatus(404) + + receiver.assertPayloadReceived(() => { + const error = new Error('should not request skippable') + done(error) + }, ({ url }) => url === '/api/v2/ci/tests/skippable').catch(() => {}) + + receiver.assertPayloadReceived(({ headers, payload }) => { + assert.propertyVal(headers, 'dd-api-key', '1') + const eventTypes = payload.events.map(event => event.type) + // because they are not skipped + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + ) + assert.equal(numSuites, 2) + const testSession = payload.events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + const testModule = payload.events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + }, ({ url }) => url === '/api/v2/citestcycle').then(() => done()).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + }) + + it('does not skip tests if test skipping is disabled by the API', (done) => { + receiver.setSettings({ + itr_enabled: true, + code_coverage: true, + tests_skipping: false + }) + + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: 'ci-visibility/test/ci-visibility-test.js' + } + }]) + + receiver.assertPayloadReceived(() => { + const error = new Error('should not request skippable') + done(error) + }, ({ url }) => url === '/api/v2/ci/tests/skippable').catch(() => {}) + + receiver.assertPayloadReceived(({ headers, payload }) => { + assert.propertyVal(headers, 'dd-api-key', '1') + const eventTypes = payload.events.map(event => event.type) + // because they are not skipped + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + ) + assert.equal(numSuites, 2) + }, ({ url }) => url === '/api/v2/citestcycle').then(() => done()).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + }) + + it('does not skip suites if suite is marked as unskippable', (done) => { + receiver.setSuitesToSkip([ + { + type: 'suite', + attributes: { + suite: 'ci-visibility/unskippable-test/test-to-skip.js' + } + }, + { + type: 'suite', + attributes: { + suite: 'ci-visibility/unskippable-test/test-unskippable.js' + } + } + ]) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const suites = events.filter(event => event.type === 'test_suite_end') + + assert.equal(suites.length, 3) + + const testSession = events.find(event => event.type === 'test_session_end').content + const testModule = events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testSession.meta, TEST_ITR_FORCED_RUN, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_UNSKIPPABLE, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_FORCED_RUN, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_UNSKIPPABLE, 'true') + + const passedSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-to-run.js' + ) + const skippedSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-to-skip.js' + ) + const forcedToRunSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-unskippable.js' + ) + // It does not mark as unskippable if there is no docblock + assert.propertyVal(passedSuite.content.meta, TEST_STATUS, 'pass') + assert.notProperty(passedSuite.content.meta, TEST_ITR_UNSKIPPABLE) + assert.notProperty(passedSuite.content.meta, TEST_ITR_FORCED_RUN) + + assert.propertyVal(skippedSuite.content.meta, TEST_STATUS, 'skip') + assert.notProperty(skippedSuite.content.meta, TEST_ITR_UNSKIPPABLE) + assert.notProperty(skippedSuite.content.meta, TEST_ITR_FORCED_RUN) + + assert.propertyVal(forcedToRunSuite.content.meta, TEST_STATUS, 'pass') + assert.propertyVal(forcedToRunSuite.content.meta, TEST_ITR_UNSKIPPABLE, 'true') + assert.propertyVal(forcedToRunSuite.content.meta, TEST_ITR_FORCED_RUN, 'true') + }, 25000) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './unskippable-test/test-to-run.js', + './unskippable-test/test-to-skip.js', + './unskippable-test/test-unskippable.js' + ]) + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('only sets forced to run if suite was going to be skipped by ITR', (done) => { + receiver.setSuitesToSkip([ + { + type: 'suite', + attributes: { + suite: 'ci-visibility/unskippable-test/test-to-skip.js' + } + } + ]) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const suites = events.filter(event => event.type === 'test_suite_end') + + assert.equal(suites.length, 3) + + const testSession = events.find(event => event.type === 'test_session_end').content + const testModule = events.find(event => event.type === 'test_module_end').content + assert.notProperty(testSession.meta, TEST_ITR_FORCED_RUN) + assert.propertyVal(testSession.meta, TEST_ITR_UNSKIPPABLE, 'true') + assert.notProperty(testModule.meta, TEST_ITR_FORCED_RUN) + assert.propertyVal(testModule.meta, TEST_ITR_UNSKIPPABLE, 'true') + + const passedSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-to-run.js' + ) + const skippedSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-to-skip.js' + ).content + const nonSkippedSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-unskippable.js' + ).content + + // It does not mark as unskippable if there is no docblock + assert.propertyVal(passedSuite.content.meta, TEST_STATUS, 'pass') + assert.notProperty(passedSuite.content.meta, TEST_ITR_UNSKIPPABLE) + assert.notProperty(passedSuite.content.meta, TEST_ITR_FORCED_RUN) + + assert.propertyVal(skippedSuite.meta, TEST_STATUS, 'skip') + + assert.propertyVal(nonSkippedSuite.meta, TEST_STATUS, 'pass') + assert.propertyVal(nonSkippedSuite.meta, TEST_ITR_UNSKIPPABLE, 'true') + // it was not forced to run because it wasn't going to be skipped + assert.notProperty(nonSkippedSuite.meta, TEST_ITR_FORCED_RUN) + }, 25000) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './unskippable-test/test-to-run.js', + './unskippable-test/test-to-skip.js', + './unskippable-test/test-unskippable.js' + ]) + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('sets _dd.ci.itr.tests_skipped to false if the received suite is not skipped', (done) => { + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: 'ci-visibility/test/not-existing-test.js' + } + }]) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + const testModule = events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + }, 25000) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('reports itr_correlation_id in test suites', (done) => { + const itrCorrelationId = '4321' + receiver.setItrCorrelationId(itrCorrelationId) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSuites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) + testSuites.forEach(testSuite => { + assert.equal(testSuite.itr_correlation_id, itrCorrelationId) + }) + }, 25000) + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + }) + + context('early flake detection', () => { + it('retries new tests', (done) => { + // Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new + receiver.setKnownTests({ + mocha: { + 'ci-visibility/test/ci-visibility-test.js': ['ci visibility can report tests'] + } + }) + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 100 + } + }) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + // TODO: maybe check in stdout for the "Retried by Datadog" + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + // no other tests are considered new + const oldTests = tests.filter(test => + test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test.js' + ) + oldTests.forEach(test => { + assert.notProperty(test.meta, TEST_IS_NEW) + }) + assert.equal(oldTests.length, 1) + + const newTests = tests.filter(test => + test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test-2.js' + ) + newTests.forEach(test => { + assert.propertyVal(test.meta, TEST_IS_NEW, 'true') + }) + const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + // all but one has been retried + assert.equal( + newTests.length - 1, + retriedTests.length + ) + assert.equal(retriedTests.length, NUM_RETRIES_EFD) + // Test name does not change + newTests.forEach(test => { + assert.equal(test.meta[TEST_NAME], 'ci visibility 2 can report tests 2') + }) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './test/ci-visibility-test.js', + './test/ci-visibility-test-2.js' + ]) + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('handles parameterized tests as a single unit', (done) => { + // Tests from ci-visibility/test-early-flake-detection/test-parameterized.js will be considered new + receiver.setKnownTests({ + mocha: { + 'ci-visibility/test-early-flake-detection/test.js': ['ci visibility can report tests'] + } + }) + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': 3 + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + + const newTests = tests.filter(test => + test.meta[TEST_SUITE] === 'ci-visibility/test-early-flake-detection/mocha-parameterized.js' + ) + newTests.forEach(test => { + assert.propertyVal(test.meta, TEST_IS_NEW, 'true') + }) + // Each parameter is repeated independently + const testsForFirstParameter = tests.filter(test => test.resource === + 'ci-visibility/test-early-flake-detection/mocha-parameterized.js.parameterized test parameter 1' + ) + + const testsForSecondParameter = tests.filter(test => test.resource === + 'ci-visibility/test-early-flake-detection/mocha-parameterized.js.parameterized test parameter 2' + ) + + assert.equal(testsForFirstParameter.length, testsForSecondParameter.length) + + // all but one have been retried + assert.equal( + testsForFirstParameter.length - 1, + testsForFirstParameter.filter(test => test.meta[TEST_IS_RETRY] === 'true').length + ) + + assert.equal( + testsForSecondParameter.length - 1, + testsForSecondParameter.filter(test => test.meta[TEST_IS_RETRY] === 'true').length + ) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './test-early-flake-detection/test.js', + './test-early-flake-detection/mocha-parameterized.js' + ]) + }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('is disabled if DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED is false', (done) => { + // Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new + receiver.setKnownTests({ + mocha: { + 'ci-visibility/test/ci-visibility-test.js': ['ci visibility can report tests'] + } + }) + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': 3 + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const newTests = tests.filter(test => + test.meta[TEST_IS_NEW] === 'true' + ) + // new tests are not detected + assert.equal(newTests.length, 0) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './test/ci-visibility-test.js', + './test/ci-visibility-test-2.js' + ]), + DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED: 'false' + }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('retries flaky tests', (done) => { + // Tests from ci-visibility/test/occasionally-failing-test will be considered new + receiver.setKnownTests({}) + + const NUM_RETRIES_EFD = 5 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + // all but one has been retried + assert.equal( + tests.length - 1, + retriedTests.length + ) + assert.equal(retriedTests.length, NUM_RETRIES_EFD) + // Out of NUM_RETRIES_EFD + 1 total runs, half will be passing and half will be failing, + // based on the global counter in the test file + const passingTests = tests.filter(test => test.meta[TEST_STATUS] === 'pass') + const failingTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(passingTests.length, (NUM_RETRIES_EFD + 1) / 2) + assert.equal(failingTests.length, (NUM_RETRIES_EFD + 1) / 2) + // Test name does not change + retriedTests.forEach(test => { + assert.equal(test.meta[TEST_NAME], 'fail occasionally fails') + }) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './test-early-flake-detection/occasionally-failing-test.js' + ]) + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', (exitCode) => { + // TODO: check exit code: if a new, retried test fails, the exit code should remain 0 + eventsPromise.then(() => { + assert.equal(exitCode, 0) + done() + }).catch(done) + }) + }) + + it('does not retry new tests that are skipped', (done) => { + // Tests from ci-visibility/test/skipped-and-todo-test will be considered new + receiver.setKnownTests({}) + + const NUM_RETRIES_EFD = 5 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const newSkippedTests = tests.filter( + test => test.meta[TEST_NAME] === 'ci visibility skip will not be retried' + ) + assert.equal(newSkippedTests.length, 1) + assert.notProperty(newSkippedTests[0].meta, TEST_IS_RETRY) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './test-early-flake-detection/skipped-and-todo-test.js' + ]) + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('handles spaces in test names', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': 3 + }, + faulty_session_threshold: 100 + } + }) + // Tests from ci-visibility/test/skipped-and-todo-test will be considered new + receiver.setKnownTests({ + mocha: { + 'ci-visibility/test-early-flake-detection/weird-test-names.js': [ + 'no describe can do stuff', + 'describe trailing space ' + ] + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + assert.equal(tests.length, 2) + + const resourceNames = tests.map(test => test.resource) + + assert.includeMembers(resourceNames, + [ + 'ci-visibility/test-early-flake-detection/weird-test-names.js.no describe can do stuff', + 'ci-visibility/test-early-flake-detection/weird-test-names.js.describe trailing space ' + ] + ) + + const newTests = tests.filter( + test => test.meta[TEST_IS_NEW] === 'true' + ) + // no new tests + assert.equal(newTests.length, 0) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './test-early-flake-detection/weird-test-names.js' + ]) + }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('does not run EFD if the known tests request fails', (done) => { + receiver.setKnownTestsResponseCode(500) + + const NUM_RETRIES_EFD = 5 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + assert.equal(tests.length, 2) + const newTests = tests.filter( + test => test.meta[TEST_IS_NEW] === 'true' + ) + assert.equal(newTests.length, 0) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './test/ci-visibility-test.js', + './test/ci-visibility-test-2.js' + ]) + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => done()).catch(done) + }) + }) + + it('retries flaky tests and sets exit code to 0 as long as one attempt passes', (done) => { + // Tests from ci-visibility/test/occasionally-failing-test will be considered new + receiver.setKnownTests({}) + + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + // all but one has been retried + assert.equal( + tests.length - 1, + retriedTests.length + ) + assert.equal(retriedTests.length, NUM_RETRIES_EFD) + // Out of NUM_RETRIES_EFD + 1 total runs, half will be passing and half will be failing, + // based on the global counter in the test file + const passingTests = tests.filter(test => test.meta[TEST_STATUS] === 'pass') + const failingTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(passingTests.length, (NUM_RETRIES_EFD + 1) / 2) + assert.equal(failingTests.length, (NUM_RETRIES_EFD + 1) / 2) + // Test name does not change + retriedTests.forEach(test => { + assert.equal(test.meta[TEST_NAME], 'fail occasionally fails') + }) + }) + + childProcess = exec( + 'node ./node_modules/mocha/bin/mocha ci-visibility/test-early-flake-detection/occasionally-failing-test*', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: '**/ci-visibility/test-early-flake-detection/occasionally-failing-test*' + }, + stdio: 'inherit' + } + ) + + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + + childProcess.on('exit', (exitCode) => { + assert.include(testOutput, '2 passing') + assert.include(testOutput, '2 failing') + assert.equal(exitCode, 0) + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('bails out of EFD if the percentage of new tests is too high', (done) => { + const NUM_RETRIES_EFD = 5 + + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 0 + } + }) + // Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new + receiver.setKnownTests({ + mocha: { + 'ci-visibility/test/ci-visibility-test.js': ['ci visibility can report tests'] + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ABORT_REASON, 'faulty') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') + assert.equal(newTests.length, 0) + + const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 0) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './test/ci-visibility-test.js', + './test/ci-visibility-test-2.js' + ]) + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + context('parallel mode', () => { + it('retries new tests', (done) => { + // Tests from ci-visibility/test/occasionally-failing-test will be considered new + receiver.setKnownTests({}) + + const NUM_RETRIES_EFD = 5 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + assert.propertyVal(testSession.meta, MOCHA_IS_PARALLEL, 'true') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + // all but one has been retried + assert.equal( + tests.length - 1, + retriedTests.length + ) + assert.equal(retriedTests.length, NUM_RETRIES_EFD) + // Out of NUM_RETRIES_EFD + 1 total runs, half will be passing and half will be failing, + // based on the global counter in the test file + const passingTests = tests.filter(test => test.meta[TEST_STATUS] === 'pass') + const failingTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(passingTests.length, (NUM_RETRIES_EFD + 1) / 2) + assert.equal(failingTests.length, (NUM_RETRIES_EFD + 1) / 2) + // Test name does not change + retriedTests.forEach(test => { + assert.equal(test.meta[TEST_NAME], 'fail occasionally fails') + }) + }) + + childProcess = exec( + 'mocha --parallel ./ci-visibility/test-early-flake-detection/occasionally-failing-test.js', { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + }) + + childProcess.on('exit', (exitCode) => { + eventsPromise.then(() => { + assert.equal(exitCode, 0) + done() + }).catch(done) + }) + }) + it('retries new tests when using the programmatic API', (done) => { + // Tests from ci-visibility/test/occasionally-failing-test will be considered new + receiver.setKnownTests({}) + + const NUM_RETRIES_EFD = 5 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + assert.propertyVal(testSession.meta, MOCHA_IS_PARALLEL, 'true') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + // all but one has been retried + assert.equal( + tests.length - 1, + retriedTests.length + ) + assert.equal(retriedTests.length, NUM_RETRIES_EFD) + // Out of NUM_RETRIES_EFD + 1 total runs, half will be passing and half will be failing, + // based on the global counter in the test file + const passingTests = tests.filter(test => test.meta[TEST_STATUS] === 'pass') + const failingTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(passingTests.length, (NUM_RETRIES_EFD + 1) / 2) + assert.equal(failingTests.length, (NUM_RETRIES_EFD + 1) / 2) + // Test name does not change + retriedTests.forEach(test => { + assert.equal(test.meta[TEST_NAME], 'fail occasionally fails') + }) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + RUN_IN_PARALLEL: true, + TESTS_TO_RUN: JSON.stringify([ + './test-early-flake-detection/occasionally-failing-test.js' + ]) + }, + stdio: 'inherit' + } + ) + childProcess.on('exit', (exitCode) => { + eventsPromise.then(() => { + assert.equal(exitCode, 0) + done() + }).catch(done) + }) + }) + it('bails out of EFD if the percentage of new tests is too high', (done) => { + const NUM_RETRIES_EFD = 5 + + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 0 + } + }) + // Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new + receiver.setKnownTests({ + mocha: { + 'ci-visibility/test/ci-visibility-test.js': ['ci visibility can report tests'] + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ABORT_REASON, 'faulty') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') + assert.equal(newTests.length, 0) + + const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 0) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + RUN_IN_PARALLEL: true, + TESTS_TO_RUN: JSON.stringify([ + './test/ci-visibility-test.js', + './test/ci-visibility-test-2.js' + ]) + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + }) + }) + + context('auto test retries', () => { + it('retries failed tests automatically', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + flaky_test_retries_enabled: true, + early_flake_detection: { + enabled: false + } + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './test-flaky-test-retries/eventually-passing-test.js' + ]) + }, + stdio: 'inherit' + } + ) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + assert.equal(tests.length, 3) // two failed retries and then the pass + + const failedAttempts = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(failedAttempts.length, 2) + + failedAttempts.forEach((failedTest, index) => { + assert.include(failedTest.meta[ERROR_MESSAGE], `expected ${index + 1} to equal 3`) + }) + + // The first attempt is not marked as a retry + const retriedFailure = failedAttempts.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedFailure.length, 1) + + const passedAttempt = tests.find(test => test.meta[TEST_STATUS] === 'pass') + assert.equal(passedAttempt.meta[TEST_IS_RETRY], 'true') + }) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('is disabled if DD_CIVISIBILITY_FLAKY_RETRY_ENABLED is false', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + flaky_test_retries_enabled: true, + early_flake_detection: { + enabled: false + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + assert.equal(tests.length, 1) + + const retries = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retries.length, 0) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './test-flaky-test-retries/eventually-passing-test.js' + ]), + DD_CIVISIBILITY_FLAKY_RETRY_ENABLED: 'false' + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('retries DD_CIVISIBILITY_FLAKY_RETRY_COUNT times', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + flaky_test_retries_enabled: true, + early_flake_detection: { + enabled: false + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + assert.equal(tests.length, 2) // one retry + + const failedAttempts = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(failedAttempts.length, 2) + + const retriedFailure = failedAttempts.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedFailure.length, 1) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './test-flaky-test-retries/eventually-passing-test.js' + ]), + DD_CIVISIBILITY_FLAKY_RETRY_COUNT: 1 + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => done()).catch(done) + }) + }) + }) + + it('takes into account untested files if "all" is passed to nyc', (done) => { + const linePctMatchRegex = /Lines\s*:\s*(\d+)%/ + let linePctMatch + let linesPctFromNyc = 0 + let codeCoverageWithUntestedFiles = 0 + let codeCoverageWithoutUntestedFiles = 0 + + let eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + codeCoverageWithUntestedFiles = testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT] + }) + + childProcess = exec( + './node_modules/nyc/bin/nyc.js -r=text-summary --all --nycrc-path ./my-nyc.config.js ' + + 'node ./node_modules/mocha/bin/mocha.js ./ci-visibility/test/ci-visibility-test.js', + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + + childProcess.on('exit', () => { + linePctMatch = testOutput.match(linePctMatchRegex) + linesPctFromNyc = linePctMatch ? Number(linePctMatch[1]) : null + + assert.equal( + linesPctFromNyc, + codeCoverageWithUntestedFiles, + 'nyc --all output does not match the reported coverage' + ) + + // reset test output for next test session + testOutput = '' + // we run the same tests without the all flag + childProcess = exec( + './node_modules/nyc/bin/nyc.js -r=text-summary --nycrc-path ./my-nyc.config.js ' + + 'node ./node_modules/mocha/bin/mocha.js ./ci-visibility/test/ci-visibility-test.js', + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + + eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + codeCoverageWithoutUntestedFiles = testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT] + }) + + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + + childProcess.on('exit', () => { + linePctMatch = testOutput.match(linePctMatchRegex) + linesPctFromNyc = linePctMatch ? Number(linePctMatch[1]) : null + + assert.equal( + linesPctFromNyc, + codeCoverageWithoutUntestedFiles, + 'nyc output does not match the reported coverage (no --all flag)' + ) + + eventsPromise.then(() => { + assert.isAbove(codeCoverageWithoutUntestedFiles, codeCoverageWithUntestedFiles) + done() + }).catch(done) + }) + }) + }) +}) diff --git a/integration-tests/my-nyc.config.js b/integration-tests/my-nyc.config.js new file mode 100644 index 00000000000..b0d1235ecd2 --- /dev/null +++ b/integration-tests/my-nyc.config.js @@ -0,0 +1,5 @@ +// non default name so that it only gets picked up intentionally +module.exports = { + exclude: ['node_modules/**'], + include: process.env.NYC_INCLUDE ? JSON.parse(process.env.NYC_INCLUDE) : ['ci-visibility/test/**'] +} diff --git a/integration-tests/opentelemetry.spec.js b/integration-tests/opentelemetry.spec.js index c05b28520b3..ee307568cb4 100644 --- a/integration-tests/opentelemetry.spec.js +++ b/integration-tests/opentelemetry.spec.js @@ -5,6 +5,7 @@ const { fork } = require('child_process') const { join } = require('path') const { assert } = require('chai') const { satisfies } = require('semver') +const axios = require('axios') function check (agent, proc, timeout, onMessage = () => { }, isMetrics) { const messageReceiver = isMetrics @@ -56,7 +57,11 @@ describe('opentelemetry', () => { before(async () => { const dependencies = [ - '@opentelemetry/api' + '@opentelemetry/api@1.8.0', + '@opentelemetry/instrumentation', + '@opentelemetry/instrumentation-http', + '@opentelemetry/instrumentation-express', + 'express' ] if (satisfies(process.version.slice(1), '>=14')) { dependencies.push('@opentelemetry/sdk-node') @@ -74,6 +79,214 @@ describe('opentelemetry', () => { await sandbox.remove() }) + it('should not capture telemetry DD and OTEL vars dont conflict', () => { + proc = fork(join(cwd, 'opentelemetry/basic.js'), { + cwd, + env: { + DD_TRACE_AGENT_PORT: agent.port, + DD_TRACE_OTEL_ENABLED: 1, + DD_TELEMETRY_HEARTBEAT_INTERVAL: 1, + TIMEOUT: 1500, + DD_SERVICE: 'service', + DD_TRACE_LOG_LEVEL: 'error', + DD_TRACE_SAMPLE_RATE: '0.5', + DD_TRACE_ENABLED: 'true', + DD_RUNTIME_METRICS_ENABLED: 'true', + DD_TAGS: 'foo:bar,baz:qux', + DD_TRACE_PROPAGATION_STYLE: 'datadog' + } + }) + + return check(agent, proc, timeout, ({ payload }) => { + assert.strictEqual(payload.request_type, 'generate-metrics') + + const metrics = payload.payload + assert.strictEqual(metrics.namespace, 'tracers') + + const otelHiding = metrics.series.filter(({ metric }) => metric === 'otel.env.hiding') + const otelInvalid = metrics.series.filter(({ metric }) => metric === 'otel.env.invalid') + + assert.strictEqual(otelHiding.length, 0) + assert.strictEqual(otelInvalid.length, 0) + }, true) + }) + + it('should capture telemetry if both DD and OTEL env vars are set', () => { + proc = fork(join(cwd, 'opentelemetry/basic.js'), { + cwd, + env: { + DD_TRACE_AGENT_PORT: agent.port, + DD_TRACE_OTEL_ENABLED: 1, + DD_TELEMETRY_HEARTBEAT_INTERVAL: 1, + TIMEOUT: 1500, + DD_SERVICE: 'service', + OTEL_SERVICE_NAME: 'otel_service', + DD_TRACE_LOG_LEVEL: 'error', + OTEL_LOG_LEVEL: 'debug', + DD_TRACE_SAMPLE_RATE: '0.5', + OTEL_TRACES_SAMPLER: 'traceidratio', + OTEL_TRACES_SAMPLER_ARG: '1.0', + DD_TRACE_ENABLED: 'true', + OTEL_TRACES_EXPORTER: 'none', + DD_RUNTIME_METRICS_ENABLED: 'true', + OTEL_METRICS_EXPORTER: 'none', + DD_TAGS: 'foo:bar,baz:qux', + OTEL_RESOURCE_ATTRIBUTES: 'foo+bar13baz+qux1', + DD_TRACE_PROPAGATION_STYLE: 'datadog, tracecontext', + OTEL_PROPAGATORS: 'datadog, tracecontext', + OTEL_LOGS_EXPORTER: 'none', + OTEL_SDK_DISABLED: 'false' + } + }) + + return check(agent, proc, timeout, ({ payload }) => { + assert.strictEqual(payload.request_type, 'generate-metrics') + + const metrics = payload.payload + + assert.strictEqual(metrics.namespace, 'tracers') + + const otelHiding = metrics.series.filter(({ metric }) => metric === 'otel.env.hiding') + const otelInvalid = metrics.series.filter(({ metric }) => metric === 'otel.env.invalid') + assert.strictEqual(otelHiding.length, 9) + assert.strictEqual(otelInvalid.length, 0) + + assert.deepStrictEqual(otelHiding[0].tags, [ + 'config_datadog:dd_trace_log_level', 'config_opentelemetry:otel_log_level', + `version:${process.version}` + ]) + assert.deepStrictEqual(otelHiding[1].tags, [ + 'config_datadog:dd_trace_propagation_style', 'config_opentelemetry:otel_propagators', + `version:${process.version}` + ]) + assert.deepStrictEqual(otelHiding[2].tags, [ + 'config_datadog:dd_service', 'config_opentelemetry:otel_service_name', + `version:${process.version}` + ]) + + assert.deepStrictEqual(otelHiding[3].tags, [ + 'config_datadog:dd_trace_sample_rate', 'config_opentelemetry:otel_traces_sampler', `version:${process.version}` + ]) + + assert.deepStrictEqual(otelHiding[4].tags, [ + 'config_datadog:dd_trace_sample_rate', 'config_opentelemetry:otel_traces_sampler_arg', + `version:${process.version}` + ]) + + assert.deepStrictEqual(otelHiding[5].tags, [ + 'config_datadog:dd_trace_enabled', 'config_opentelemetry:otel_traces_exporter', + `version:${process.version}` + ]) + + assert.deepStrictEqual(otelHiding[6].tags, [ + 'config_datadog:dd_runtime_metrics_enabled', 'config_opentelemetry:otel_metrics_exporter', + `version:${process.version}` + ]) + + assert.deepStrictEqual(otelHiding[7].tags, [ + 'config_datadog:dd_tags', 'config_opentelemetry:otel_resource_attributes', + `version:${process.version}` + ]) + + assert.deepStrictEqual(otelHiding[8].tags, [ + 'config_datadog:dd_trace_otel_enabled', 'config_opentelemetry:otel_sdk_disabled', + `version:${process.version}` + ]) + + for (const metric of otelHiding) { + assert.strictEqual(metric.points[0][1], 1) + } + }, true) + }) + + it('should capture telemetry when OTEL env vars are invalid', () => { + proc = fork(join(cwd, 'opentelemetry/basic.js'), { + cwd, + env: { + DD_TRACE_AGENT_PORT: agent.port, + DD_TRACE_OTEL_ENABLED: 1, + DD_TELEMETRY_HEARTBEAT_INTERVAL: 1, + TIMEOUT: 1500, + OTEL_SERVICE_NAME: 'otel_service', + OTEL_LOG_LEVEL: 'foo', + OTEL_TRACES_SAMPLER: 'foo', + OTEL_TRACES_SAMPLER_ARG: 'foo', + OTEL_TRACES_EXPORTER: 'foo', + OTEL_METRICS_EXPORTER: 'foo', + OTEL_RESOURCE_ATTRIBUTES: 'foo', + OTEL_PROPAGATORS: 'foo', + OTEL_LOGS_EXPORTER: 'foo', + OTEL_SDK_DISABLED: 'foo' + } + }) + + return check(agent, proc, timeout, ({ payload }) => { + assert.strictEqual(payload.request_type, 'generate-metrics') + + const metrics = payload.payload + + assert.strictEqual(metrics.namespace, 'tracers') + + const otelHiding = metrics.series.filter(({ metric }) => metric === 'otel.env.hiding') + const otelInvalid = metrics.series.filter(({ metric }) => metric === 'otel.env.invalid') + + assert.strictEqual(otelHiding.length, 1) + assert.strictEqual(otelInvalid.length, 8) + + assert.deepStrictEqual(otelHiding[0].tags, [ + 'config_datadog:dd_trace_otel_enabled', 'config_opentelemetry:otel_sdk_disabled', + `version:${process.version}` + ]) + + assert.deepStrictEqual(otelInvalid[0].tags, [ + 'config_datadog:dd_trace_log_level', 'config_opentelemetry:otel_log_level', + `version:${process.version}` + ]) + + assert.deepStrictEqual(otelInvalid[1].tags, [ + 'config_datadog:dd_trace_sample_rate', + 'config_opentelemetry:otel_traces_sampler', + `version:${process.version}` + ]) + + assert.deepStrictEqual(otelInvalid[2].tags, [ + 'config_datadog:dd_trace_sample_rate', + 'config_opentelemetry:otel_traces_sampler_arg', + `version:${process.version}` + ]) + assert.deepStrictEqual(otelInvalid[3].tags, [ + 'config_datadog:dd_trace_enabled', 'config_opentelemetry:otel_traces_exporter', + `version:${process.version}` + ]) + + assert.deepStrictEqual(otelInvalid[4].tags, [ + 'config_datadog:dd_runtime_metrics_enabled', + 'config_opentelemetry:otel_metrics_exporter', + `version:${process.version}` + ]) + + assert.deepStrictEqual(otelInvalid[5].tags, [ + 'config_datadog:dd_trace_otel_enabled', 'config_opentelemetry:otel_sdk_disabled', + `version:${process.version}` + ]) + + assert.deepStrictEqual(otelInvalid[6].tags, [ + 'config_opentelemetry:otel_logs_exporter', + `version:${process.version}` + ]) + + assert.deepStrictEqual(otelInvalid[7].tags, [ + 'config_datadog:dd_trace_propagation_style', + 'config_opentelemetry:otel_propagators', + `version:${process.version}` + ]) + + for (const metric of otelInvalid) { + assert.strictEqual(metric.points[0][1], 1) + } + }, true) + }) + it('should start a trace in isolation', async () => { proc = fork(join(cwd, 'opentelemetry/basic.js'), { cwd, @@ -110,8 +323,8 @@ describe('opentelemetry', () => { const metrics = payload.payload assert.strictEqual(metrics.namespace, 'tracers') - const spanCreated = metrics.series.find(({ metric }) => metric === 'span_created') - const spanFinished = metrics.series.find(({ metric }) => metric === 'span_finished') + const spanCreated = metrics.series.find(({ metric }) => metric === 'spans_created') + const spanFinished = metrics.series.find(({ metric }) => metric === 'spans_finished') // Validate common fields between start and finish for (const series of [spanCreated, spanFinished]) { @@ -135,6 +348,52 @@ describe('opentelemetry', () => { }, true) }) + it('should capture auto-instrumentation telemetry', async () => { + const SERVER_PORT = 6666 + proc = fork(join(cwd, 'opentelemetry/auto-instrumentation.js'), { + cwd, + env: { + DD_TRACE_AGENT_PORT: agent.port, + DD_TRACE_OTEL_ENABLED: 1, + SERVER_PORT, + DD_TRACE_DISABLED_INSTRUMENTATIONS: 'http,dns,express,net', + DD_TELEMETRY_HEARTBEAT_INTERVAL: 1 + } + }) + await new Promise(resolve => setTimeout(resolve, 1000)) // Adjust the delay as necessary + await axios.get(`http://localhost:${SERVER_PORT}/first-endpoint`) + + return check(agent, proc, 10000, ({ payload }) => { + assert.strictEqual(payload.request_type, 'generate-metrics') + + const metrics = payload.payload + assert.strictEqual(metrics.namespace, 'tracers') + + const spanCreated = metrics.series.find(({ metric }) => metric === 'spans_created') + const spanFinished = metrics.series.find(({ metric }) => metric === 'spans_finished') + + // Validate common fields between start and finish + for (const series of [spanCreated, spanFinished]) { + assert.ok(series) + + assert.strictEqual(series.points.length, 1) + assert.strictEqual(series.points[0].length, 2) + + const [ts, value] = series.points[0] + assert.ok(nearNow(ts, Date.now() / 1e3)) + assert.strictEqual(value, 9) + + assert.strictEqual(series.type, 'count') + assert.strictEqual(series.common, true) + assert.deepStrictEqual(series.tags, [ + 'integration_name:otel.library', + 'otel_enabled:true', + `version:${process.version}` + ]) + } + }, true) + }) + it('should work within existing datadog-traced http request', async () => { proc = fork(join(cwd, 'opentelemetry/server.js'), { cwd, @@ -163,6 +422,56 @@ describe('opentelemetry', () => { }) }) + it('should work with otel express & http auto instrumentation', async () => { + const SERVER_PORT = 6666 + proc = fork(join(cwd, 'opentelemetry/auto-instrumentation.js'), { + cwd, + env: { + DD_TRACE_AGENT_PORT: agent.port, + DD_TRACE_OTEL_ENABLED: 1, + SERVER_PORT, + DD_TRACE_DISABLED_INSTRUMENTATIONS: 'http,dns,express,net' + } + }) + await new Promise(resolve => setTimeout(resolve, 1000)) // Adjust the delay as necessary + await axios.get(`http://localhost:${SERVER_PORT}/first-endpoint`) + + return check(agent, proc, 10000, ({ payload }) => { + assert.strictEqual(payload.length, 2) + // combine the traces + const trace = payload.flat() + assert.strictEqual(trace.length, 9) + + // Should have expected span names and ordering + assert.isTrue(eachEqual(trace, [ + 'GET /second-endpoint', + 'middleware - query', + 'middleware - expressInit', + 'request handler - /second-endpoint', + 'GET /first-endpoint', + 'middleware - query', + 'middleware - expressInit', + 'request handler - /first-endpoint', + 'GET' + ], + (span) => span.name)) + + assert.isTrue(allEqual(trace, (span) => { + span.trace_id.toString() + })) + + const [get3, query2, init2, handler2, get1, query1, init1, handler1, get2] = trace + isChildOf(query1, get1) + isChildOf(init1, get1) + isChildOf(handler1, get1) + isChildOf(get2, get1) + isChildOf(get3, get2) + isChildOf(query2, get3) + isChildOf(init2, get3) + isChildOf(handler2, get3) + }) + }) + if (satisfies(process.version.slice(1), '>=14')) { it('should auto-instrument @opentelemetry/sdk-node', async () => { proc = fork(join(cwd, 'opentelemetry/env-var.js'), { @@ -184,3 +493,9 @@ describe('opentelemetry', () => { }) } }) + +function isChildOf (childSpan, parentSpan) { + assert.strictEqual(childSpan.trace_id.toString(), parentSpan.trace_id.toString()) + assert.notStrictEqual(childSpan.span_id.toString(), parentSpan.span_id.toString()) + assert.strictEqual(childSpan.parent_id.toString(), parentSpan.span_id.toString()) +} diff --git a/integration-tests/opentelemetry/auto-instrumentation.js b/integration-tests/opentelemetry/auto-instrumentation.js new file mode 100644 index 00000000000..8a1ba5c2c77 --- /dev/null +++ b/integration-tests/opentelemetry/auto-instrumentation.js @@ -0,0 +1,55 @@ +const tracer = require('dd-trace').init() +const { TracerProvider } = tracer +const provider = new TracerProvider() +provider.register() + +const { registerInstrumentations } = require('@opentelemetry/instrumentation') +const { HttpInstrumentation } = require('@opentelemetry/instrumentation-http') +const { ExpressInstrumentation } = require('@opentelemetry/instrumentation-express') + +registerInstrumentations({ + instrumentations: [ + new HttpInstrumentation({ + ignoreIncomingRequestHook (req) { + // Ignore spans from static assets. + return req.path === '/v0.4/traces' || req.path === '/v0.7/config' || + req.path === '/telemetry/proxy/api/v2/apmtelemetry' + }, + ignoreOutgoingRequestHook (req) { + // Ignore spans from static assets. + return req.path === '/v0.4/traces' || req.path === '/v0.7/config' || + req.path === '/telemetry/proxy/api/v2/apmtelemetry' + } + }), + new ExpressInstrumentation() + ], + tracerProvider: provider +}) + +const express = require('express') +const http = require('http') +const app = express() +const PORT = process.env.SERVER_PORT + +app.get('/second-endpoint', (req, res) => { + res.send('Response from second endpoint') + server.close(() => { + }) +}) + +app.get('/first-endpoint', async (req, res) => { + try { + const response = await new Promise((resolve, reject) => { + http.get(`http://localhost:${PORT}/second-endpoint`).on('finish', (response) => { + resolve(response) + }).on('error', (error) => { + reject(error) + }) + }) + res.send(`First endpoint received: ${response}`) + } catch (error) { + res.status(500).send(`Error occurred while making nested call ${error}`) + } +}) + +const server = app.listen(PORT, () => {}) diff --git a/integration-tests/package-guardrails.spec.js b/integration-tests/package-guardrails.spec.js new file mode 100644 index 00000000000..4ee05c033cb --- /dev/null +++ b/integration-tests/package-guardrails.spec.js @@ -0,0 +1,113 @@ +const { + runAndCheckWithTelemetry: testFile, + useEnv, + useSandbox, + sandboxCwd +} = require('./helpers') +const path = require('path') +const fs = require('fs') +const assert = require('assert') + +const NODE_OPTIONS = '--require dd-trace/init.js' +const DD_TRACE_DEBUG = 'true' +const DD_INJECTION_ENABLED = 'tracing' +const DD_LOG_LEVEL = 'error' + +// These are on by default in release tests, so we'll turn them off for +// more fine-grained control of these variables in these tests. +delete process.env.DD_INJECTION_ENABLED +delete process.env.DD_INJECT_FORCE + +describe('package guardrails', () => { + useEnv({ NODE_OPTIONS }) + const runTest = (...args) => + testFile('package-guardrails/index.js', ...args) + + context('when package is out of range', () => { + useSandbox(['bluebird@1.0.0']) + context('with DD_INJECTION_ENABLED', () => { + useEnv({ DD_INJECTION_ENABLED }) + it('should not instrument the package, and send telemetry', () => + runTest('false\n', + 'complete', 'injection_forced:false', + 'abort.integration', 'integration:bluebird,integration_version:1.0.0' + )) + }) + context('with logging disabled', () => { + it('should not instrument the package', () => runTest('false\n')) + }) + context('with logging enabled', () => { + useEnv({ DD_TRACE_DEBUG }) + it('should not instrument the package', () => + runTest(`Application instrumentation bootstrapping complete +Found incompatible integration version: bluebird@1.0.0 +false +`)) + }) + }) + + context('when package is in range', () => { + context('when bluebird is 2.9.0', () => { + useSandbox(['bluebird@2.9.0']) + it('should instrument the package', () => runTest('true\n')) + }) + context('when bluebird is 3.7.2', () => { + useSandbox(['bluebird@3.7.2']) + it('should instrument the package', () => runTest('true\n')) + }) + }) + + context('when package is in range (fastify)', () => { + context('when fastify is latest', () => { + useSandbox(['fastify']) + it('should instrument the package', () => runTest('true\n')) + }) + context('when fastify is latest and logging enabled', () => { + useSandbox(['fastify']) + useEnv({ DD_TRACE_DEBUG }) + it('should instrument the package', () => + runTest('Application instrumentation bootstrapping complete\ntrue\n')) + }) + }) + + context('when package errors out', () => { + useSandbox(['bluebird']) + before(() => { + const file = path.join(sandboxCwd(), 'node_modules/dd-trace/packages/datadog-instrumentations/src/bluebird.js') + fs.writeFileSync(file, ` +const { addHook } = require('./helpers/instrument') + +addHook({ name: 'bluebird', versions: ['*'] }, Promise => { + throw new ReferenceError('this is a test error') + return Promise +}) + `) + }) + + context('with DD_INJECTION_ENABLED', () => { + useEnv({ DD_INJECTION_ENABLED }) + it('should not instrument the package, and send telemetry', () => + runTest('false\n', + 'complete', 'injection_forced:false', + 'error', 'error_type:ReferenceError,integration:bluebird,integration_version:3.7.2' + )) + }) + + context('with logging disabled', () => { + it('should not instrument the package', () => runTest('false\n')) + }) + + context('with logging enabled', () => { + useEnv({ DD_TRACE_DEBUG, DD_LOG_LEVEL }) + it('should not instrument the package', () => + runTest( + log => { + assert.ok(log.includes(` +Error during ddtrace instrumentation of application, aborting. +ReferenceError: this is a test error + at `)) + assert.ok(log.includes('\nfalse\n')) + })) + }) + }) +}) diff --git a/integration-tests/package-guardrails/index.js b/integration-tests/package-guardrails/index.js new file mode 100644 index 00000000000..4130270b9e1 --- /dev/null +++ b/integration-tests/package-guardrails/index.js @@ -0,0 +1,16 @@ +'use strict' + +/* eslint-disable no-console */ +/* eslint-disable import/no-extraneous-dependencies */ + +try { + const P = require('bluebird') + + const isWrapped = P.prototype._then.toString().includes('AsyncResource') + + console.log(isWrapped) +} catch (e) { + const fastify = require('fastify') + + console.log(fastify.toString().startsWith('function shim')) +} diff --git a/integration-tests/pino.spec.js b/integration-tests/pino.spec.js new file mode 100644 index 00000000000..4566eae63fb --- /dev/null +++ b/integration-tests/pino.spec.js @@ -0,0 +1,72 @@ +/* eslint-disable comma-dangle */ +'use strict' + +const { FakeAgent, spawnProc, createSandbox, curl } = require('./helpers') +const path = require('path') +const { assert } = require('chai') +const { once } = require('events') + +describe('pino test', () => { + let agent + let proc + let sandbox + let cwd + let startupTestFile + + before(async () => { + sandbox = await createSandbox(['pino']) + cwd = sandbox.folder + startupTestFile = path.join(cwd, 'pino/index.js') + }) + + after(async () => { + await sandbox.remove() + }) + + context('Log injection', () => { + beforeEach(async () => { + agent = await new FakeAgent().start() + }) + + afterEach(async () => { + proc.kill() + await agent.stop() + }) + + it('Log injection enabled', async () => { + proc = await spawnProc(startupTestFile, { + cwd, + env: { + AGENT_PORT: agent.port, + lOG_INJECTION: true, + }, + stdio: 'pipe', + }) + const [data] = await Promise.all([once(proc.stdout, 'data'), curl(proc)]) + const stdoutData = JSON.parse(data.toString()) + assert.containsAllKeys(stdoutData, ['dd']) + assert.containsAllKeys(stdoutData.dd, ['trace_id', 'span_id']) + assert.strictEqual( + stdoutData.dd.trace_id, + stdoutData.custom.trace_id + ) + assert.strictEqual( + stdoutData.dd.span_id, + stdoutData.custom.span_id + ) + }) + + it('Log injection disabled', async () => { + proc = await spawnProc(startupTestFile, { + cwd, + env: { + AGENT_PORT: agent.port, + }, + stdio: 'pipe', + }) + const [data] = await Promise.all([once(proc.stdout, 'data'), curl(proc)]) + const stdoutData = JSON.parse(data.toString()) + assert.doesNotHaveAnyKeys(stdoutData, ['dd']) + }) + }) +}) diff --git a/integration-tests/pino/index.js b/integration-tests/pino/index.js new file mode 100644 index 00000000000..40e35388fac --- /dev/null +++ b/integration-tests/pino/index.js @@ -0,0 +1,32 @@ +'use strict' + +const options = {} + +if (process.env.AGENT_PORT) { + options.port = process.env.AGENT_PORT +} + +if (process.env.lOG_INJECTION) { + options.logInjection = process.env.lOG_INJECTION +} + +const tracer = require('dd-trace').init(options) + +const http = require('http') +const logger = require('pino')() + +const server = http + .createServer((req, res) => { + const span = tracer.scope().active() + const contextTraceId = span.context().toTraceId() + const contextSpanId = span.context().toSpanId() + logger.info( + { custom: { trace_id: contextTraceId, span_id: contextSpanId } }, + 'Creating server' + ) + res.end('hello, world\n') + }) + .listen(0, () => { + const port = server.address().port + process.send({ port }) + }) diff --git a/integration-tests/playwright.config.js b/integration-tests/playwright.config.js index 1490ee32fe1..34b0a69a859 100644 --- a/integration-tests/playwright.config.js +++ b/integration-tests/playwright.config.js @@ -1,9 +1,10 @@ // Playwright config file for integration tests const { devices } = require('@playwright/test') -module.exports = { +const config = { baseURL: process.env.PW_BASE_URL, - testDir: './ci-visibility/playwright-tests', + testDir: process.env.TEST_DIR || './ci-visibility/playwright-tests', + timeout: Number(process.env.TEST_TIMEOUT) || 30000, reporter: 'line', /* Configure projects for major browsers */ projects: [ @@ -16,3 +17,9 @@ module.exports = { ], testMatch: '**/*-test.js' } + +if (process.env.MAX_FAILURES) { + config.maxFailures = Number(process.env.MAX_FAILURES) +} + +module.exports = config diff --git a/integration-tests/playwright/playwright.spec.js b/integration-tests/playwright/playwright.spec.js index bb0329086b4..440cf13d637 100644 --- a/integration-tests/playwright/playwright.spec.js +++ b/integration-tests/playwright/playwright.spec.js @@ -12,13 +12,31 @@ const { } = require('../helpers') const { FakeCiVisIntake } = require('../ci-visibility-intake') const webAppServer = require('../ci-visibility/web-app-server') -const { TEST_STATUS, TEST_SOURCE_START, TEST_TYPE } = require('../../packages/dd-trace/src/plugins/util/test') +const { + TEST_STATUS, + TEST_SOURCE_START, + TEST_TYPE, + TEST_SOURCE_FILE, + TEST_CONFIGURATION_BROWSER_NAME, + TEST_IS_NEW, + TEST_IS_RETRY, + TEST_EARLY_FLAKE_ENABLED, + TEST_SUITE, + TEST_CODE_OWNERS, + TEST_SESSION_NAME, + TEST_LEVEL_EVENT_TYPES +} = require('../../packages/dd-trace/src/plugins/util/test') +const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') +const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants') + +const NUM_RETRIES_EFD = 3 const versions = ['1.18.0', 'latest'] versions.forEach((version) => { describe(`playwright@${version}`, () => { let sandbox, cwd, receiver, childProcess, webAppPort + before(async function () { // bump from 30 to 60 seconds because playwright dependencies are heavy this.timeout(60000) @@ -51,10 +69,19 @@ versions.forEach((version) => { context(`reporting via ${reportMethod}`, () => { it('can run and report tests', (done) => { const envVars = reportMethod === 'agentless' - ? getCiVisAgentlessConfig(receiver.port) : getCiVisEvpProxyConfig(receiver.port) + ? getCiVisAgentlessConfig(receiver.port) + : getCiVisEvpProxyConfig(receiver.port) const reportUrl = reportMethod === 'agentless' ? '/api/v2/citestcycle' : '/evp_proxy/v2/api/v2/citestcycle' receiver.gatherPayloadsMaxTimeout(({ url }) => url === reportUrl, payloads => { + const metadataDicts = payloads.flatMap(({ payload }) => payload.metadata) + + metadataDicts.forEach(metadata => { + for (const testLevel of TEST_LEVEL_EVENT_TYPES) { + assert.equal(metadata[testLevel][TEST_SESSION_NAME], 'my-test-session') + } + }) + const events = payloads.flatMap(({ payload }) => payload.events) const testSessionEvent = events.find(event => event.type === 'test_session_end') @@ -70,6 +97,10 @@ versions.forEach((version) => { assert.equal(testModuleEvent.content.meta[TEST_STATUS], 'fail') assert.equal(testSessionEvent.content.meta[TEST_TYPE], 'browser') assert.equal(testModuleEvent.content.meta[TEST_TYPE], 'browser') + + assert.exists(testSessionEvent.content.meta[ERROR_MESSAGE]) + assert.exists(testModuleEvent.content.meta[ERROR_MESSAGE]) + assert.includeMembers(testSuiteEvents.map(suite => suite.content.resource), [ 'test_suite.todo-list-page-test.js', 'test_suite.landing-page-test.js', @@ -82,6 +113,15 @@ versions.forEach((version) => { 'skip' ]) + testSuiteEvents.forEach(testSuiteEvent => { + if (testSuiteEvent.content.meta[TEST_STATUS] === 'fail') { + assert.exists(testSuiteEvent.content.meta[ERROR_MESSAGE]) + } + assert.isTrue(testSuiteEvent.content.meta[TEST_SOURCE_FILE].endsWith('-test.js')) + assert.equal(testSuiteEvent.content.metrics[TEST_SOURCE_START], 1) + assert.exists(testSuiteEvent.content.metrics[DD_HOST_CPU_COUNT]) + }) + assert.includeMembers(testEvents.map(test => test.content.resource), [ 'landing-page-test.js.should work with passing tests', 'landing-page-test.js.should work with skipped tests', @@ -99,6 +139,15 @@ versions.forEach((version) => { testEvents.forEach(testEvent => { assert.exists(testEvent.content.metrics[TEST_SOURCE_START]) + assert.equal( + testEvent.content.meta[TEST_SOURCE_FILE].startsWith('ci-visibility/playwright-tests/'), true + ) + // Can read DD_TAGS + assert.propertyVal(testEvent.content.meta, 'test.customtag', 'customvalue') + assert.propertyVal(testEvent.content.meta, 'test.customtag2', 'customvalue2') + // Adds the browser used + assert.propertyVal(testEvent.content.meta, TEST_CONFIGURATION_BROWSER_NAME, 'chromium') + assert.exists(testEvent.content.metrics[DD_HOST_CPU_COUNT]) }) stepEvents.forEach(stepEvent => { @@ -120,7 +169,9 @@ versions.forEach((version) => { cwd, env: { ...envVars, - PW_BASE_URL: `http://localhost:${webAppPort}` + PW_BASE_URL: `http://localhost:${webAppPort}`, + DD_TAGS: 'test.customtag:customvalue,test.customtag2:customvalue2', + DD_TEST_SESSION_NAME: 'my-test-session' }, stdio: 'pipe' } @@ -128,6 +179,7 @@ versions.forEach((version) => { }) }) }) + it('works when tests are compiled to a different location', (done) => { let testOutput = '' @@ -163,5 +215,501 @@ versions.forEach((version) => { testOutput += chunk.toString() }) }) + + it('works when before all fails and step durations are negative', (done) => { + receiver.gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSuiteEvent = events.find(event => event.type === 'test_suite_end').content + const testSessionEvent = events.find(event => event.type === 'test_session_end').content + + assert.propertyVal(testSuiteEvent.meta, TEST_STATUS, 'fail') + assert.propertyVal(testSessionEvent.meta, TEST_STATUS, 'fail') + assert.exists(testSuiteEvent.meta[ERROR_MESSAGE]) + assert.include(testSessionEvent.meta[ERROR_MESSAGE], 'Test suites failed: 1') + }).then(() => done()).catch(done) + + childProcess = exec( + './node_modules/.bin/playwright test -c playwright.config.js', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + PW_BASE_URL: `http://localhost:${webAppPort}`, + TEST_DIR: './ci-visibility/playwright-tests-error', + TEST_TIMEOUT: 3000 + }, + stdio: 'pipe' + } + ) + }) + + if (version === 'latest') { + context('early flake detection', () => { + it('retries new tests', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } + } + }) + + receiver.setKnownTests( + { + playwright: { + 'landing-page-test.js': [ + // 'should work with passing tests', // it will be considered new + 'should work with skipped tests', + 'should work with fixme', + 'should work with annotated tests' + ], + 'skipped-suite-test.js': [ + 'should work with fixme root' + ], + 'todo-list-page-test.js': [ + 'should work with failing tests', + 'should work with fixme root' + ] + } + } + ) + + const receiverPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const newTests = tests.filter(test => + test.resource === + 'landing-page-test.js.should work with passing tests' + ) + newTests.forEach(test => { + assert.propertyVal(test.meta, TEST_IS_NEW, 'true') + }) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + + assert.equal(retriedTests.length, NUM_RETRIES_EFD) + + // all but one has been retried + assert.equal(retriedTests.length, newTests.length - 1) + }) + + childProcess = exec( + './node_modules/.bin/playwright test -c playwright.config.js', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + PW_BASE_URL: `http://localhost:${webAppPort}` + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + receiverPromise.then(() => done()).catch(done) + }) + }) + + it('is disabled if DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED is false', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } + } + }) + + receiver.setKnownTests( + { + playwright: { + 'landing-page-test.js': [ + // 'should work with passing tests', // it will be considered new + 'should work with skipped tests', + 'should work with fixme', + 'should work with annotated tests' + ], + 'skipped-suite-test.js': [ + 'should work with fixme root' + ], + 'todo-list-page-test.js': [ + 'should work with failing tests', + 'should work with fixme root' + ] + } + } + ) + + const receiverPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const newTests = tests.filter(test => + test.resource === + 'landing-page-test.js.should work with passing tests' + ) + newTests.forEach(test => { + assert.notProperty(test.meta, TEST_IS_NEW) + }) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + + assert.equal(retriedTests.length, 0) + }) + + childProcess = exec( + './node_modules/.bin/playwright test -c playwright.config.js', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + PW_BASE_URL: `http://localhost:${webAppPort}`, + DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED: 'false' + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + receiverPromise.then(() => done()).catch(done) + }) + }) + + it('does not retry tests that are skipped', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } + } + }) + + receiver.setKnownTests( + { + playwright: { + 'landing-page-test.js': [ + 'should work with passing tests', + // 'should work with skipped tests', // new but not retried because it's skipped + // 'should work with fixme', // new but not retried because it's skipped + 'should work with annotated tests' + ], + 'skipped-suite-test.js': [ + 'should work with fixme root' + ], + 'todo-list-page-test.js': [ + 'should work with failing tests', + 'should work with fixme root' + ] + } + } + ) + + const receiverPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const newTests = tests.filter(test => + test.resource === + 'landing-page-test.js.should work with skipped tests' || + test.resource === 'landing-page-test.js.should work with fixme' + ) + // no retries + assert.equal(newTests.length, 2) + newTests.forEach(test => { + assert.propertyVal(test.meta, TEST_IS_NEW, 'true') + }) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + + assert.equal(retriedTests.length, 0) + }) + + childProcess = exec( + './node_modules/.bin/playwright test -c playwright.config.js', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + PW_BASE_URL: `http://localhost:${webAppPort}` + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + receiverPromise.then(() => done()).catch(done) + }) + }) + + it('does not run EFD if the known tests request fails', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } + } + }) + + receiver.setKnownTestsResponseCode(500) + receiver.setKnownTests({}) + + const receiverPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + assert.equal(tests.length, 7) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + + const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') + assert.equal(newTests.length, 0) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 0) + }) + + childProcess = exec( + './node_modules/.bin/playwright test -c playwright.config.js', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + PW_BASE_URL: `http://localhost:${webAppPort}` + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + receiverPromise + .then(() => done()) + .catch(done) + }) + }) + }) + } + + it('does not crash when maxFailures=1 and there is an error', (done) => { + receiver.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testEvents = events.filter(event => event.type === 'test') + + assert.includeMembers(testEvents.map(test => test.content.resource), [ + 'failing-test-and-another-test.js.should work with failing tests', + 'failing-test-and-another-test.js.does not crash afterwards' + ]) + }).then(() => done()).catch(done) + + childProcess = exec( + './node_modules/.bin/playwright test -c playwright.config.js', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + PW_BASE_URL: `http://localhost:${webAppPort}`, + MAX_FAILURES: 1, + TEST_DIR: './ci-visibility/playwright-tests-max-failures' + }, + stdio: 'pipe' + } + ) + }) + + context('flaky test retries', () => { + it('can automatically retry flaky tests', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + flaky_test_retries_enabled: true, + early_flake_detection: { + enabled: false + } + }) + + const receiverPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + assert.equal(tests.length, 3) + + const failedTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(failedTests.length, 2) + + const failedRetryTests = failedTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(failedRetryTests.length, 1) // the first one is not a retry + + const passedTests = tests.filter(test => test.meta[TEST_STATUS] === 'pass') + assert.equal(passedTests.length, 1) + assert.equal(passedTests[0].meta[TEST_IS_RETRY], 'true') + }, 30000) + + childProcess = exec( + './node_modules/.bin/playwright test -c playwright.config.js', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + PW_BASE_URL: `http://localhost:${webAppPort}`, + TEST_DIR: './ci-visibility/playwright-tests-automatic-retry' + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + receiverPromise + .then(() => done()) + .catch(done) + }) + }) + + it('is disabled if DD_CIVISIBILITY_FLAKY_RETRY_ENABLED is false', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + flaky_test_retries_enabled: true, + early_flake_detection: { + enabled: false + } + }) + + const receiverPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + assert.equal(tests.length, 1) + assert.equal(tests.filter((test) => test.meta[TEST_IS_RETRY]).length, 0) + }, 30000) + + childProcess = exec( + './node_modules/.bin/playwright test -c playwright.config.js', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + PW_BASE_URL: `http://localhost:${webAppPort}`, + DD_CIVISIBILITY_FLAKY_RETRY_ENABLED: 'false', + TEST_DIR: './ci-visibility/playwright-tests-automatic-retry' + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + receiverPromise + .then(() => done()) + .catch(done) + }) + }) + + it('retries DD_CIVISIBILITY_FLAKY_RETRY_COUNT times', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + flaky_test_retries_enabled: true, + early_flake_detection: { + enabled: false + } + }) + + const receiverPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + assert.equal(tests.length, 2) + + const failedTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(failedTests.length, 2) + + const failedRetryTests = failedTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(failedRetryTests.length, 1) + }, 30000) + + childProcess = exec( + './node_modules/.bin/playwright test -c playwright.config.js', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + PW_BASE_URL: `http://localhost:${webAppPort}`, + TEST_DIR: './ci-visibility/playwright-tests-automatic-retry', + DD_CIVISIBILITY_FLAKY_RETRY_COUNT: 1 + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + receiverPromise + .then(() => done()) + .catch(done) + }) + }) + }) + + it('correctly calculates test code owners when working directory is not repository root', (done) => { + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const test = events.find(event => event.type === 'test').content + const testSuite = events.find(event => event.type === 'test_suite_end').content + // The test is in a subproject + assert.notEqual(test.meta[TEST_SOURCE_FILE], test.meta[TEST_SUITE]) + assert.equal(test.meta[TEST_CODE_OWNERS], JSON.stringify(['@datadog-dd-trace-js'])) + assert.equal(testSuite.meta[TEST_CODE_OWNERS], JSON.stringify(['@datadog-dd-trace-js'])) + }) + + childProcess = exec( + '../../node_modules/.bin/playwright test', + { + cwd: `${cwd}/ci-visibility/subproject`, + env: { + ...getCiVisAgentlessConfig(receiver.port), + PW_BASE_URL: `http://localhost:${webAppPort}`, + PW_RUNNER_DEBUG: '1', + TEST_DIR: '.' + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) }) }) diff --git a/integration-tests/profiler.spec.js b/integration-tests/profiler.spec.js deleted file mode 100644 index 654d89a25ce..00000000000 --- a/integration-tests/profiler.spec.js +++ /dev/null @@ -1,334 +0,0 @@ -'use strict' - -const { - FakeAgent, - createSandbox -} = require('./helpers') -const childProcess = require('child_process') -const { fork } = childProcess -const path = require('path') -const { assert } = require('chai') -const fs = require('node:fs/promises') -const fsync = require('node:fs') -const zlib = require('node:zlib') -const { Profile } = require('pprof-format') - -async function checkProfiles (agent, proc, timeout, - expectedProfileTypes = ['wall', 'space'], expectBadExit = false, multiplicity = 1) { - const resultPromise = agent.assertMessageReceived(({ headers, payload, files }) => { - assert.propertyVal(headers, 'host', `127.0.0.1:${agent.port}`) - assert.propertyVal(payload, 'format', 'pprof') - assert.deepPropertyVal(payload, 'types', expectedProfileTypes) - for (const [index, profileType] of expectedProfileTypes.entries()) { - assert.propertyVal(files[index], 'originalname', `${profileType}.pb.gz`) - } - }, timeout, multiplicity) - - await processExitPromise(proc, timeout, expectBadExit) - return resultPromise -} - -function processExitPromise (proc, timeout, expectBadExit = false) { - return new Promise((resolve, reject) => { - const timeoutObj = setTimeout(() => { - reject(new Error('Process timed out')) - }, timeout) - - function checkExitCode (code) { - clearTimeout(timeoutObj) - - if ((code !== 0) !== expectBadExit) { - reject(new Error(`Process exited with unexpected status code ${code}.`)) - } else { - resolve() - } - } - - proc - .on('error', reject) - .on('exit', checkExitCode) - }) -} - -async function getLatestProfile (cwd, pattern) { - const dirEntries = await fs.readdir(cwd) - // Get the latest file matching the pattern - const pprofEntries = dirEntries.filter(name => pattern.test(name)) - assert.isTrue(pprofEntries.length > 0, `No file matching pattern ${pattern} found in ${cwd}`) - const pprofEntry = pprofEntries - .map(name => ({ name, modified: fsync.statSync(path.join(cwd, name), { bigint: true }).mtimeNs })) - .reduce((a, b) => a.modified > b.modified ? a : b) - .name - const pprofGzipped = await fs.readFile(path.join(cwd, pprofEntry)) - const pprofUnzipped = zlib.gunzipSync(pprofGzipped) - return Profile.decode(pprofUnzipped) -} -describe('profiler', () => { - let agent - let proc - let sandbox - let cwd - let profilerTestFile - let oomTestFile - let oomEnv - let oomExecArgv - const timeout = 5000 - - before(async () => { - sandbox = await createSandbox() - cwd = sandbox.folder - profilerTestFile = path.join(cwd, 'profiler/index.js') - oomTestFile = path.join(cwd, 'profiler/oom.js') - oomExecArgv = ['--max-old-space-size=50'] - }) - - after(async () => { - await sandbox.remove() - }) - - it('code hotspots and endpoint tracing works', async () => { - const procStart = BigInt(Date.now() * 1000000) - const proc = fork(path.join(cwd, 'profiler/codehotspots.js'), { - cwd, - env: { - DD_PROFILING_PROFILERS: 'wall', - DD_PROFILING_EXPORTERS: 'file', - DD_PROFILING_ENABLED: 1, - DD_PROFILING_CODEHOTSPOTS_ENABLED: 1, - DD_PROFILING_ENDPOINT_COLLECTION_ENABLED: 1, - DD_PROFILING_EXPERIMENTAL_TIMELINE_ENABLED: 1 - } - }) - - await processExitPromise(proc, 5000) - const procEnd = BigInt(Date.now() * 1000000) - - const prof = await getLatestProfile(cwd, /^wall_.+\.pprof$/) - - // We check the profile for following invariants: - // - every sample needs to have an 'end_timestamp_ns' label that has values (nanos since UNIX - // epoch) between process start and end. - // - it needs to have samples with 9 total different 'span id's, and 3 different - // 'local root span id's - // - samples with spans also must have a 'trace endpoint' label with values 'endpoint-0', - // 'endpoint-1', or 'endpoint-2' - // - every occurrence of a span must have the same root span and endpoint - const rootSpans = new Set() - const endpoints = new Set() - const spans = new Map() - const strings = prof.stringTable - const tsKey = strings.dedup('end_timestamp_ns') - const spanKey = strings.dedup('span id') - const rootSpanKey = strings.dedup('local root span id') - const endpointKey = strings.dedup('trace endpoint') - const threadNameKey = strings.dedup('thread name') - const threadNameValue = strings.dedup('Main Event Loop') - for (const sample of prof.sample) { - let ts, spanId, rootSpanId, endpoint, threadName - for (const label of sample.label) { - switch (label.key) { - case tsKey: ts = label.num; break - case spanKey: spanId = label.str; break - case rootSpanKey: rootSpanId = label.str; break - case endpointKey: endpoint = label.str; break - case threadNameKey: threadName = label.str; break - default: assert.fail(`Unexpected label key ${strings.dedup(label.key)}`) - } - } - // Timestamp must be defined and be between process start and end time - assert.isDefined(ts) - assert.isTrue(ts <= procEnd) - assert.isTrue(ts >= procStart) - // Thread name must be defined and exactly equal "Main Event Loop" - assert.equal(threadName, threadNameValue) - // Either all or none of span-related labels are defined - if (spanId || rootSpanId || endpoint) { - assert.isDefined(spanId) - assert.isDefined(rootSpanId) - assert.isDefined(endpoint) - - rootSpans.add(rootSpanId) - const spanData = { rootSpanId, endpoint } - const existingSpanData = spans.get(spanId) - if (existingSpanData) { - // Span's root span and endpoint must be consistent across samples - assert.deepEqual(spanData, existingSpanData) - } else { - // New span id, store span data - spans.set(spanId, spanData) - // Verify endpoint value - const endpointVal = strings.strings[endpoint] - switch (endpointVal) { - case 'endpoint-0': - case 'endpoint-1': - case 'endpoint-2': - endpoints.add(endpoint) - break - default: - assert.fail(`Unexpected endpoint value ${endpointVal}`) - } - } - } - } - // Need to have a total of 9 different spans, with 3 different root spans - // and 3 different endpoints. - assert.equal(spans.size, 9) - assert.equal(rootSpans.size, 3) - assert.equal(endpoints.size, 3) - }) - - it('dns timeline events work', async () => { - const procStart = BigInt(Date.now() * 1000000) - const proc = fork(path.join(cwd, 'profiler/dnstest.js'), { - cwd, - env: { - DD_PROFILING_PROFILERS: 'wall', - DD_PROFILING_EXPORTERS: 'file', - DD_PROFILING_ENABLED: 1, - DD_PROFILING_EXPERIMENTAL_TIMELINE_ENABLED: 1 - } - }) - - await processExitPromise(proc, 5000) - const procEnd = BigInt(Date.now() * 1000000) - - const prof = await getLatestProfile(cwd, /^events_.+\.pprof$/) - assert.isAtLeast(prof.sample.length, 5) - - const strings = prof.stringTable - const tsKey = strings.dedup('end_timestamp_ns') - const eventKey = strings.dedup('event') - const hostKey = strings.dedup('host') - const addressKey = strings.dedup('address') - const threadNameKey = strings.dedup('thread name') - const nameKey = strings.dedup('operation') - const dnsEventValue = strings.dedup('dns') - const dnsEvents = [] - for (const sample of prof.sample) { - let ts, event, host, address, name, threadName - for (const label of sample.label) { - switch (label.key) { - case tsKey: ts = label.num; break - case nameKey: name = label.str; break - case eventKey: event = label.str; break - case hostKey: host = label.str; break - case addressKey: address = label.str; break - case threadNameKey: threadName = label.str; break - default: assert.fail(`Unexpected label key ${label.key} ${strings.strings[label.key]}`) - } - } - // Timestamp must be defined and be between process start and end time - assert.isDefined(ts) - assert.isTrue(ts <= procEnd) - assert.isTrue(ts >= procStart) - // Gather only DNS events; ignore sporadic GC events - if (event === dnsEventValue) { - // Thread name must be defined and exactly equal "Main DNS" - assert.isTrue(strings.strings[threadName].startsWith('Main DNS-')) - assert.isDefined(name) - // Exactly one of these is defined - assert.isTrue(!!address !== !!host) - const ev = { name: strings.strings[name] } - if (address) { - ev.address = strings.strings[address] - } else { - ev.host = strings.strings[host] - } - dnsEvents.push(ev) - } - } - assert.sameDeepMembers(dnsEvents, [ - { name: 'lookup', host: 'example.org' }, - { name: 'lookup', host: 'example.com' }, - { name: 'lookup', host: 'datadoghq.com' }, - { name: 'queryA', host: 'datadoghq.com' }, - { name: 'lookupService', address: '13.224.103.60:80' } - ]) - }) - - context('shutdown', () => { - beforeEach(async () => { - agent = await new FakeAgent().start() - oomEnv = { - DD_TRACE_AGENT_PORT: agent.port, - DD_PROFILING_ENABLED: 1, - DD_TRACE_DEBUG: 1, - DD_TRACE_LOG_LEVEL: 'warn' - } - }) - - afterEach(async () => { - proc.kill() - await agent.stop() - }) - - it('records profile on process exit', async () => { - proc = fork(profilerTestFile, { - cwd, - env: { - DD_TRACE_AGENT_PORT: agent.port, - DD_PROFILING_ENABLED: 1 - } - }) - return checkProfiles(agent, proc, timeout) - }) - - it('sends a heap profile on OOM with external process', async () => { - proc = fork(oomTestFile, { - cwd, - execArgv: oomExecArgv, - env: oomEnv - }) - return checkProfiles(agent, proc, timeout, ['space'], true) - }) - - it('sends a heap profile on OOM with external process and exits successfully', async () => { - proc = fork(oomTestFile, { - cwd, - execArgv: oomExecArgv, - env: { - ...oomEnv, - DD_PROFILING_EXPERIMENTAL_OOM_HEAP_LIMIT_EXTENSION_SIZE: 15000000, - DD_PROFILING_EXPERIMENTAL_OOM_MAX_HEAP_EXTENSION_COUNT: 3 - } - }) - return checkProfiles(agent, proc, timeout, ['space'], false, 2) - }) - - it('sends a heap profile on OOM with async callback', async () => { - proc = fork(oomTestFile, { - cwd, - execArgv: oomExecArgv, - env: { - ...oomEnv, - DD_PROFILING_EXPERIMENTAL_OOM_HEAP_LIMIT_EXTENSION_SIZE: 10000000, - DD_PROFILING_EXPERIMENTAL_OOM_MAX_HEAP_EXTENSION_COUNT: 1, - DD_PROFILING_EXPERIMENTAL_OOM_EXPORT_STRATEGIES: 'async' - } - }) - return checkProfiles(agent, proc, timeout, ['space'], true) - }) - - it('sends heap profiles on OOM with multiple strategies', async () => { - proc = fork(oomTestFile, { - cwd, - execArgv: oomExecArgv, - env: { - ...oomEnv, - DD_PROFILING_EXPERIMENTAL_OOM_HEAP_LIMIT_EXTENSION_SIZE: 10000000, - DD_PROFILING_EXPERIMENTAL_OOM_MAX_HEAP_EXTENSION_COUNT: 1, - DD_PROFILING_EXPERIMENTAL_OOM_EXPORT_STRATEGIES: 'async,process' - } - }) - return checkProfiles(agent, proc, timeout, ['space'], true, 2) - }) - - it('sends a heap profile on OOM in worker thread and exits successfully', async () => { - proc = fork(oomTestFile, [1, 50], { - cwd, - env: { ...oomEnv, DD_PROFILING_WALLTIME_ENABLED: 0 } - }) - return checkProfiles(agent, proc, timeout, ['space'], false, 2) - }) - }) -}) diff --git a/integration-tests/profiler/codehotspots.js b/integration-tests/profiler/codehotspots.js index 9cffc768185..749f15d8790 100644 --- a/integration-tests/profiler/codehotspots.js +++ b/integration-tests/profiler/codehotspots.js @@ -8,8 +8,8 @@ function busyLoop () { const start = process.hrtime.bigint() for (;;) { const now = process.hrtime.bigint() - // Busy cycle for 20ms - if (now - start > 20000000n) { + // Busy cycle for 100ms + if (now - start > 100000000n) { break } } diff --git a/integration-tests/profiler/dnstest.js b/integration-tests/profiler/dnstest.js index 4af0f00750e..36398cb2a05 100644 --- a/integration-tests/profiler/dnstest.js +++ b/integration-tests/profiler/dnstest.js @@ -8,6 +8,3 @@ require('dd-trace').init().profilerStarted().then(() => { dns.resolve4('datadoghq.com', () => {}) dns.lookup('dfslkgsjkrtgrdg.com', () => {}) }) - -// Give the event processor chance to collect events -setTimeout(() => {}, 3000) diff --git a/integration-tests/profiler/nettest.js b/integration-tests/profiler/nettest.js new file mode 100644 index 00000000000..e9f3002d6b0 --- /dev/null +++ b/integration-tests/profiler/nettest.js @@ -0,0 +1,35 @@ +const net = require('net') + +async function streamToString (stream) { + const chunks = [] + + for await (const chunk of stream) { + chunks.push(Buffer.from(chunk)) + } + + return Buffer.concat(chunks).toString('utf8') +} + +const port1 = Number(process.argv[2]) +const port2 = Number(process.argv[3]) +const msg = process.argv[4] + +async function oneTimeConnect (hostSpec) { + return new Promise((resolve, reject) => { + const socket = net.createConnection(hostSpec, async () => { + const resp = await streamToString(socket) + if (resp !== msg) { + reject(new Error(`Expected response ${msg}, got ${resp} instead.`)) + } else { + resolve() + } + }) + }) +} + +require('dd-trace').init().profilerStarted() + .then(() => { + oneTimeConnect({ host: '127.0.0.1', port: port1 }) + }).then(() => { + oneTimeConnect({ host: '127.0.0.1', port: port2 }) + }) diff --git a/integration-tests/profiler/profiler.spec.js b/integration-tests/profiler/profiler.spec.js new file mode 100644 index 00000000000..7306d7051ad --- /dev/null +++ b/integration-tests/profiler/profiler.spec.js @@ -0,0 +1,579 @@ +'use strict' + +const { + FakeAgent, + createSandbox +} = require('../helpers') +const childProcess = require('child_process') +const { fork } = childProcess +const path = require('path') +const { assert } = require('chai') +const fs = require('fs/promises') +const fsync = require('fs') +const net = require('net') +const zlib = require('zlib') +const { Profile } = require('pprof-format') +const semver = require('semver') + +const DEFAULT_PROFILE_TYPES = ['wall', 'space'] +if (process.platform !== 'win32') { + DEFAULT_PROFILE_TYPES.push('events') +} + +function checkProfiles (agent, proc, timeout, + expectedProfileTypes = DEFAULT_PROFILE_TYPES, expectBadExit = false, multiplicity = 1 +) { + return Promise.all([ + processExitPromise(proc, timeout, expectBadExit), + expectProfileMessagePromise(agent, timeout, expectedProfileTypes, multiplicity) + ]) +} + +function expectProfileMessagePromise (agent, timeout, + expectedProfileTypes = DEFAULT_PROFILE_TYPES, multiplicity = 1 +) { + const fileNames = expectedProfileTypes.map(type => `${type}.pprof`) + return agent.assertMessageReceived(({ headers, _, files }) => { + let event + try { + assert.propertyVal(headers, 'host', `127.0.0.1:${agent.port}`) + assert.propertyVal(files[0], 'originalname', 'event.json') + event = JSON.parse(files[0].buffer.toString()) + assert.propertyVal(event, 'family', 'node') + assert.isString(event.info.profiler.activation) + assert.isString(event.info.profiler.ssi.mechanism) + assert.deepPropertyVal(event, 'attachments', fileNames) + for (const [index, fileName] of fileNames.entries()) { + assert.propertyVal(files[index + 1], 'originalname', fileName) + } + } catch (e) { + e.message += ` ${JSON.stringify({ headers, files, event })}` + throw e + } + }, timeout, multiplicity) +} + +function processExitPromise (proc, timeout, expectBadExit = false) { + return new Promise((resolve, reject) => { + const timeoutObj = setTimeout(() => { + reject(new Error('Process timed out')) + }, timeout) + + function checkExitCode (code) { + clearTimeout(timeoutObj) + + if ((code !== 0) !== expectBadExit) { + reject(new Error(`Process exited with unexpected status code ${code}.`)) + } else { + resolve() + } + } + + proc + .on('error', reject) + .on('exit', checkExitCode) + }) +} + +async function getLatestProfile (cwd, pattern) { + const dirEntries = await fs.readdir(cwd) + // Get the latest file matching the pattern + const pprofEntries = dirEntries.filter(name => pattern.test(name)) + assert.isTrue(pprofEntries.length > 0, `No file matching pattern ${pattern} found in ${cwd}`) + const pprofEntry = pprofEntries + .map(name => ({ name, modified: fsync.statSync(path.join(cwd, name), { bigint: true }).mtimeNs })) + .reduce((a, b) => a.modified > b.modified ? a : b) + .name + const pprofGzipped = await fs.readFile(path.join(cwd, pprofEntry)) + const pprofUnzipped = zlib.gunzipSync(pprofGzipped) + return { profile: Profile.decode(pprofUnzipped), encoded: pprofGzipped.toString('base64') } +} + +function expectTimeout (messagePromise, allowErrors = false) { + return messagePromise.then( + () => { + throw new Error('Received unexpected message') + }, (e) => { + if (e.message !== 'timeout' && (!allowErrors || !e.message.startsWith('timeout, additionally:'))) { + throw e + } + } + ) +} + +async function gatherNetworkTimelineEvents (cwd, scriptFilePath, eventType, args) { + const procStart = BigInt(Date.now() * 1000000) + const proc = fork(path.join(cwd, scriptFilePath), args, { + cwd, + env: { + DD_PROFILING_PROFILERS: 'wall', + DD_PROFILING_EXPORTERS: 'file', + DD_PROFILING_ENABLED: 1, + DD_PROFILING_EXPERIMENTAL_TIMELINE_ENABLED: 1 + } + }) + + await processExitPromise(proc, 30000) + const procEnd = BigInt(Date.now() * 1000000) + + const { profile, encoded } = await getLatestProfile(cwd, /^events_.+\.pprof$/) + + const strings = profile.stringTable + const tsKey = strings.dedup('end_timestamp_ns') + const eventKey = strings.dedup('event') + const hostKey = strings.dedup('host') + const addressKey = strings.dedup('address') + const portKey = strings.dedup('port') + const nameKey = strings.dedup('operation') + const spanIdKey = strings.dedup('span id') + const localRootSpanIdKey = strings.dedup('local root span id') + const eventValue = strings.dedup(eventType) + const events = [] + for (const sample of profile.sample) { + let ts, event, host, address, port, name, spanId, localRootSpanId + for (const label of sample.label) { + switch (label.key) { + case tsKey: ts = label.num; break + case nameKey: name = label.str; break + case eventKey: event = label.str; break + case hostKey: host = label.str; break + case addressKey: address = label.str; break + case portKey: port = label.num; break + case spanIdKey: spanId = label.str; break + case localRootSpanIdKey: localRootSpanId = label.str; break + default: assert.fail(`Unexpected label key ${label.key} ${strings.strings[label.key]} ${encoded}`) + } + } + // Timestamp must be defined and be between process start and end time + assert.isDefined(ts, encoded) + assert.isTrue(ts <= procEnd, encoded) + assert.isTrue(ts >= procStart, encoded) + if (process.platform !== 'win32') { + assert.isDefined(spanId, encoded) + assert.isDefined(localRootSpanId, encoded) + } else { + assert.isUndefined(spanId, encoded) + assert.isUndefined(localRootSpanId, encoded) + } + // Gather only DNS events; ignore sporadic GC events + if (event === eventValue) { + assert.isDefined(name, encoded) + // Exactly one of these is defined + assert.isTrue(!!address !== !!host, encoded) + const ev = { name: strings.strings[name] } + if (address) { + ev.address = strings.strings[address] + } else { + ev.host = strings.strings[host] + } + if (port) { + ev.port = port + } + events.push(ev) + } + } + return events +} + +describe('profiler', () => { + let agent + let proc + let sandbox + let cwd + let profilerTestFile + let ssiTestFile + let oomTestFile + let oomEnv + let oomExecArgv + const timeout = 30000 + + before(async () => { + sandbox = await createSandbox() + cwd = sandbox.folder + profilerTestFile = path.join(cwd, 'profiler/index.js') + ssiTestFile = path.join(cwd, 'profiler/ssi.js') + oomTestFile = path.join(cwd, 'profiler/oom.js') + oomExecArgv = ['--max-old-space-size=50'] + }) + + after(async () => { + await sandbox.remove() + }) + + if (process.platform !== 'win32') { + it('code hotspots and endpoint tracing works', async () => { + const procStart = BigInt(Date.now() * 1000000) + const proc = fork(path.join(cwd, 'profiler/codehotspots.js'), { + cwd, + env: { + DD_PROFILING_PROFILERS: 'wall', + DD_PROFILING_EXPORTERS: 'file', + DD_PROFILING_ENABLED: 1, + DD_PROFILING_CODEHOTSPOTS_ENABLED: 1, + DD_PROFILING_ENDPOINT_COLLECTION_ENABLED: 1, + DD_PROFILING_EXPERIMENTAL_TIMELINE_ENABLED: 1 + } + }) + + await processExitPromise(proc, 30000) + const procEnd = BigInt(Date.now() * 1000000) + + const { profile, encoded } = await getLatestProfile(cwd, /^wall_.+\.pprof$/) + + // We check the profile for following invariants: + // - every sample needs to have an 'end_timestamp_ns' label that has values (nanos since UNIX + // epoch) between process start and end. + // - it needs to have samples with 9 total different 'span id's, and 3 different + // 'local root span id's + // - samples with spans also must have a 'trace endpoint' label with values 'endpoint-0', + // 'endpoint-1', or 'endpoint-2' + // - every occurrence of a span must have the same root span and endpoint + const rootSpans = new Set() + const endpoints = new Set() + const spans = new Map() + const strings = profile.stringTable + const tsKey = strings.dedup('end_timestamp_ns') + const spanKey = strings.dedup('span id') + const rootSpanKey = strings.dedup('local root span id') + const endpointKey = strings.dedup('trace endpoint') + const threadNameKey = strings.dedup('thread name') + const threadIdKey = strings.dedup('thread id') + const osThreadIdKey = strings.dedup('os thread id') + const threadNameValue = strings.dedup('Main Event Loop') + const nonJSThreadNameValue = strings.dedup('Non-JS threads') + + for (const sample of profile.sample) { + let ts, spanId, rootSpanId, endpoint, threadName, threadId, osThreadId + for (const label of sample.label) { + switch (label.key) { + case tsKey: ts = label.num; break + case spanKey: spanId = label.str; break + case rootSpanKey: rootSpanId = label.str; break + case endpointKey: endpoint = label.str; break + case threadNameKey: threadName = label.str; break + case threadIdKey: threadId = label.str; break + case osThreadIdKey: osThreadId = label.str; break + default: assert.fail(`Unexpected label key ${strings.dedup(label.key)} ${encoded}`) + } + } + if (threadName !== nonJSThreadNameValue) { + // Timestamp must be defined and be between process start and end time + assert.isDefined(ts, encoded) + assert.isNumber(osThreadId, encoded) + assert.equal(threadId, strings.dedup('0'), encoded) + assert.isTrue(ts <= procEnd, encoded) + assert.isTrue(ts >= procStart, encoded) + // Thread name must be defined and exactly equal "Main Event Loop" + assert.equal(threadName, threadNameValue, encoded) + } else { + assert.equal(threadId, strings.dedup('NA'), encoded) + } + // Either all or none of span-related labels are defined + if (endpoint === undefined) { + // It is possible to catch a sample executing in tracer's startSpan so + // that endpoint is not yet set. We'll ignore those samples. + continue + } + if (spanId || rootSpanId) { + assert.isDefined(spanId, encoded) + assert.isDefined(rootSpanId, encoded) + + rootSpans.add(rootSpanId) + if (spanId === rootSpanId) { + // It is possible to catch a sample executing in the root span before + // it entered the nested span; we ignore these too, although we'll + // still record the root span ID as we want to assert there'll only be + // 3 of them. + continue + } + const spanData = { rootSpanId, endpoint } + const existingSpanData = spans.get(spanId) + if (existingSpanData) { + // Span's root span and endpoint must be consistent across samples + assert.deepEqual(spanData, existingSpanData, encoded) + } else { + // New span id, store span data + spans.set(spanId, spanData) + // Verify endpoint value + const endpointVal = strings.strings[endpoint] + switch (endpointVal) { + case 'endpoint-0': + case 'endpoint-1': + case 'endpoint-2': + endpoints.add(endpoint) + break + default: + assert.fail(`Unexpected endpoint value ${endpointVal} ${encoded}`) + } + } + } + } + // Need to have a total of 9 different spans, with 3 different root spans + // and 3 different endpoints. + assert.equal(spans.size, 9, encoded) + assert.equal(rootSpans.size, 3, encoded) + assert.equal(endpoints.size, 3, encoded) + }) + + if (semver.gte(process.version, '16.0.0')) { + it('dns timeline events work', async () => { + const dnsEvents = await gatherNetworkTimelineEvents(cwd, 'profiler/dnstest.js', 'dns') + assert.sameDeepMembers(dnsEvents, [ + { name: 'lookup', host: 'example.org' }, + { name: 'lookup', host: 'example.com' }, + { name: 'lookup', host: 'datadoghq.com' }, + { name: 'queryA', host: 'datadoghq.com' }, + { name: 'lookupService', address: '13.224.103.60', port: 80 } + ]) + }) + + it('net timeline events work', async () => { + // Simple server that writes a constant message to the socket. + const msg = 'cya later!\n' + function createServer () { + const server = net.createServer((socket) => { + socket.end(msg, 'utf8') + }).on('error', (err) => { + throw err + }) + return server + } + // Create two instances of the server + const server1 = createServer() + try { + const server2 = createServer() + try { + // Have the servers listen on ephemeral ports + const p = new Promise(resolve => { + server1.listen(0, () => { + server2.listen(0, async () => { + resolve([server1.address().port, server2.address().port]) + }) + }) + }) + const [port1, port2] = await p + const args = [String(port1), String(port2), msg] + // Invoke the profiled program, passing it the ports of the servers and + // the expected message. + const events = await gatherNetworkTimelineEvents(cwd, 'profiler/nettest.js', 'net', args) + // The profiled program should have two TCP connection events to the two + // servers. + assert.sameDeepMembers(events, [ + { name: 'connect', host: '127.0.0.1', port: port1 }, + { name: 'connect', host: '127.0.0.1', port: port2 } + ]) + } finally { + server2.close() + } + } finally { + server1.close() + } + }) + } + } + + context('shutdown', () => { + beforeEach(async () => { + agent = await new FakeAgent().start() + oomEnv = { + DD_TRACE_AGENT_PORT: agent.port, + DD_PROFILING_ENABLED: 1, + DD_TRACE_DEBUG: 1, + DD_TRACE_LOG_LEVEL: 'warn' + } + }) + + afterEach(async () => { + proc.kill() + await agent.stop() + }) + + it('records profile on process exit', () => { + proc = fork(profilerTestFile, { + cwd, + env: { + DD_TRACE_AGENT_PORT: agent.port, + DD_PROFILING_ENABLED: 1 + } + }) + const checkTelemetry = agent.assertTelemetryReceived(_ => {}, 1000, 'generate-metrics') + // SSI telemetry is not supposed to have been emitted when DD_INJECTION_ENABLED is absent, + // so expect telemetry callback to time out + return Promise.all([checkProfiles(agent, proc, timeout), expectTimeout(checkTelemetry)]) + }) + + it('records SSI telemetry on process exit', () => { + proc = fork(profilerTestFile, { + cwd, + env: { + DD_TRACE_AGENT_PORT: agent.port, + DD_INJECTION_ENABLED: 'tracing', + DD_PROFILING_ENABLED: 1 + } + }) + + function checkTags (tags) { + assert.include(tags, 'enablement_choice:manually_enabled') + assert.include(tags, 'heuristic_hypothetical_decision:no_span_short_lived') + assert.include(tags, 'installation:ssi') + // There's a race between metrics and on-shutdown profile, so tag value + // can be either false or true but it must be present + assert.isTrue(tags.some(tag => tag === 'has_sent_profiles:false' || tag === 'has_sent_profiles:true')) + } + + const checkTelemetry = agent.assertTelemetryReceived(({ headers, payload }) => { + const pp = payload.payload + assert.equal(pp.namespace, 'profilers') + const series = pp.series + assert.lengthOf(series, 2) + assert.equal(series[0].metric, 'ssi_heuristic.number_of_profiles') + assert.equal(series[0].type, 'count') + checkTags(series[0].tags) + // There's a race between metrics and on-shutdown profile, so metric + // value will be either 0 or 1 + assert.isAtMost(series[0].points[0][1], 1) + + assert.equal(series[1].metric, 'ssi_heuristic.number_of_runtime_id') + assert.equal(series[1].type, 'count') + checkTags(series[1].tags) + assert.equal(series[1].points[0][1], 1) + }, timeout, 'generate-metrics') + return Promise.all([checkProfiles(agent, proc, timeout), checkTelemetry]) + }) + + if (process.platform !== 'win32') { // PROF-8905 + it('sends a heap profile on OOM with external process', () => { + proc = fork(oomTestFile, { + cwd, + execArgv: oomExecArgv, + env: oomEnv + }) + return checkProfiles(agent, proc, timeout, ['space'], true) + }) + + it('sends a heap profile on OOM with external process and exits successfully', () => { + proc = fork(oomTestFile, { + cwd, + execArgv: oomExecArgv, + env: { + ...oomEnv, + DD_PROFILING_EXPERIMENTAL_OOM_HEAP_LIMIT_EXTENSION_SIZE: 15000000, + DD_PROFILING_EXPERIMENTAL_OOM_MAX_HEAP_EXTENSION_COUNT: 3 + } + }) + return checkProfiles(agent, proc, timeout, ['space'], false, 2) + }) + + it('sends a heap profile on OOM with async callback', () => { + proc = fork(oomTestFile, { + cwd, + execArgv: oomExecArgv, + env: { + ...oomEnv, + DD_PROFILING_EXPERIMENTAL_OOM_HEAP_LIMIT_EXTENSION_SIZE: 10000000, + DD_PROFILING_EXPERIMENTAL_OOM_MAX_HEAP_EXTENSION_COUNT: 1, + DD_PROFILING_EXPERIMENTAL_OOM_EXPORT_STRATEGIES: 'async' + } + }) + return checkProfiles(agent, proc, timeout, ['space'], true) + }) + + it('sends heap profiles on OOM with multiple strategies', () => { + proc = fork(oomTestFile, { + cwd, + execArgv: oomExecArgv, + env: { + ...oomEnv, + DD_PROFILING_EXPERIMENTAL_OOM_HEAP_LIMIT_EXTENSION_SIZE: 10000000, + DD_PROFILING_EXPERIMENTAL_OOM_MAX_HEAP_EXTENSION_COUNT: 1, + DD_PROFILING_EXPERIMENTAL_OOM_EXPORT_STRATEGIES: 'async,process' + } + }) + return checkProfiles(agent, proc, timeout, ['space'], true, 2) + }) + + it('sends a heap profile on OOM in worker thread and exits successfully', () => { + proc = fork(oomTestFile, [1, 50], { + cwd, + env: { ...oomEnv, DD_PROFILING_WALLTIME_ENABLED: 0 } + }) + return checkProfiles(agent, proc, timeout, ['space'], false, 2) + }) + } + }) + + context('SSI heuristics', () => { + beforeEach(async () => { + agent = await new FakeAgent().start() + }) + + afterEach(async () => { + proc.kill() + await agent.stop() + }) + + describe('does not trigger for', () => { + it('a short-lived app that creates no spans', () => { + return heuristicsDoesNotTriggerFor([], false, false) + }) + + it('a short-lived app that creates a span', () => { + return heuristicsDoesNotTriggerFor(['create-span'], true, false) + }) + + it('a long-lived app that creates no spans', () => { + return heuristicsDoesNotTriggerFor(['long-lived'], false, false) + }) + + it('a short-lived app that creates no spans with the auto env var', () => { + return heuristicsDoesNotTriggerFor([], false, true) + }) + + it('a short-lived app that creates a span with the auto env var', () => { + return heuristicsDoesNotTriggerFor(['create-span'], true, true) + }) + + it('a long-lived app that creates no spans with the auto env var', () => { + return heuristicsDoesNotTriggerFor(['long-lived'], false, true) + }) + }) + + it('triggers for long-lived span-creating app', () => { + return heuristicsTrigger(false) + }) + + it('triggers for long-lived span-creating app with the auto env var', () => { + return heuristicsTrigger(true) + }) + }) + + function forkSsi (args, whichEnv) { + const profilerEnablingEnv = whichEnv ? { DD_PROFILING_ENABLED: 'auto' } : { DD_INJECTION_ENABLED: 'profiler' } + return fork(ssiTestFile, args, { + cwd, + env: { + DD_TRACE_AGENT_PORT: agent.port, + DD_INTERNAL_PROFILING_LONG_LIVED_THRESHOLD: '1300', + ...profilerEnablingEnv + } + }) + } + + function heuristicsTrigger (whichEnv) { + return checkProfiles(agent, + forkSsi(['create-span', 'long-lived'], whichEnv), + timeout, + DEFAULT_PROFILE_TYPES, + false, + // Will receive 2 messages: first one is for the trace, second one is for the profile. We + // only need the assertions in checkProfiles to succeed for the one with the profile. + 2) + } + + function heuristicsDoesNotTriggerFor (args, allowTraceMessage, whichEnv) { + return Promise.all([ + processExitPromise(forkSsi(args, whichEnv), timeout, false), + expectTimeout(expectProfileMessagePromise(agent, 1500), allowTraceMessage) + ]) + } +}) diff --git a/integration-tests/profiler/ssi.js b/integration-tests/profiler/ssi.js new file mode 100644 index 00000000000..b184d64762b --- /dev/null +++ b/integration-tests/profiler/ssi.js @@ -0,0 +1,24 @@ +'use strict' + +const DDTrace = require('dd-trace') + +const tracer = DDTrace.init() + +async function run () { + const tasks = [] + // If launched with 'create-span', the app will create a span. + if (process.argv.includes('create-span')) { + tasks.push(tracer.trace('woo', _ => { + return new Promise(setImmediate) + })) + } + // If launched with 'long-lived', the app will remain alive long enough to + // be considered long-lived by profiler activation heuristics. + if (process.argv.includes('long-lived')) { + const longLivedThreshold = Number(process.env.DD_INTERNAL_PROFILING_LONG_LIVED_THRESHOLD) + tasks.push(new Promise(resolve => setTimeout(resolve, longLivedThreshold + 200))) + } + await Promise.all(tasks) +} + +tracer.profilerStarted().then(run) diff --git a/integration-tests/selenium/selenium.spec.js b/integration-tests/selenium/selenium.spec.js new file mode 100644 index 00000000000..50fc9d19568 --- /dev/null +++ b/integration-tests/selenium/selenium.spec.js @@ -0,0 +1,146 @@ +const { exec } = require('child_process') + +const { assert } = require('chai') +const getPort = require('get-port') + +const { + createSandbox, + getCiVisAgentlessConfig +} = require('../helpers') +const { FakeCiVisIntake } = require('../ci-visibility-intake') +const { + TEST_BROWSER_DRIVER, + TEST_BROWSER_NAME, + TEST_BROWSER_VERSION, + TEST_BROWSER_DRIVER_VERSION, + TEST_IS_RUM_ACTIVE, + TEST_TYPE +} = require('../../packages/dd-trace/src/plugins/util/test') +const { NODE_MAJOR } = require('../../version') + +const cucumberVersion = NODE_MAJOR <= 16 ? '9' : 'latest' + +const webAppServer = require('../ci-visibility/web-app-server') + +const versionRange = ['4.11.0', 'latest'] + +versionRange.forEach(version => { + describe(`selenium ${version}`, () => { + let receiver + let childProcess + let sandbox + let cwd + let webAppPort + + before(async function () { + sandbox = await createSandbox([ + 'mocha', + 'jest', + `@cucumber/cucumber@${cucumberVersion}`, + 'chai@v4', + `selenium-webdriver@${version}` + ]) + cwd = sandbox.folder + + webAppPort = await getPort() + webAppServer.listen(webAppPort) + }) + + after(async function () { + await sandbox.remove() + await new Promise(resolve => webAppServer.close(resolve)) + }) + + beforeEach(async function () { + const port = await getPort() + receiver = await new FakeCiVisIntake(port).start() + }) + + afterEach(async () => { + childProcess.kill() + await receiver.stop() + }) + + const testFrameworks = [ + { + name: 'mocha', + command: 'mocha ./ci-visibility/test/selenium-test.js --timeout 10000' + }, + { + name: 'jest', + command: 'node ./node_modules/jest/bin/jest --config config-jest.js' + }, + { + name: 'cucumber', + command: './node_modules/.bin/cucumber-js ci-visibility/features-selenium/*.feature' + } + ] + testFrameworks.forEach(({ name, command }) => { + context(`with ${name}`, () => { + it('identifies tests using selenium as browser tests', (done) => { + const assertionPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const seleniumTest = events.find(event => event.type === 'test').content + assert.include(seleniumTest.meta, { + [TEST_BROWSER_DRIVER]: 'selenium', + [TEST_BROWSER_NAME]: 'chrome', + [TEST_TYPE]: 'browser', + [TEST_IS_RUM_ACTIVE]: 'true' + }) + assert.property(seleniumTest.meta, TEST_BROWSER_VERSION) + assert.property(seleniumTest.meta, TEST_BROWSER_DRIVER_VERSION) + }) + + childProcess = exec( + command, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + WEB_APP_URL: `http://localhost:${webAppPort}`, + TESTS_TO_RUN: '**/ci-visibility/test/selenium-test*' + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + assertionPromise.then(() => { + done() + }).catch(done) + }) + }) + }) + }) + + it('does not crash when used outside a known test framework', (done) => { + let testOutput = '' + childProcess = exec( + 'node ./ci-visibility/test/selenium-no-framework.js', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + WEB_APP_URL: `http://localhost:${webAppPort}`, + TESTS_TO_RUN: '**/ci-visibility/test/selenium-test*' + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', (code) => { + assert.equal(code, 0) + assert.notInclude(testOutput, 'InvalidArgumentError') + done() + }) + + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + }) + }) +}) diff --git a/integration-tests/standalone-asm.spec.js b/integration-tests/standalone-asm.spec.js new file mode 100644 index 00000000000..d57a96f738e --- /dev/null +++ b/integration-tests/standalone-asm.spec.js @@ -0,0 +1,306 @@ +'use strict' + +const { assert } = require('chai') +const path = require('path') + +const { + createSandbox, + FakeAgent, + spawnProc, + curlAndAssertMessage, + curl +} = require('./helpers') + +describe('Standalone ASM', () => { + let sandbox, cwd, startupTestFile, agent, proc, env + + before(async () => { + sandbox = await createSandbox(['express']) + cwd = sandbox.folder + startupTestFile = path.join(cwd, 'standalone-asm/index.js') + }) + + after(async () => { + await sandbox.remove() + }) + + describe('enabled', () => { + beforeEach(async () => { + agent = await new FakeAgent().start() + + env = { + AGENT_PORT: agent.port, + DD_EXPERIMENTAL_APPSEC_STANDALONE_ENABLED: 'true' + } + + const execArgv = [] + + proc = await spawnProc(startupTestFile, { cwd, env, execArgv }) + }) + + afterEach(async () => { + proc.kill() + await agent.stop() + }) + + function assertKeep (payload, manual = true) { + const { meta, metrics } = payload + if (manual) { + assert.propertyVal(meta, 'manual.keep', 'true') + } else { + assert.notProperty(meta, 'manual.keep') + } + assert.propertyVal(meta, '_dd.p.appsec', '1') + + assert.propertyVal(metrics, '_sampling_priority_v1', 2) + assert.propertyVal(metrics, '_dd.apm.enabled', 0) + } + + function assertDrop (payload) { + const { metrics } = payload + assert.propertyVal(metrics, '_sampling_priority_v1', 0) + assert.propertyVal(metrics, '_dd.apm.enabled', 0) + assert.notProperty(metrics, '_dd.p.appsec') + } + + async function doWarmupRequests (procOrUrl, number = 3) { + for (let i = number; i > 0; i--) { + await curl(procOrUrl) + } + } + + // first req initializes the waf and reports the first appsec event adding manual.keep tag + it('should send correct headers and tags on first req', async () => { + return curlAndAssertMessage(agent, proc, ({ headers, payload }) => { + assert.propertyVal(headers, 'datadog-client-computed-stats', 'yes') + assert.isArray(payload) + assert.strictEqual(payload.length, 1) + assert.isArray(payload[0]) + + // express.request + 4 middlewares + assert.strictEqual(payload[0].length, 5) + + assertKeep(payload[0][0]) + }) + }) + + it('should keep second req because RateLimiter allows 1 req/min and discard the next', async () => { + // 1st req kept because waf init + // 2nd req kept because it's the first one hitting RateLimiter + // next in the first minute are dropped + await doWarmupRequests(proc) + + return curlAndAssertMessage(agent, proc, ({ headers, payload }) => { + assert.propertyVal(headers, 'datadog-client-computed-stats', 'yes') + assert.isArray(payload) + assert.strictEqual(payload.length, 4) + + const secondReq = payload[1] + assert.isArray(secondReq) + assert.strictEqual(secondReq.length, 5) + + const { meta, metrics } = secondReq[0] + assert.notProperty(meta, 'manual.keep') + assert.notProperty(meta, '_dd.p.appsec') + + assert.propertyVal(metrics, '_sampling_priority_v1', 1) + assert.propertyVal(metrics, '_dd.apm.enabled', 0) + + assertDrop(payload[2][0]) + + assertDrop(payload[3][0]) + }) + }) + + it('should keep attack requests', async () => { + await doWarmupRequests(proc) + + const urlAttack = proc.url + '?query=1 or 1=1' + return curlAndAssertMessage(agent, urlAttack, ({ headers, payload }) => { + assert.propertyVal(headers, 'datadog-client-computed-stats', 'yes') + assert.isArray(payload) + assert.strictEqual(payload.length, 4) + + assertKeep(payload[3][0]) + }) + }) + + it('should keep sdk events', async () => { + await doWarmupRequests(proc) + + const url = proc.url + '/login?user=test' + return curlAndAssertMessage(agent, url, ({ headers, payload }) => { + assert.propertyVal(headers, 'datadog-client-computed-stats', 'yes') + assert.isArray(payload) + assert.strictEqual(payload.length, 4) + + assertKeep(payload[3][0]) + }) + }) + + it('should keep custom sdk events', async () => { + await doWarmupRequests(proc) + + const url = proc.url + '/sdk' + return curlAndAssertMessage(agent, url, ({ headers, payload }) => { + assert.propertyVal(headers, 'datadog-client-computed-stats', 'yes') + assert.isArray(payload) + assert.strictEqual(payload.length, 4) + + assertKeep(payload[3][0]) + }) + }) + + it('should keep iast events', async () => { + await doWarmupRequests(proc) + + const url = proc.url + '/vulnerableHash' + return curlAndAssertMessage(agent, url, ({ headers, payload }) => { + assert.propertyVal(headers, 'datadog-client-computed-stats', 'yes') + assert.isArray(payload) + assert.strictEqual(payload.length, 4) + + const expressReq4 = payload[3][0] + assertKeep(expressReq4) + assert.property(expressReq4.meta, '_dd.iast.json') + assert.propertyVal(expressReq4.metrics, '_dd.iast.enabled', 1) + }) + }) + + describe('propagation', () => { + let proc2 + let port2 + + beforeEach(async () => { + const execArgv = [] + + proc2 = await spawnProc(startupTestFile, { cwd, env, execArgv }) + + port2 = parseInt(proc2.url.substring(proc2.url.lastIndexOf(':') + 1), 10) + }) + + afterEach(async () => { + proc2.kill() + }) + + // proc/drop-and-call-sdk: + // after setting a manual.drop calls to downstream proc2/sdk which triggers an appsec event + it('should keep trace even if parent prio is -1 but there is an event in the local trace', async () => { + await doWarmupRequests(proc) + await doWarmupRequests(proc2) + + const url = `${proc.url}/propagation-after-drop-and-call-sdk?port=${port2}` + return curlAndAssertMessage(agent, url, ({ headers, payload }) => { + assert.propertyVal(headers, 'datadog-client-computed-stats', 'yes') + assert.isArray(payload) + + const innerReq = payload.find(p => p[0].resource === 'GET /sdk') + assert.notStrictEqual(innerReq, undefined) + assertKeep(innerReq[0]) + }, undefined, undefined, true) + }) + + // proc/propagation-with-event triggers an appsec event and calls downstream proc2/down with no event + it('should keep if parent trace is (prio:2, _dd.p.appsec:1) but there is no event in the local trace', + async () => { + await doWarmupRequests(proc) + await doWarmupRequests(proc2) + + const url = `${proc.url}/propagation-with-event?port=${port2}` + return curlAndAssertMessage(agent, url, ({ headers, payload }) => { + assert.propertyVal(headers, 'datadog-client-computed-stats', 'yes') + assert.isArray(payload) + + const innerReq = payload.find(p => p[0].resource === 'GET /down') + assert.notStrictEqual(innerReq, undefined) + assertKeep(innerReq[0], false) + }, undefined, undefined, true) + }) + + it('should remove parent trace data if there is no event in the local trace', async () => { + await doWarmupRequests(proc) + await doWarmupRequests(proc2) + + const url = `${proc.url}/propagation-without-event?port=${port2}` + return curlAndAssertMessage(agent, url, ({ headers, payload }) => { + assert.propertyVal(headers, 'datadog-client-computed-stats', 'yes') + assert.isArray(payload) + + const innerReq = payload.find(p => p[0].resource === 'GET /down') + assert.notStrictEqual(innerReq, undefined) + assert.notProperty(innerReq[0].meta, '_dd.p.other') + }, undefined, undefined, true) + }) + + it('should not remove parent trace data if there is event in the local trace', async () => { + await doWarmupRequests(proc) + + const url = `${proc.url}/propagation-with-event?port=${port2}` + return curlAndAssertMessage(agent, url, ({ headers, payload }) => { + assert.propertyVal(headers, 'datadog-client-computed-stats', 'yes') + assert.isArray(payload) + + const innerReq = payload.find(p => p[0].resource === 'GET /down') + assert.notStrictEqual(innerReq, undefined) + assert.property(innerReq[0].meta, '_dd.p.other') + }, undefined, undefined, true) + }) + }) + }) + + describe('disabled', () => { + beforeEach(async () => { + agent = await new FakeAgent().start() + + env = { + AGENT_PORT: agent.port + } + + proc = await spawnProc(startupTestFile, { cwd, env }) + }) + + afterEach(async () => { + proc.kill() + await agent.stop() + }) + + it('should not add standalone related tags in iast events', () => { + const url = proc.url + '/vulnerableHash' + return curlAndAssertMessage(agent, url, ({ headers, payload }) => { + assert.notProperty(headers, 'datadog-client-computed-stats') + assert.isArray(payload) + assert.strictEqual(payload.length, 1) + assert.isArray(payload[0]) + + // express.request + 4 middlewares + assert.strictEqual(payload[0].length, 5) + + const { meta, metrics } = payload[0][0] + assert.property(meta, '_dd.iast.json') // WEAK_HASH and XCONTENTTYPE_HEADER_MISSING reported + + assert.notProperty(meta, '_dd.p.appsec') + assert.notProperty(metrics, '_dd.apm.enabled') + }) + }) + + it('should not add standalone related tags in appsec events', () => { + const urlAttack = proc.url + '?query=1 or 1=1' + + return curlAndAssertMessage(agent, urlAttack, ({ headers, payload }) => { + assert.notProperty(headers, 'datadog-client-computed-stats') + assert.isArray(payload) + assert.strictEqual(payload.length, 1) + assert.isArray(payload[0]) + + // express.request + 4 middlewares + assert.strictEqual(payload[0].length, 5) + + const { meta, metrics } = payload[0][0] + assert.property(meta, '_dd.appsec.json') // crs-942-100 triggered + + assert.notProperty(meta, '_dd.p.appsec') + assert.notProperty(metrics, '_dd.apm.enabled') + }) + }) + }) +}) diff --git a/integration-tests/standalone-asm/index.js b/integration-tests/standalone-asm/index.js new file mode 100644 index 00000000000..e0c92d61ff1 --- /dev/null +++ b/integration-tests/standalone-asm/index.js @@ -0,0 +1,116 @@ +'use strict' + +const options = { + appsec: { + enabled: true + }, + experimental: { + iast: { + enabled: true, + requestSampling: 100 + } + } +} + +if (process.env.AGENT_PORT) { + options.port = process.env.AGENT_PORT +} + +if (process.env.AGENT_URL) { + options.url = process.env.AGENT_URL +} + +const tracer = require('dd-trace') +tracer.init(options) + +const http = require('http') +const express = require('express') +const app = express() + +const valueToHash = 'iast-showcase-demo' +const crypto = require('crypto') + +async function makeRequest (url) { + return new Promise((resolve, reject) => { + http.get(url, function (res) { + const chunks = [] + + res.on('data', function (chunk) { + chunks.push(chunk) + }) + + res.on('end', () => { + resolve(Buffer.concat(chunks).toString('utf8')) + }) + + res.on('error', reject) + }) + }) +} + +app.get('/', (req, res) => { + res.status(200).send('hello world') +}) + +app.get('/login', (req, res) => { + tracer.appsec.trackUserLoginSuccessEvent({ id: req.query.user }) + res.status(200).send('login') +}) + +app.get('/sdk', (req, res) => { + tracer.appsec.trackCustomEvent('custom-event') + res.status(200).send('sdk') +}) + +app.get('/vulnerableHash', (req, res) => { + const result = crypto.createHash('sha1').update(valueToHash).digest('hex') + res.status(200).send(result) +}) + +app.get('/propagation-with-event', async (req, res) => { + tracer.appsec.trackCustomEvent('custom-event') + + const span = tracer.scope().active() + span.context()._trace.tags['_dd.p.other'] = '1' + + const port = req.query.port || server.address().port + const url = `http://localhost:${port}/down` + + await makeRequest(url) + + res.status(200).send('propagation-with-event') +}) + +app.get('/propagation-without-event', async (req, res) => { + const port = req.query.port || server.address().port + const url = `http://localhost:${port}/down` + + const span = tracer.scope().active() + span.context()._trace.tags['_dd.p.other'] = '1' + + await makeRequest(url) + + res.status(200).send('propagation-without-event') +}) + +app.get('/down', (req, res) => { + res.status(200).send('down') +}) + +app.get('/propagation-after-drop-and-call-sdk', async (req, res) => { + const span = tracer.scope().active() + span?.setTag('manual.drop', 'true') + + const port = req.query.port + + const url = `http://localhost:${port}/sdk` + + const sdkRes = await makeRequest(url) + + res.status(200).send(`drop-and-call-sdk ${sdkRes}`) +}) + +const server = http.createServer(app).listen(0, () => { + const port = server.address().port + process.send?.({ port }) +}) diff --git a/integration-tests/startup.spec.js b/integration-tests/startup.spec.js index 4033bc65ee9..c0365ef5fee 100644 --- a/integration-tests/startup.spec.js +++ b/integration-tests/startup.spec.js @@ -8,154 +8,179 @@ const { } = require('./helpers') const path = require('path') const { assert } = require('chai') - -describe('startup', () => { - let agent - let proc - let sandbox - let cwd - let startupTestFile - - before(async () => { - sandbox = await createSandbox() - cwd = sandbox.folder - startupTestFile = path.join(cwd, 'startup/index.js') - }) - - after(async () => { - await sandbox.remove() - }) - - context('programmatic', () => { - beforeEach(async () => { - agent = await new FakeAgent().start() +const semver = require('semver') + +const execArgvs = [ + { + execArgv: [] + }, + { + execArgv: ['--import', 'dd-trace/register.js'], + skip: semver.satisfies(process.versions.node, '<20.6') + }, + { + execArgv: ['--loader', 'dd-trace/loader-hook.mjs'], + skip: semver.satisfies(process.versions.node, '>=20.6') + } +] + +execArgvs.forEach(({ execArgv, skip }) => { + const describe = skip ? globalThis.describe.skip : globalThis.describe + + describe(`startup ${execArgv.join(' ')}`, () => { + let agent + let proc + let sandbox + let cwd + let startupTestFile + + before(async () => { + sandbox = await createSandbox() + cwd = sandbox.folder + startupTestFile = path.join(cwd, 'startup/index.js') }) - afterEach(async () => { - proc.kill() - await agent.stop() + after(async () => { + await sandbox.remove() }) - it('works for options.port', async () => { - proc = await spawnProc(startupTestFile, { - cwd, - env: { - AGENT_PORT: agent.port - } - }) - return curlAndAssertMessage(agent, proc, ({ headers, payload }) => { - assert.propertyVal(headers, 'host', `127.0.0.1:${agent.port}`) - assert.isArray(payload) - assert.strictEqual(payload.length, 1) - assert.isArray(payload[0]) - assert.strictEqual(payload[0].length, 1) - assert.propertyVal(payload[0][0], 'name', 'web.request') + context('programmatic', () => { + beforeEach(async () => { + agent = await new FakeAgent().start() }) - }) - it('works for options.url', async () => { - proc = await spawnProc(startupTestFile, { - cwd, - env: { - AGENT_URL: `http://localhost:${agent.port}` - } - }) - return curlAndAssertMessage(agent, proc, ({ headers, payload }) => { - assert.propertyVal(headers, 'host', `localhost:${agent.port}`) - assert.isArray(payload) - assert.strictEqual(payload.length, 1) - assert.isArray(payload[0]) - assert.strictEqual(payload[0].length, 1) - assert.propertyVal(payload[0][0], 'name', 'web.request') + afterEach(async () => { + proc.kill() + await agent.stop() }) - }) - }) - - context('env var', () => { - beforeEach(async () => { - agent = await new FakeAgent().start() - }) - afterEach(async () => { - proc.kill() - await agent.stop() - }) - - it('works for DD_TRACE_AGENT_PORT', async () => { - proc = await spawnProc(startupTestFile, { - cwd, - env: { - DD_TRACE_AGENT_PORT: agent.port - } + it('works for options.port', async () => { + proc = await spawnProc(startupTestFile, { + cwd, + execArgv, + env: { + AGENT_PORT: agent.port + } + }) + return curlAndAssertMessage(agent, proc, ({ headers, payload }) => { + assert.propertyVal(headers, 'host', `127.0.0.1:${agent.port}`) + assert.isArray(payload) + assert.strictEqual(payload.length, 1) + assert.isArray(payload[0]) + assert.strictEqual(payload[0].length, 1) + assert.propertyVal(payload[0][0], 'name', 'web.request') + }) }) - return curlAndAssertMessage(agent, proc, ({ headers, payload }) => { - assert.propertyVal(headers, 'host', `127.0.0.1:${agent.port}`) - assert.isArray(payload) - assert.strictEqual(payload.length, 1) - assert.isArray(payload[0]) - assert.strictEqual(payload[0].length, 1) - assert.propertyVal(payload[0][0], 'name', 'web.request') + + it('works for options.url', async () => { + proc = await spawnProc(startupTestFile, { + cwd, + execArgv, + env: { + AGENT_URL: `http://localhost:${agent.port}` + } + }) + return curlAndAssertMessage(agent, proc, ({ headers, payload }) => { + assert.propertyVal(headers, 'host', `localhost:${agent.port}`) + assert.isArray(payload) + assert.strictEqual(payload.length, 1) + assert.isArray(payload[0]) + assert.strictEqual(payload[0].length, 1) + assert.propertyVal(payload[0][0], 'name', 'web.request') + }) }) }) - it('works for DD_TRACE_AGENT_URL', async () => { - proc = await spawnProc(startupTestFile, { - cwd, - env: { - DD_TRACE_AGENT_URL: `http://localhost:${agent.port}` - } + context('env var', () => { + beforeEach(async () => { + agent = await new FakeAgent().start() }) - return curlAndAssertMessage(agent, proc, ({ headers, payload }) => { - assert.propertyVal(headers, 'host', `localhost:${agent.port}`) - assert.isArray(payload) - assert.strictEqual(payload.length, 1) - assert.isArray(payload[0]) - assert.strictEqual(payload[0].length, 1) - assert.propertyVal(payload[0][0], 'name', 'web.request') + + afterEach(async () => { + proc.kill() + await agent.stop() }) - }) - }) - context('default', () => { - beforeEach(async () => { - // Note that this test will *always* listen on the default port. If that - // port is unavailable, the test will fail. - agent = await new FakeAgent(8126).start() - }) + it('works for DD_TRACE_AGENT_PORT', async () => { + proc = await spawnProc(startupTestFile, { + cwd, + execArgv, + env: { + DD_TRACE_AGENT_PORT: agent.port + } + }) + return curlAndAssertMessage(agent, proc, ({ headers, payload }) => { + assert.propertyVal(headers, 'host', `127.0.0.1:${agent.port}`) + assert.isArray(payload) + assert.strictEqual(payload.length, 1) + assert.isArray(payload[0]) + assert.strictEqual(payload[0].length, 1) + assert.propertyVal(payload[0][0], 'name', 'web.request') + }) + }) - afterEach(async () => { - proc.kill() - await agent.stop() + it('works for DD_TRACE_AGENT_URL', async () => { + proc = await spawnProc(startupTestFile, { + cwd, + execArgv, + env: { + DD_TRACE_AGENT_URL: `http://localhost:${agent.port}` + } + }) + return curlAndAssertMessage(agent, proc, ({ headers, payload }) => { + assert.propertyVal(headers, 'host', `localhost:${agent.port}`) + assert.isArray(payload) + assert.strictEqual(payload.length, 1) + assert.isArray(payload[0]) + assert.strictEqual(payload[0].length, 1) + assert.propertyVal(payload[0][0], 'name', 'web.request') + }) + }) }) - it('works for hostname and port', async () => { - proc = await spawnProc(startupTestFile, { - cwd + context('default', () => { + beforeEach(async () => { + // Note that this test will *always* listen on the default port. If that + // port is unavailable, the test will fail. + agent = await new FakeAgent(8126).start() }) - return curlAndAssertMessage(agent, proc, ({ headers, payload }) => { - assert.propertyVal(headers, 'host', '127.0.0.1:8126') - assert.isArray(payload) - assert.strictEqual(payload.length, 1) - assert.isArray(payload[0]) - assert.strictEqual(payload[0].length, 1) - assert.propertyVal(payload[0][0], 'name', 'web.request') + + afterEach(async () => { + proc.kill() + await agent.stop() }) - }) - it('works with stealthy-require', async () => { - proc = await spawnProc(startupTestFile, { - cwd, - env: { - STEALTHY_REQUIRE: 'true' - } + it('works for hostname and port', async () => { + proc = await spawnProc(startupTestFile, { + cwd, + execArgv + }) + return curlAndAssertMessage(agent, proc, ({ headers, payload }) => { + assert.propertyVal(headers, 'host', '127.0.0.1:8126') + assert.isArray(payload) + assert.strictEqual(payload.length, 1) + assert.isArray(payload[0]) + assert.strictEqual(payload[0].length, 1) + assert.propertyVal(payload[0][0], 'name', 'web.request') + }) }) - return curlAndAssertMessage(agent, proc, ({ headers, payload }) => { - assert.propertyVal(headers, 'host', '127.0.0.1:8126') - assert.isArray(payload) - assert.strictEqual(payload.length, 1) - assert.isArray(payload[0]) - assert.strictEqual(payload[0].length, 1) - assert.propertyVal(payload[0][0], 'name', 'web.request') + + it('works with stealthy-require', async () => { + proc = await spawnProc(startupTestFile, { + cwd, + execArgv, + env: { + STEALTHY_REQUIRE: 'true' + } + }) + return curlAndAssertMessage(agent, proc, ({ headers, payload }) => { + assert.propertyVal(headers, 'host', '127.0.0.1:8126') + assert.isArray(payload) + assert.strictEqual(payload.length, 1) + assert.isArray(payload[0]) + assert.strictEqual(payload[0].length, 1) + assert.propertyVal(payload[0][0], 'name', 'web.request') + }) }) }) }) diff --git a/integration-tests/telemetry-forwarder.sh b/integration-tests/telemetry-forwarder.sh new file mode 100755 index 00000000000..5fe156993be --- /dev/null +++ b/integration-tests/telemetry-forwarder.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +# Implemented this in bash instead of Node.js for two reasons: +# 1. It's trivial in bash. +# 2. We're using NODE_OPTIONS in tests to init the tracer, and we don't want that for this script. + +echo "$1 $(cat -)" >> $FORWARDER_OUT diff --git a/integration-tests/test-api-manual.spec.js b/integration-tests/test-api-manual.spec.js index 7873a300ac3..419c7c736c5 100644 --- a/integration-tests/test-api-manual.spec.js +++ b/integration-tests/test-api-manual.spec.js @@ -17,6 +17,7 @@ const { describe('test-api-manual', () => { let sandbox, cwd, receiver, childProcess, webAppPort + before(async () => { sandbox = await createSandbox([], true) cwd = sandbox.folder @@ -72,7 +73,7 @@ describe('test-api-manual', () => { '--require ./ci-visibility/test-api-manual/test.fake.js ./ci-visibility/test-api-manual/run-fake-test-framework', { cwd, - env: { ...getCiVisAgentlessConfig(receiver.port), DD_CIVISIBILITY_MANUAL_API_ENABLED: '1' }, + env: getCiVisAgentlessConfig(receiver.port), stdio: 'pipe' } ) @@ -81,7 +82,7 @@ describe('test-api-manual', () => { }) }) - it('does not report test spans if DD_CIVISIBILITY_MANUAL_API_ENABLED is not set', (done) => { + it('does not report test spans if DD_CIVISIBILITY_MANUAL_API_ENABLED is set to false', (done) => { receiver.assertPayloadReceived(() => { const error = new Error('should not report spans') done(error) @@ -92,7 +93,10 @@ describe('test-api-manual', () => { '--require ./ci-visibility/test-api-manual/test.fake.js ./ci-visibility/test-api-manual/run-fake-test-framework', { cwd, - env: getCiVisAgentlessConfig(receiver.port), + env: { + ...getCiVisAgentlessConfig(receiver.port), + DD_CIVISIBILITY_MANUAL_API_ENABLED: 'false' + }, stdio: 'pipe' } ) diff --git a/integration-tests/vitest.config.mjs b/integration-tests/vitest.config.mjs new file mode 100644 index 00000000000..9a1572fb499 --- /dev/null +++ b/integration-tests/vitest.config.mjs @@ -0,0 +1,19 @@ +import { defineConfig } from 'vite' + +const config = { + test: { + include: [ + process.env.TEST_DIR || 'ci-visibility/vitest-tests/test-visibility*' + ] + } +} + +if (process.env.COVERAGE_PROVIDER) { + config.test.coverage = { + provider: process.env.COVERAGE_PROVIDER || 'v8', + include: ['ci-visibility/vitest-tests/**'], + reporter: ['text-summary'] + } +} + +export default defineConfig(config) diff --git a/integration-tests/vitest/vitest.spec.js b/integration-tests/vitest/vitest.spec.js new file mode 100644 index 00000000000..de38feee9da --- /dev/null +++ b/integration-tests/vitest/vitest.spec.js @@ -0,0 +1,900 @@ +'use strict' + +const { exec } = require('child_process') + +const { assert } = require('chai') + +const { + createSandbox, + getCiVisAgentlessConfig +} = require('../helpers') +const { FakeCiVisIntake } = require('../ci-visibility-intake') +const { + TEST_STATUS, + TEST_TYPE, + TEST_IS_RETRY, + TEST_CODE_OWNERS, + TEST_CODE_COVERAGE_LINES_PCT, + TEST_SESSION_NAME, + TEST_COMMAND, + TEST_LEVEL_EVENT_TYPES, + TEST_SOURCE_FILE, + TEST_SOURCE_START, + TEST_IS_NEW, + TEST_NAME, + TEST_EARLY_FLAKE_ENABLED, + TEST_EARLY_FLAKE_ABORT_REASON, + TEST_SUITE +} = require('../../packages/dd-trace/src/plugins/util/test') +const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') + +const NUM_RETRIES_EFD = 3 + +const versions = ['1.6.0', 'latest'] + +const linePctMatchRegex = /Lines\s+:\s+([\d.]+)%/ + +versions.forEach((version) => { + describe(`vitest@${version}`, () => { + let sandbox, cwd, receiver, childProcess, testOutput + + before(async function () { + sandbox = await createSandbox([ + `vitest@${version}`, + `@vitest/coverage-istanbul@${version}`, + `@vitest/coverage-v8@${version}` + ], true) + cwd = sandbox.folder + }) + + after(async () => { + await sandbox.remove() + }) + + beforeEach(async function () { + receiver = await new FakeCiVisIntake().start() + }) + + afterEach(async () => { + testOutput = '' + childProcess.kill() + await receiver.stop() + }) + + it('can run and report tests', (done) => { + receiver.gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => { + const metadataDicts = payloads.flatMap(({ payload }) => payload.metadata) + + metadataDicts.forEach(metadata => { + for (const testLevel of TEST_LEVEL_EVENT_TYPES) { + assert.equal(metadata[testLevel][TEST_SESSION_NAME], 'my-test-session') + } + }) + + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSessionEvent = events.find(event => event.type === 'test_session_end') + const testModuleEvent = events.find(event => event.type === 'test_module_end') + const testSuiteEvents = events.filter(event => event.type === 'test_suite_end') + const testEvents = events.filter(event => event.type === 'test') + + assert.include(testSessionEvent.content.resource, 'test_session.vitest run') + assert.equal(testSessionEvent.content.meta[TEST_STATUS], 'fail') + assert.include(testModuleEvent.content.resource, 'test_module.vitest run') + assert.equal(testModuleEvent.content.meta[TEST_STATUS], 'fail') + assert.equal(testSessionEvent.content.meta[TEST_TYPE], 'test') + assert.equal(testModuleEvent.content.meta[TEST_TYPE], 'test') + + const passedSuite = testSuiteEvents.find( + suite => suite.content.resource === 'test_suite.ci-visibility/vitest-tests/test-visibility-passed-suite.mjs' + ) + assert.equal(passedSuite.content.meta[TEST_STATUS], 'pass') + + const failedSuite = testSuiteEvents.find( + suite => suite.content.resource === 'test_suite.ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + ) + assert.equal(failedSuite.content.meta[TEST_STATUS], 'fail') + + const failedSuiteHooks = testSuiteEvents.find( + suite => suite.content.resource === 'test_suite.ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs' + ) + assert.equal(failedSuiteHooks.content.meta[TEST_STATUS], 'fail') + + assert.includeMembers(testEvents.map(test => test.content.resource), + [ + 'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + + '.test-visibility-failed-suite-first-describe can report failed test', + 'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + + '.test-visibility-failed-suite-first-describe can report more', + 'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + + '.test-visibility-failed-suite-second-describe can report passed test', + 'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + + '.test-visibility-failed-suite-second-describe can report more', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.context can report passed test', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.context can report more', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can report passed test', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can report more', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can skip', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can todo', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report failed test', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report more', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report passed test', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report more', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.no suite', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.skip no suite', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.programmatic skip no suite' + ] + ) + + const failedTests = testEvents.filter(test => test.content.meta[TEST_STATUS] === 'fail') + + assert.includeMembers( + failedTests.map(test => test.content.resource), + [ + 'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + + '.test-visibility-failed-suite-first-describe can report failed test', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report failed test', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report more', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report passed test', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report more' + ] + ) + + const skippedTests = testEvents.filter(test => test.content.meta[TEST_STATUS] === 'skip') + + assert.includeMembers( + skippedTests.map(test => test.content.resource), + [ + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can skip', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can todo', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can programmatic skip' + ] + ) + + testEvents.forEach(test => { + assert.equal(test.content.meta[TEST_COMMAND], 'vitest run') + assert.exists(test.content.metrics[DD_HOST_CPU_COUNT]) + }) + + testSuiteEvents.forEach(testSuite => { + assert.equal(testSuite.content.meta[TEST_COMMAND], 'vitest run') + assert.isTrue( + testSuite.content.meta[TEST_SOURCE_FILE].startsWith('ci-visibility/vitest-tests/test-visibility') + ) + assert.equal(testSuite.content.metrics[TEST_SOURCE_START], 1) + assert.exists(testSuite.content.metrics[DD_HOST_CPU_COUNT]) + }) + // TODO: check error messages + }).then(() => done()).catch(done) + + childProcess = exec( + './node_modules/.bin/vitest run', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init', // ESM requires more flags + DD_TEST_SESSION_NAME: 'my-test-session' + }, + stdio: 'pipe' + } + ) + }) + + context('flaky test retries', () => { + it('can retry flaky tests', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + flaky_test_retries_enabled: true, + early_flake_detection: { + enabled: false + } + }) + + receiver.gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testEvents = events.filter(event => event.type === 'test') + assert.equal(testEvents.length, 11) + assert.includeMembers(testEvents.map(test => test.content.resource), [ + 'ci-visibility/vitest-tests/flaky-test-retries.mjs.flaky test retries can retry tests that eventually pass', + 'ci-visibility/vitest-tests/flaky-test-retries.mjs.flaky test retries can retry tests that eventually pass', + 'ci-visibility/vitest-tests/flaky-test-retries.mjs.flaky test retries can retry tests that eventually pass', + // passes at the third retry + 'ci-visibility/vitest-tests/flaky-test-retries.mjs.flaky test retries can retry tests that eventually pass', + 'ci-visibility/vitest-tests/flaky-test-retries.mjs.flaky test retries can retry tests that never pass', + 'ci-visibility/vitest-tests/flaky-test-retries.mjs.flaky test retries can retry tests that never pass', + 'ci-visibility/vitest-tests/flaky-test-retries.mjs.flaky test retries can retry tests that never pass', + 'ci-visibility/vitest-tests/flaky-test-retries.mjs.flaky test retries can retry tests that never pass', + 'ci-visibility/vitest-tests/flaky-test-retries.mjs.flaky test retries can retry tests that never pass', + // never passes + 'ci-visibility/vitest-tests/flaky-test-retries.mjs.flaky test retries can retry tests that never pass', + // passes on the first try + 'ci-visibility/vitest-tests/flaky-test-retries.mjs.flaky test retries does not retry if unnecessary' + ]) + const eventuallyPassingTest = testEvents.filter( + test => test.content.resource === + 'ci-visibility/vitest-tests/flaky-test-retries.mjs.flaky test retries can retry tests that eventually pass' + ) + assert.equal(eventuallyPassingTest.length, 4) + assert.equal(eventuallyPassingTest.filter(test => test.content.meta[TEST_STATUS] === 'fail').length, 3) + assert.equal(eventuallyPassingTest.filter(test => test.content.meta[TEST_STATUS] === 'pass').length, 1) + assert.equal(eventuallyPassingTest.filter(test => test.content.meta[TEST_IS_RETRY] === 'true').length, 3) + + const neverPassingTest = testEvents.filter( + test => test.content.resource === + 'ci-visibility/vitest-tests/flaky-test-retries.mjs.flaky test retries can retry tests that never pass' + ) + assert.equal(neverPassingTest.length, 6) + assert.equal(neverPassingTest.filter(test => test.content.meta[TEST_STATUS] === 'fail').length, 6) + assert.equal(neverPassingTest.filter(test => test.content.meta[TEST_STATUS] === 'pass').length, 0) + assert.equal(neverPassingTest.filter(test => test.content.meta[TEST_IS_RETRY] === 'true').length, 5) + }).then(() => done()).catch(done) + + childProcess = exec( + './node_modules/.bin/vitest run', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TEST_DIR: 'ci-visibility/vitest-tests/flaky-test-retries*', + NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init' // ESM requires more flags + }, + stdio: 'pipe' + } + ) + }) + + it('is disabled if DD_CIVISIBILITY_FLAKY_RETRY_ENABLED is false', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + flaky_test_retries_enabled: true, + early_flake_detection: { + enabled: false + } + }) + + receiver.gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testEvents = events.filter(event => event.type === 'test') + assert.equal(testEvents.length, 3) + assert.includeMembers(testEvents.map(test => test.content.resource), [ + 'ci-visibility/vitest-tests/flaky-test-retries.mjs.flaky test retries can retry tests that eventually pass', + 'ci-visibility/vitest-tests/flaky-test-retries.mjs.flaky test retries can retry tests that never pass', + 'ci-visibility/vitest-tests/flaky-test-retries.mjs.flaky test retries does not retry if unnecessary' + ]) + assert.equal(testEvents.filter(test => test.content.meta[TEST_IS_RETRY] === 'true').length, 0) + }).then(() => done()).catch(done) + + childProcess = exec( + './node_modules/.bin/vitest run', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TEST_DIR: 'ci-visibility/vitest-tests/flaky-test-retries*', + DD_CIVISIBILITY_FLAKY_RETRY_ENABLED: 'false', + NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init' // ESM requires more flags + }, + stdio: 'pipe' + } + ) + }) + + it('retries DD_CIVISIBILITY_FLAKY_RETRY_COUNT times', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + flaky_test_retries_enabled: true, + early_flake_detection: { + enabled: false + } + }) + + receiver.gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testEvents = events.filter(event => event.type === 'test') + assert.equal(testEvents.length, 5) + assert.includeMembers(testEvents.map(test => test.content.resource), [ + 'ci-visibility/vitest-tests/flaky-test-retries.mjs.flaky test retries can retry tests that eventually pass', + 'ci-visibility/vitest-tests/flaky-test-retries.mjs.flaky test retries can retry tests that eventually pass', + 'ci-visibility/vitest-tests/flaky-test-retries.mjs.flaky test retries can retry tests that never pass', + 'ci-visibility/vitest-tests/flaky-test-retries.mjs.flaky test retries can retry tests that never pass', + 'ci-visibility/vitest-tests/flaky-test-retries.mjs.flaky test retries does not retry if unnecessary' + ]) + assert.equal(testEvents.filter(test => test.content.meta[TEST_IS_RETRY] === 'true').length, 2) + }).then(() => done()).catch(done) + + childProcess = exec( + './node_modules/.bin/vitest run', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TEST_DIR: 'ci-visibility/vitest-tests/flaky-test-retries*', + DD_CIVISIBILITY_FLAKY_RETRY_COUNT: 1, + NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init' // ESM requires more flags + }, + stdio: 'pipe' + } + ) + }) + }) + + it('correctly calculates test code owners when working directory is not repository root', (done) => { + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const test = events.find(event => event.type === 'test').content + const testSuite = events.find(event => event.type === 'test_suite_end').content + assert.equal(test.meta[TEST_CODE_OWNERS], JSON.stringify(['@datadog-dd-trace-js'])) + assert.equal(testSuite.meta[TEST_CODE_OWNERS], JSON.stringify(['@datadog-dd-trace-js'])) + }) + + childProcess = exec( + '../../node_modules/.bin/vitest run', + { + cwd: `${cwd}/ci-visibility/subproject`, + env: { + ...getCiVisAgentlessConfig(receiver.port), + NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init', // ESM requires more flags + TEST_DIR: './vitest-test.mjs' + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + // total code coverage only works for >=2.0.0 + if (version === 'latest') { + const coverageProviders = ['v8', 'istanbul'] + + coverageProviders.forEach((coverageProvider) => { + it(`reports code coverage for ${coverageProvider} provider`, (done) => { + let codeCoverageExtracted + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + + codeCoverageExtracted = testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT] + }) + + childProcess = exec( + './node_modules/.bin/vitest run --coverage', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init', + COVERAGE_PROVIDER: coverageProvider, + TEST_DIR: 'ci-visibility/vitest-tests/coverage-test*' + }, + stdio: 'inherit' + } + ) + + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + const linePctMatch = testOutput.match(linePctMatchRegex) + const linesPctFromNyc = linePctMatch ? Number(linePctMatch[1]) : null + + assert.equal( + linesPctFromNyc, + codeCoverageExtracted, + 'coverage reported by vitest does not match extracted coverage' + ) + done() + }).catch(done) + }) + }) + }) + } + // maybe only latest version? + context('early flake detection', () => { + it('retries new tests', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } + } + }) + + receiver.setKnownTests({ + vitest: { + 'ci-visibility/vitest-tests/early-flake-detection.mjs': [ + // 'early flake detection can retry tests that eventually pass', // will be considered new + // 'early flake detection can retry tests that always pass', // will be considered new + // 'early flake detection can retry tests that eventually fail', // will be considered new + // 'early flake detection does not retry if the test is skipped', // skipped so not retried + 'early flake detection does not retry if it is not new' + ] + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(test => test.content) + + assert.equal(tests.length, 14) + + assert.includeMembers(tests.map(test => test.meta[TEST_NAME]), [ + 'early flake detection can retry tests that eventually pass', + 'early flake detection can retry tests that eventually pass', + 'early flake detection can retry tests that eventually pass', + 'early flake detection can retry tests that eventually pass', + 'early flake detection can retry tests that eventually fail', + 'early flake detection can retry tests that eventually fail', + 'early flake detection can retry tests that eventually fail', + 'early flake detection can retry tests that eventually fail', + 'early flake detection can retry tests that always pass', + 'early flake detection can retry tests that always pass', + 'early flake detection can retry tests that always pass', + 'early flake detection can retry tests that always pass', + 'early flake detection does not retry if it is not new', + 'early flake detection does not retry if the test is skipped' + ]) + const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') + assert.equal(newTests.length, 12) // 4 executions of the three new tests + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 9) // 3 retries of the three new tests + + // exit code should be 0 and test session should be reported as passed, + // even though there are some failing executions + const failedTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(failedTests.length, 3) + const testSessionEvent = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSessionEvent.meta, TEST_STATUS, 'pass') + assert.propertyVal(testSessionEvent.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + }) + + childProcess = exec( + './node_modules/.bin/vitest run', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TEST_DIR: 'ci-visibility/vitest-tests/early-flake-detection*', + NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init', + SHOULD_ADD_EVENTUALLY_FAIL: '1' + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', (exitCode) => { + eventsPromise.then(() => { + assert.equal(exitCode, 0) + done() + }).catch(done) + }) + }) + + it('fails if all the attempts fail', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } + } + }) + + receiver.setKnownTests({ + vitest: { + 'ci-visibility/vitest-tests/early-flake-detection.mjs': [ + // 'early flake detection can retry tests that eventually pass', // will be considered new + // 'early flake detection can retry tests that always pass', // will be considered new + // 'early flake detection does not retry if the test is skipped', // skipped so not retried + 'early flake detection does not retry if it is not new' + ] + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(test => test.content) + + assert.equal(tests.length, 10) + + assert.includeMembers(tests.map(test => test.meta[TEST_NAME]), [ + 'early flake detection can retry tests that eventually pass', + 'early flake detection can retry tests that eventually pass', + 'early flake detection can retry tests that eventually pass', + 'early flake detection can retry tests that eventually pass', + 'early flake detection can retry tests that always pass', + 'early flake detection can retry tests that always pass', + 'early flake detection can retry tests that always pass', + 'early flake detection can retry tests that always pass', + 'early flake detection does not retry if it is not new', + 'early flake detection does not retry if the test is skipped' + ]) + const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') + assert.equal(newTests.length, 8) // 4 executions of the two new tests + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 6) // 3 retries of the two new tests + + // the multiple attempts did not result in a single pass, + // so the test session should be reported as failed + const failedTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(failedTests.length, 6) + const testSessionEvent = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSessionEvent.meta, TEST_STATUS, 'fail') + assert.propertyVal(testSessionEvent.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + }) + + childProcess = exec( + './node_modules/.bin/vitest run', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TEST_DIR: 'ci-visibility/vitest-tests/early-flake-detection*', + NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init', + ALWAYS_FAIL: 'true' + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', (exitCode) => { + eventsPromise.then(() => { + assert.equal(exitCode, 1) + done() + }).catch(done) + }) + }) + + it('bails out of EFD if the percentage of new tests is too high', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 0 + } + }) + + receiver.setKnownTests({ + vitest: {} + }) // tests from ci-visibility/vitest-tests/early-flake-detection.mjs will be new + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ABORT_REASON, 'faulty') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + assert.equal(tests.length, 4) + + const newTests = tests.filter( + test => test.meta[TEST_IS_NEW] === 'true' + ) + // no new tests + assert.equal(newTests.length, 0) + }) + + childProcess = exec( + './node_modules/.bin/vitest run', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TEST_DIR: 'ci-visibility/vitest-tests/early-flake-detection*', + NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init', + DD_TRACE_DEBUG: '1', + DD_TRACE_LOG_LEVEL: 'error' + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', (exitCode) => { + eventsPromise.then(() => { + assert.equal(exitCode, 1) + done() + }).catch(done) + }) + }) + + it('is disabled if DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED is false', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } + } + }) + + receiver.setKnownTests({ + vitest: { + 'ci-visibility/vitest-tests/early-flake-detection.mjs': [ + // 'early flake detection can retry tests that eventually pass', // will be considered new + // 'early flake detection can retry tests that always pass', // will be considered new + // 'early flake detection does not retry if the test is skipped', // skipped so not retried + 'early flake detection does not retry if it is not new' + ] + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(test => test.content) + + assert.equal(tests.length, 4) + + assert.includeMembers(tests.map(test => test.meta[TEST_NAME]), [ + 'early flake detection can retry tests that eventually pass', + 'early flake detection can retry tests that always pass', + 'early flake detection does not retry if it is not new', + 'early flake detection does not retry if the test is skipped' + ]) + const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') + assert.equal(newTests.length, 0) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 0) + + const failedTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(failedTests.length, 1) + const testSessionEvent = events.find(event => event.type === 'test_session_end').content + assert.equal(testSessionEvent.meta[TEST_STATUS], 'fail') + }) + + childProcess = exec( + './node_modules/.bin/vitest run', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TEST_DIR: 'ci-visibility/vitest-tests/early-flake-detection*', + NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init', + DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED: 'false' + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', (exitCode) => { + eventsPromise.then(() => { + assert.equal(exitCode, 1) + done() + }).catch(done) + }) + }) + + it('does not run EFD if the known tests request fails', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } + } + }) + + receiver.setKnownTestsResponseCode(500) + receiver.setKnownTests({}) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(test => test.content) + + assert.equal(tests.length, 4) + + assert.includeMembers(tests.map(test => test.meta[TEST_NAME]), [ + 'early flake detection can retry tests that eventually pass', + 'early flake detection can retry tests that always pass', + 'early flake detection does not retry if it is not new', + 'early flake detection does not retry if the test is skipped' + ]) + const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') + assert.equal(newTests.length, 0) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 0) + + const failedTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(failedTests.length, 1) + const testSessionEvent = events.find(event => event.type === 'test_session_end').content + assert.equal(testSessionEvent.meta[TEST_STATUS], 'fail') + }) + + childProcess = exec( + './node_modules/.bin/vitest run', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TEST_DIR: 'ci-visibility/vitest-tests/early-flake-detection*', + NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init' + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', (exitCode) => { + eventsPromise.then(() => { + assert.equal(exitCode, 1) + done() + }).catch(done) + }) + }) + + it('works when the cwd is not the repository root', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } + } + }) + + receiver.setKnownTests({ + vitest: { + 'ci-visibility/subproject/vitest-test.mjs': [ + 'context can report passed test' // no test will be considered new + ] + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(test => test.content) + + // no retries + assert.equal(tests.length, 1) + + assert.propertyVal(tests[0].meta, TEST_SUITE, 'ci-visibility/subproject/vitest-test.mjs') + // it's not considered new + assert.notProperty(tests[0].meta, TEST_IS_NEW) + }) + + childProcess = exec( + '../../node_modules/.bin/vitest run', + { + cwd: `${cwd}/ci-visibility/subproject`, + env: { + ...getCiVisAgentlessConfig(receiver.port), + NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init', // ESM requires more flags + TEST_DIR: './vitest-test.mjs' + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', (exitCode) => { + eventsPromise.then(() => { + assert.equal(exitCode, 0) + done() + }).catch(done) + }) + }) + + it('works with repeats config when EFD is disabled', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: false + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(test => test.content) + + assert.equal(tests.length, 8) + + assert.includeMembers(tests.map(test => test.meta[TEST_NAME]), [ + 'early flake detection can retry tests that eventually pass', + 'early flake detection can retry tests that eventually pass', + 'early flake detection can retry tests that eventually pass', // repeated twice + 'early flake detection can retry tests that always pass', + 'early flake detection can retry tests that always pass', + 'early flake detection can retry tests that always pass', // repeated twice + 'early flake detection does not retry if it is not new', + 'early flake detection does not retry if the test is skipped' + ]) + const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') + assert.equal(newTests.length, 0) // no new test detected + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 4) // 2 repetitions on 2 tests + + // vitest reports the test as failed if any of the repetitions fail, so we'll follow that + // TODO: we might want to improve htis + const failedTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(failedTests.length, 3) + + const testSessionEvent = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSessionEvent.meta, TEST_STATUS, 'fail') + assert.notProperty(testSessionEvent.meta, TEST_EARLY_FLAKE_ENABLED) + }) + + childProcess = exec( + './node_modules/.bin/vitest run', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TEST_DIR: 'ci-visibility/vitest-tests/early-flake-detection*', + NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init', + SHOULD_REPEAT: '1' + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', (exitCode) => { + eventsPromise.then(() => { + assert.equal(exitCode, 1) + done() + }).catch(done) + }) + }) + }) + }) +}) diff --git a/lib-injection/Dockerfile b/lib-injection/Dockerfile deleted file mode 100644 index e062c325399..00000000000 --- a/lib-injection/Dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -FROM scratch AS nodejs_agent -COPY . / - - -FROM node:16-alpine AS build - -ARG npm_pkg -ARG UID=10000 - -RUN addgroup -g 10000 -S datadog \ - && adduser -u ${UID} -S datadog -G datadog - -WORKDIR /operator-build -COPY --from=nodejs_agent . . - -COPY . . -RUN chmod 755 npm_dd_trace.sh -RUN ./npm_dd_trace.sh - -USER ${UID} - -ADD copy-lib.sh /operator-build/copy-lib.sh diff --git a/lib-injection/copy-lib.sh b/lib-injection/copy-lib.sh deleted file mode 100644 index 8803500f98a..00000000000 --- a/lib-injection/copy-lib.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -cp -r node_modules "$1/node_modules" -ls "$1" diff --git a/lib-injection/npm_dd_trace.sh b/lib-injection/npm_dd_trace.sh deleted file mode 100755 index f212f6c97b3..00000000000 --- a/lib-injection/npm_dd_trace.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh -if [ -e "dd-trace.tgz" ]; then - npm install ./dd-trace.tgz -else - npm install dd-trace -fi \ No newline at end of file diff --git a/lib-injection/package.json b/lib-injection/package.json deleted file mode 100644 index df7bb200cf9..00000000000 --- a/lib-injection/package.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "name": "@datadog/k8s-autoinstrumentation", - "version": "0.0.1", - "private": true, - "dependencies": {} -} diff --git a/node-upstream-tests/node/run_tests.js b/node-upstream-tests/node/run_tests.js index e8df64c97bf..8f2c758d07e 100644 --- a/node-upstream-tests/node/run_tests.js +++ b/node-upstream-tests/node/run_tests.js @@ -6,8 +6,8 @@ const childProcess = require('child_process') const path = require('path') const fsUtils = require('./fs_utils') -const NODE_BIN = process.env['NODE_BIN'] || '/usr/bin/node' -const NODE_REPO_PATH = process.env['NODE_REPO_PATH'] +const NODE_BIN = process.env.NODE_BIN || '/usr/bin/node' +const NODE_REPO_PATH = process.env.NODE_REPO_PATH if (NODE_REPO_PATH === undefined) { throw new Error('The env variable NODE_REPO_PATH is not set. This is required to locate the root of the nodejs repo') } @@ -244,6 +244,7 @@ class TestResult { this.isPass = null this.isIgnore = null } + async init () { this.isPass = this.rc === 0 @@ -257,6 +258,7 @@ class TestResult { return this } + errorMessage () { let message = '' message += `Test output: rc ${this.rc}\n` diff --git a/package.json b/package.json index 3ad73ae3260..84fbe163eab 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "dd-trace", - "version": "5.0.0-pre", + "version": "6.0.0-pre", "description": "Datadog APM tracing client for JavaScript", "main": "index.js", "typings": "index.d.ts", @@ -12,37 +12,45 @@ "bench:e2e:ci-visibility": "node benchmark/e2e-ci/benchmark-run.js", "type:doc": "cd docs && yarn && yarn build", "type:test": "cd docs && yarn && yarn test", - "lint": "node scripts/check_licenses.js && eslint . && yarn audit --groups dependencies", + "lint": "node scripts/check_licenses.js && eslint . && yarn audit", + "lint-fix": "node scripts/check_licenses.js && eslint . --fix && yarn audit", "services": "node ./scripts/install_plugin_modules && node packages/dd-trace/test/setup/services", - "test": "SERVICES=* yarn services && mocha --colors --exit --expose-gc 'packages/dd-trace/test/setup/node.js' 'packages/*/test/**/*.spec.js'", - "test:appsec": "mocha --colors --exit -r \"packages/dd-trace/test/setup/mocha.js\" --exclude \"packages/dd-trace/test/appsec/**/*.plugin.spec.js\" \"packages/dd-trace/test/appsec/**/*.spec.js\"", + "test": "SERVICES=* yarn services && mocha --expose-gc 'packages/dd-trace/test/setup/node.js' 'packages/*/test/**/*.spec.js'", + "test:appsec": "mocha -r \"packages/dd-trace/test/setup/mocha.js\" --exclude \"packages/dd-trace/test/appsec/**/*.plugin.spec.js\" \"packages/dd-trace/test/appsec/**/*.spec.js\"", "test:appsec:ci": "nyc --no-clean --include \"packages/dd-trace/src/appsec/**/*.js\" --exclude \"packages/dd-trace/test/appsec/**/*.plugin.spec.js\" -- npm run test:appsec", - "test:appsec:plugins": "mocha --colors --exit -r \"packages/dd-trace/test/setup/mocha.js\" \"packages/dd-trace/test/appsec/**/*.@($(echo $PLUGINS)).plugin.spec.js\"", + "test:appsec:plugins": "mocha -r \"packages/dd-trace/test/setup/mocha.js\" \"packages/dd-trace/test/appsec/**/*.@($(echo $PLUGINS)).plugin.spec.js\"", "test:appsec:plugins:ci": "yarn services && nyc --no-clean --include \"packages/dd-trace/src/appsec/**/*.js\" -- npm run test:appsec:plugins", + "test:debugger": "mocha -r 'packages/dd-trace/test/setup/mocha.js' 'packages/dd-trace/test/debugger/**/*.spec.js'", + "test:debugger:ci": "nyc --no-clean --include 'packages/dd-trace/src/debugger/**/*.js' -- npm run test:debugger", "test:trace:core": "tap packages/dd-trace/test/*.spec.js \"packages/dd-trace/test/{ci-visibility,datastreams,encode,exporters,opentelemetry,opentracing,plugins,service-naming,telemetry}/**/*.spec.js\"", "test:trace:core:ci": "npm run test:trace:core -- --coverage --nyc-arg=--include=\"packages/dd-trace/src/**/*.js\"", - "test:instrumentations": "mocha --colors -r 'packages/dd-trace/test/setup/mocha.js' 'packages/datadog-instrumentations/test/**/*.spec.js'", + "test:instrumentations": "mocha -r 'packages/dd-trace/test/setup/mocha.js' 'packages/datadog-instrumentations/test/**/*.spec.js'", "test:instrumentations:ci": "nyc --no-clean --include 'packages/datadog-instrumentations/src/**/*.js' -- npm run test:instrumentations", "test:core": "tap \"packages/datadog-core/test/**/*.spec.js\"", "test:core:ci": "npm run test:core -- --coverage --nyc-arg=--include=\"packages/datadog-core/src/**/*.js\"", - "test:lambda": "mocha --colors --exit -r \"packages/dd-trace/test/setup/mocha.js\" \"packages/dd-trace/test/lambda/**/*.spec.js\"", + "test:lambda": "mocha -r \"packages/dd-trace/test/setup/mocha.js\" \"packages/dd-trace/test/lambda/**/*.spec.js\"", "test:lambda:ci": "nyc --no-clean --include \"packages/dd-trace/src/lambda/**/*.js\" -- npm run test:lambda", - "test:plugins": "mocha --colors --exit -r \"packages/dd-trace/test/setup/mocha.js\" \"packages/datadog-instrumentations/test/@($(echo $PLUGINS)).spec.js\" \"packages/datadog-plugin-@($(echo $PLUGINS))/test/**/*.spec.js\"", + "test:plugins": "mocha -r \"packages/dd-trace/test/setup/mocha.js\" \"packages/datadog-instrumentations/test/@($(echo $PLUGINS)).spec.js\" \"packages/datadog-plugin-@($(echo $PLUGINS))/test/**/*.spec.js\"", "test:plugins:ci": "yarn services && nyc --no-clean --include \"packages/datadog-instrumentations/src/@($(echo $PLUGINS)).js\" --include \"packages/datadog-instrumentations/src/@($(echo $PLUGINS))/**/*.js\" --include \"packages/datadog-plugin-@($(echo $PLUGINS))/src/**/*.js\" -- npm run test:plugins", "test:plugins:upstream": "node ./packages/dd-trace/test/plugins/suite.js", "test:profiler": "tap \"packages/dd-trace/test/profiling/**/*.spec.js\"", "test:profiler:ci": "npm run test:profiler -- --coverage --nyc-arg=--include=\"packages/dd-trace/src/profiling/**/*.js\"", - "test:integration": "mocha --colors --timeout 30000 \"integration-tests/*.spec.js\"", - "test:integration:cucumber": "mocha --colors --timeout 30000 \"integration-tests/cucumber/*.spec.js\"", - "test:integration:cypress": "mocha --colors --timeout 30000 \"integration-tests/cypress/*.spec.js\"", - "test:integration:playwright": "mocha --colors --timeout 30000 \"integration-tests/playwright/*.spec.js\"", - "test:integration:serverless": "mocha --colors --timeout 30000 \"integration-tests/serverless/*.spec.js\"", - "test:integration:plugins": "mocha --colors --exit -r \"packages/dd-trace/test/setup/mocha.js\" \"packages/datadog-plugin-@($(echo $PLUGINS))/test/integration-test/**/*.spec.js\"", - "test:unit:plugins": "mocha --colors --exit -r \"packages/dd-trace/test/setup/mocha.js\" \"packages/datadog-instrumentations/test/@($(echo $PLUGINS)).spec.js\" \"packages/datadog-plugin-@($(echo $PLUGINS))/test/**/*.spec.js\" --exclude \"packages/datadog-plugin-@($(echo $PLUGINS))/test/integration-test/**/*.spec.js\"", - "test:shimmer": "mocha --colors 'packages/datadog-shimmer/test/**/*.spec.js'", - "test:shimmer:ci": "nyc --no-clean --include 'packages/datadog-shimmer/src/**/*.js' -- npm run test:shimmer", - "leak:core": "node ./scripts/install_plugin_modules && (cd packages/memwatch && yarn) && NODE_PATH=./packages/memwatch/node_modules node --no-warnings ./node_modules/.bin/tape 'packages/dd-trace/test/leak/**/*.js'", - "leak:plugins": "yarn services && (cd packages/memwatch && yarn) && NODE_PATH=./packages/memwatch/node_modules node --no-warnings ./node_modules/.bin/tape \"packages/datadog-plugin-@($(echo $PLUGINS))/test/leak.js\"" + "test:integration": "mocha --timeout 60000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/*.spec.js\"", + "test:integration:appsec": "mocha --timeout 60000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/appsec/*.spec.js\"", + "test:integration:cucumber": "mocha --timeout 60000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/cucumber/*.spec.js\"", + "test:integration:cypress": "mocha --timeout 60000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/cypress/*.spec.js\"", + "test:integration:debugger": "mocha --timeout 60000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/debugger/*.spec.js\"", + "test:integration:jest": "mocha --timeout 60000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/jest/*.spec.js\"", + "test:integration:mocha": "mocha --timeout 60000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/mocha/*.spec.js\"", + "test:integration:playwright": "mocha --timeout 60000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/playwright/*.spec.js\"", + "test:integration:selenium": "mocha --timeout 60000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/selenium/*.spec.js\"", + "test:integration:vitest": "mocha --timeout 60000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/vitest/*.spec.js\"", + "test:integration:profiler": "mocha --timeout 180000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/profiler/*.spec.js\"", + "test:integration:serverless": "mocha --timeout 60000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/serverless/*.spec.js\"", + "test:integration:plugins": "mocha -r \"packages/dd-trace/test/setup/mocha.js\" \"packages/datadog-plugin-@($(echo $PLUGINS))/test/integration-test/**/*.spec.js\"", + "test:unit:plugins": "mocha -r \"packages/dd-trace/test/setup/mocha.js\" \"packages/datadog-instrumentations/test/@($(echo $PLUGINS)).spec.js\" \"packages/datadog-plugin-@($(echo $PLUGINS))/test/**/*.spec.js\" --exclude \"packages/datadog-plugin-@($(echo $PLUGINS))/test/integration-test/**/*.spec.js\"", + "test:shimmer": "mocha 'packages/datadog-shimmer/test/**/*.spec.js'", + "test:shimmer:ci": "nyc --no-clean --include 'packages/datadog-shimmer/src/**/*.js' -- npm run test:shimmer" }, "repository": { "type": "git", @@ -65,64 +73,60 @@ }, "homepage": "https://github.com/DataDog/dd-trace-js#readme", "engines": { - "node": ">=16" + "node": ">=18" }, "dependencies": { - "@datadog/native-appsec": "5.0.0", - "@datadog/native-iast-rewriter": "2.2.1", - "@datadog/native-iast-taint-tracking": "1.6.4", + "@datadog/native-appsec": "8.2.1", + "@datadog/native-iast-rewriter": "2.5.0", + "@datadog/native-iast-taint-tracking": "3.2.0", "@datadog/native-metrics": "^2.0.0", - "@datadog/pprof": "4.1.0", + "@datadog/pprof": "5.4.1", "@datadog/sketches-js": "^2.1.0", - "@opentelemetry/api": "^1.0.0", + "@opentelemetry/api": ">=1.0.0 <1.9.0", "@opentelemetry/core": "^1.14.0", "crypto-randomuuid": "^1.0.0", - "dc-polyfill": "^0.1.2", + "dc-polyfill": "^0.1.4", "ignore": "^5.2.4", - "import-in-the-middle": "^1.4.2", + "import-in-the-middle": "1.11.2", "int64-buffer": "^0.1.9", - "ipaddr.js": "^2.1.0", "istanbul-lib-coverage": "3.2.0", "jest-docblock": "^29.7.0", "koalas": "^1.0.2", - "limiter": "^1.1.4", - "lodash.kebabcase": "^4.1.1", - "lodash.pick": "^4.4.0", + "limiter": "1.1.5", "lodash.sortby": "^4.7.0", - "lodash.uniq": "^4.5.0", "lru-cache": "^7.14.0", - "methods": "^1.1.2", "module-details-from-path": "^1.0.3", "msgpack-lite": "^0.1.26", - "node-abort-controller": "^3.1.1", "opentracing": ">=0.12.1", - "path-to-regexp": "^0.1.2", - "pprof-format": "^2.0.7", - "protobufjs": "^7.2.4", + "path-to-regexp": "^0.1.10", + "pprof-format": "^2.1.0", + "protobufjs": "^7.2.5", "retry": "^0.13.1", - "semver": "^7.5.4" + "rfdc": "^1.3.1", + "semver": "^7.5.4", + "shell-quote": "^1.8.1", + "tlhunter-sorted-set": "^0.1.0" }, "devDependencies": { - "@types/node": ">=16", + "@apollo/server": "^4.11.0", + "@types/node": "^16.18.103", "autocannon": "^4.5.2", "aws-sdk": "^2.1446.0", - "axios": "^0.21.2", + "axios": "^1.7.4", "benchmark": "^2.1.4", - "body-parser": "^1.20.2", + "body-parser": "^1.20.3", "chai": "^4.3.7", "chalk": "^5.3.0", "checksum": "^1.0.0", "cli-table3": "^0.6.3", "dotenv": "16.3.1", "esbuild": "0.16.12", - "eslint": "^8.23.0", - "eslint-config-standard": "^11.0.0-beta.0", - "eslint-plugin-import": "^2.8.0", - "eslint-plugin-mocha": "^10.1.0", - "eslint-plugin-n": "^15.7.0", - "eslint-plugin-node": "^5.2.1", - "eslint-plugin-promise": "^3.6.0", - "eslint-plugin-standard": "^3.0.1", + "eslint": "^8.57.0", + "eslint-config-standard": "^17.1.0", + "eslint-plugin-import": "^2.29.1", + "eslint-plugin-mocha": "^10.4.3", + "eslint-plugin-n": "^16.6.2", + "eslint-plugin-promise": "^6.4.0", "express": "^4.18.2", "get-port": "^3.2.0", "glob": "^7.1.6", @@ -130,15 +134,15 @@ "jszip": "^3.5.0", "knex": "^2.4.2", "mkdirp": "^3.0.1", - "mocha": "8", + "mocha": "^9", "multer": "^1.4.5-lts.1", "nock": "^11.3.3", "nyc": "^15.1.0", "proxyquire": "^1.8.0", "rimraf": "^3.0.0", - "sinon": "^15.2.0", + "sinon": "^16.1.3", "sinon-chai": "^3.7.0", "tap": "^16.3.7", - "tape": "^5.6.5" + "tiktoken": "^1.0.15" } } diff --git a/packages/datadog-code-origin/index.js b/packages/datadog-code-origin/index.js new file mode 100644 index 00000000000..530dd3cc8ae --- /dev/null +++ b/packages/datadog-code-origin/index.js @@ -0,0 +1,38 @@ +'use strict' + +const { getUserLandFrames } = require('../dd-trace/src/plugins/util/stacktrace') + +const limit = Number(process.env._DD_CODE_ORIGIN_MAX_USER_FRAMES) || 8 + +module.exports = { + entryTag, + exitTag +} + +function entryTag (topOfStackFunc) { + return tag('entry', topOfStackFunc) +} + +function exitTag (topOfStackFunc) { + return tag('exit', topOfStackFunc) +} + +function tag (type, topOfStackFunc) { + const frames = getUserLandFrames(topOfStackFunc, limit) + const tags = { + '_dd.code_origin.type': type + } + for (let i = 0; i < frames.length; i++) { + const frame = frames[i] + tags[`_dd.code_origin.frames.${i}.file`] = frame.file + tags[`_dd.code_origin.frames.${i}.line`] = String(frame.line) + tags[`_dd.code_origin.frames.${i}.column`] = String(frame.column) + if (frame.method) { + tags[`_dd.code_origin.frames.${i}.method`] = frame.method + } + if (frame.type) { + tags[`_dd.code_origin.frames.${i}.type`] = frame.type + } + } + return tags +} diff --git a/packages/datadog-core/index.js b/packages/datadog-core/index.js index 72b0403aa75..9819b32f3ba 100644 --- a/packages/datadog-core/index.js +++ b/packages/datadog-core/index.js @@ -1,7 +1,7 @@ 'use strict' -const LocalStorage = require('./src/storage') +const { AsyncLocalStorage } = require('async_hooks') -const storage = new LocalStorage() +const storage = new AsyncLocalStorage() module.exports = { storage } diff --git a/packages/datadog-core/src/storage/async_hooks.js b/packages/datadog-core/src/storage/async_hooks.js deleted file mode 100644 index d8e71e1df2d..00000000000 --- a/packages/datadog-core/src/storage/async_hooks.js +++ /dev/null @@ -1,49 +0,0 @@ -'use strict' - -const { executionAsyncId } = require('async_hooks') -const AsyncResourceStorage = require('./async_resource') - -class AsyncHooksStorage extends AsyncResourceStorage { - constructor () { - super() - - this._resources = new Map() - } - - disable () { - super.disable() - - this._resources.clear() - } - - _createHook () { - return { - ...super._createHook(), - destroy: this._destroy.bind(this) - } - } - - _init (asyncId, type, triggerAsyncId, resource) { - super._init.apply(this, arguments) - - this._resources.set(asyncId, resource) - } - - _destroy (asyncId) { - this._resources.delete(asyncId) - } - - _executionAsyncResource () { - const asyncId = executionAsyncId() - - let resource = this._resources.get(asyncId) - - if (!resource) { - this._resources.set(asyncId, resource = {}) - } - - return resource - } -} - -module.exports = AsyncHooksStorage diff --git a/packages/datadog-core/src/storage/async_resource.js b/packages/datadog-core/src/storage/async_resource.js deleted file mode 100644 index 4738845e415..00000000000 --- a/packages/datadog-core/src/storage/async_resource.js +++ /dev/null @@ -1,108 +0,0 @@ -'use strict' - -const { createHook, executionAsyncResource } = require('async_hooks') -const { channel } = require('dc-polyfill') - -const beforeCh = channel('dd-trace:storage:before') -const afterCh = channel('dd-trace:storage:after') -const enterCh = channel('dd-trace:storage:enter') - -let PrivateSymbol = Symbol -function makePrivateSymbol () { - // eslint-disable-next-line no-new-func - PrivateSymbol = new Function('name', 'return %CreatePrivateSymbol(name)') -} - -try { - makePrivateSymbol() -} catch (e) { - try { - const v8 = require('v8') - v8.setFlagsFromString('--allow-natives-syntax') - makePrivateSymbol() - v8.setFlagsFromString('--no-allow-natives-syntax') - // eslint-disable-next-line no-empty - } catch (e) {} -} - -class AsyncResourceStorage { - constructor () { - this._ddResourceStore = PrivateSymbol('ddResourceStore') - this._enabled = false - this._hook = createHook(this._createHook()) - } - - disable () { - if (!this._enabled) return - - this._hook.disable() - this._enabled = false - } - - getStore () { - if (!this._enabled) return - - const resource = this._executionAsyncResource() - - return resource[this._ddResourceStore] - } - - enterWith (store) { - this._enable() - - const resource = this._executionAsyncResource() - - resource[this._ddResourceStore] = store - enterCh.publish() - } - - run (store, callback, ...args) { - this._enable() - - const resource = this._executionAsyncResource() - const oldStore = resource[this._ddResourceStore] - - resource[this._ddResourceStore] = store - enterCh.publish() - - try { - return callback(...args) - } finally { - resource[this._ddResourceStore] = oldStore - enterCh.publish() - } - } - - _createHook () { - return { - init: this._init.bind(this), - before () { - beforeCh.publish() - }, - after () { - afterCh.publish() - } - } - } - - _enable () { - if (this._enabled) return - - this._enabled = true - this._hook.enable() - } - - _init (asyncId, type, triggerAsyncId, resource) { - const currentResource = this._executionAsyncResource() - - if (Object.prototype.hasOwnProperty.call(currentResource, this._ddResourceStore)) { - resource[this._ddResourceStore] = currentResource[this._ddResourceStore] - } - } - - _executionAsyncResource () { - return executionAsyncResource() || {} - } -} - -module.exports = AsyncResourceStorage diff --git a/packages/datadog-core/src/storage/index.js b/packages/datadog-core/src/storage/index.js deleted file mode 100644 index 0d48defbc3c..00000000000 --- a/packages/datadog-core/src/storage/index.js +++ /dev/null @@ -1,14 +0,0 @@ -'use strict' - -// TODO: default to AsyncLocalStorage when it supports triggerAsyncResource - -const semver = require('semver') - -// https://github.com/nodejs/node/pull/33801 -const hasJavaScriptAsyncHooks = semver.satisfies(process.versions.node, '>=14.5') - -if (hasJavaScriptAsyncHooks) { - module.exports = require('./async_resource') -} else { - module.exports = require('./async_hooks') -} diff --git a/packages/datadog-core/src/utils/src/get.js b/packages/datadog-core/src/utils/src/get.js new file mode 100644 index 00000000000..f5913c3016c --- /dev/null +++ b/packages/datadog-core/src/utils/src/get.js @@ -0,0 +1,11 @@ +'use strict' + +module.exports = (object, path) => { + const pathArr = path.split('.') + let val = object + for (const p of pathArr) { + if (val === undefined) return val + val = val[p] + } + return val +} diff --git a/packages/datadog-core/src/utils/src/has.js b/packages/datadog-core/src/utils/src/has.js new file mode 100644 index 00000000000..fafc7cc775f --- /dev/null +++ b/packages/datadog-core/src/utils/src/has.js @@ -0,0 +1,14 @@ +'use strict' + +module.exports = (object, path) => { + const pathArr = path.split('.') + let property = object + for (const n of pathArr) { + if (property.hasOwnProperty(n)) { + property = property[n] + } else { + return false + } + } + return true +} diff --git a/packages/datadog-core/src/utils/src/kebabcase.js b/packages/datadog-core/src/utils/src/kebabcase.js new file mode 100644 index 00000000000..c55bb385013 --- /dev/null +++ b/packages/datadog-core/src/utils/src/kebabcase.js @@ -0,0 +1,16 @@ +'use strict' + +module.exports = str => { + if (typeof str !== 'string') { + throw new TypeError('Expected a string') + } + + return str + .trim() + .replace(/([a-z])([A-Z])/g, '$1-$2') + .replace(/\s+/g, '-') + .replace(/^-+|-+$/g, '') + .replace(/_/g, '-') + .replace(/-{2,}/g, '-') + .toLowerCase() +} diff --git a/packages/datadog-core/src/utils/src/pick.js b/packages/datadog-core/src/utils/src/pick.js new file mode 100644 index 00000000000..d0a101a1ef7 --- /dev/null +++ b/packages/datadog-core/src/utils/src/pick.js @@ -0,0 +1,11 @@ +'use strict' + +module.exports = (object, props) => { + const result = {} + props.forEach(prop => { + if (prop in object) { + result[prop] = object[prop] + } + }) + return result +} diff --git a/packages/datadog-core/src/utils/src/set.js b/packages/datadog-core/src/utils/src/set.js new file mode 100644 index 00000000000..e6b9fc7f12a --- /dev/null +++ b/packages/datadog-core/src/utils/src/set.js @@ -0,0 +1,16 @@ +'use strict' + +module.exports = (object, path, value) => { + const pathArr = path.split('.') + let property = object + let i + for (i = 0; i < pathArr.length - 1; i++) { + const n = pathArr[i] + if (property.hasOwnProperty(n)) { + property = property[n] + } else { + property[n] = property = {} + } + } + property[pathArr[i]] = value +} diff --git a/packages/datadog-core/src/utils/src/uniq.js b/packages/datadog-core/src/utils/src/uniq.js new file mode 100644 index 00000000000..a7a23be7013 --- /dev/null +++ b/packages/datadog-core/src/utils/src/uniq.js @@ -0,0 +1,5 @@ +'use strict' + +module.exports = function (arr) { + return [...new Set(arr)] +} diff --git a/packages/datadog-core/test/setup.js b/packages/datadog-core/test/setup.js deleted file mode 100644 index 2f8af45cdd2..00000000000 --- a/packages/datadog-core/test/setup.js +++ /dev/null @@ -1,8 +0,0 @@ -'use strict' - -require('tap').mochaGlobals() - -const chai = require('chai') -const sinonChai = require('sinon-chai') - -chai.use(sinonChai) diff --git a/packages/datadog-core/test/storage/async_hooks.spec.js b/packages/datadog-core/test/storage/async_hooks.spec.js deleted file mode 100644 index dc990ab94f4..00000000000 --- a/packages/datadog-core/test/storage/async_hooks.spec.js +++ /dev/null @@ -1,20 +0,0 @@ -'use strict' - -require('../setup') - -const StorageBackend = require('../../src/storage/async_hooks') -const testStorage = require('./test') - -describe('storage/async_hooks', () => { - let storage - - beforeEach(() => { - storage = new StorageBackend() - }) - - afterEach(() => { - storage.disable() - }) - - testStorage(() => storage) -}) diff --git a/packages/datadog-core/test/storage/async_resource.spec.js b/packages/datadog-core/test/storage/async_resource.spec.js deleted file mode 100644 index ce19b216260..00000000000 --- a/packages/datadog-core/test/storage/async_resource.spec.js +++ /dev/null @@ -1,20 +0,0 @@ -'use strict' - -require('../setup') - -const StorageBackend = require('../../src/storage/async_resource') -const testStorage = require('./test') - -describe('storage/async_resource', () => { - let storage - - beforeEach(() => { - storage = new StorageBackend() - }) - - afterEach(() => { - storage.disable() - }) - - testStorage(() => storage) -}) diff --git a/packages/datadog-core/test/storage/test.js b/packages/datadog-core/test/storage/test.js deleted file mode 100644 index 0f69a43d9f0..00000000000 --- a/packages/datadog-core/test/storage/test.js +++ /dev/null @@ -1,160 +0,0 @@ -'use strict' - -const { expect } = require('chai') -const { inspect } = require('util') -const { - AsyncResource, - executionAsyncId, - executionAsyncResource -} = require('async_hooks') - -module.exports = factory => { - let storage - let store - - beforeEach(() => { - storage = factory() - store = {} - }) - - describe('getStore()', () => { - it('should return undefined by default', () => { - expect(storage.getStore()).to.be.undefined - }) - }) - - describe('run()', () => { - it('should return the value returned by the callback', () => { - expect(storage.run(store, () => 'test')).to.equal('test') - }) - - it('should preserve the surrounding scope', () => { - expect(storage.getStore()).to.be.undefined - - storage.run(store, () => {}) - - expect(storage.getStore()).to.be.undefined - }) - - it('should run the span on the current scope', () => { - expect(storage.getStore()).to.be.undefined - - storage.run(store, () => { - expect(storage.getStore()).to.equal(store) - }) - - expect(storage.getStore()).to.be.undefined - }) - - it('should persist through setTimeout', done => { - storage.run(store, () => { - setTimeout(() => { - expect(storage.getStore()).to.equal(store) - done() - }, 0) - }) - }) - - it('should persist through setImmediate', done => { - storage.run(store, () => { - setImmediate(() => { - expect(storage.getStore()).to.equal(store) - done() - }, 0) - }) - }) - - it('should persist through setInterval', done => { - storage.run(store, () => { - let shouldReturn = false - - const timer = setInterval(() => { - expect(storage.getStore()).to.equal(store) - - if (shouldReturn) { - clearInterval(timer) - return done() - } - - shouldReturn = true - }, 0) - }) - }) - - it('should persist through process.nextTick', done => { - storage.run(store, () => { - process.nextTick(() => { - expect(storage.getStore()).to.equal(store) - done() - }, 0) - }) - }) - - it('should persist through promises', () => { - const promise = Promise.resolve() - - return storage.run(store, () => { - return promise.then(() => { - expect(storage.getStore()).to.equal(store) - }) - }) - }) - - it('should handle concurrency', done => { - storage.run(store, () => { - setImmediate(() => { - expect(storage.getStore()).to.equal(store) - done() - }) - }) - - storage.run(store, () => {}) - }) - - it('should not break propagation for nested resources', done => { - storage.run(store, () => { - const asyncResource = new AsyncResource( - 'TEST', { triggerAsyncId: executionAsyncId(), requireManualDestroy: false } - ) - - asyncResource.runInAsyncScope(() => {}) - - expect(storage.getStore()).to.equal(store) - - done() - }) - }) - - it('should not log ddResourceStore contents', done => { - function getKeys (output) { - return output.split('\n').slice(1, -1).map(line => { - return line.split(':').map(v => v.trim())[0] - }) - } - - setImmediate(() => { - const withoutStore = getKeys(inspect(executionAsyncResource(), { depth: 0 })) - storage.run(store, () => { - setImmediate(() => { - const withStore = getKeys(inspect(executionAsyncResource(), { depth: 0 })) - expect(withStore).to.deep.equal(withoutStore) - done() - }) - }) - }) - }) - }) - - describe('enterWith()', () => { - it('should transition into the context for the remainder of the current execution', () => { - const newStore = {} - - storage.run(store, () => { - storage.enterWith(newStore) - expect(storage.getStore()).to.equal(newStore) - }) - - expect(storage.getStore()).to.be.undefined - }) - }) -} diff --git a/packages/datadog-core/test/utils/src/get.spec.js b/packages/datadog-core/test/utils/src/get.spec.js new file mode 100644 index 00000000000..8878d16e95f --- /dev/null +++ b/packages/datadog-core/test/utils/src/get.spec.js @@ -0,0 +1,22 @@ +'use strict' + +require('../../../../dd-trace/test/setup/tap') + +const { expect } = require('chai') +const get = require('../../../src/utils/src/get') + +describe('get', () => { + const obj = { + a: { + b: 'c' + } + } + + it('should return value at path', () => { + expect(get(obj, 'a.b')).to.be.equal('c') + }) + + it('should return undefined if path does not exist', () => { + expect(get(obj, 'd')).to.be.undefined + }) +}) diff --git a/packages/datadog-core/test/utils/src/has.spec.js b/packages/datadog-core/test/utils/src/has.spec.js new file mode 100644 index 00000000000..58da7f3f874 --- /dev/null +++ b/packages/datadog-core/test/utils/src/has.spec.js @@ -0,0 +1,22 @@ +'use strict' + +require('../../../../dd-trace/test/setup/tap') + +const { expect } = require('chai') +const has = require('../../../src/utils/src/has') + +describe('has', () => { + const obj = { + a: { + b: 'c' + } + } + + it('should true if path exists', () => { + expect(has(obj, 'a.b')).to.be.true + }) + + it('should return false if path does not exist', () => { + expect(has(obj, 'd')).to.be.false + }) +}) diff --git a/packages/datadog-core/test/utils/src/set.spec.js b/packages/datadog-core/test/utils/src/set.spec.js new file mode 100644 index 00000000000..02f9c7c19d7 --- /dev/null +++ b/packages/datadog-core/test/utils/src/set.spec.js @@ -0,0 +1,15 @@ +'use strict' + +require('../../../../dd-trace/test/setup/tap') + +const { expect } = require('chai') +const set = require('../../../src/utils/src/set') + +describe('set', () => { + const obj = {} + + it('should set value at path', () => { + set(obj, 'a.b', 'c') + expect(obj.a.b).to.be.equal('c') + }) +}) diff --git a/packages/datadog-esbuild/index.js b/packages/datadog-esbuild/index.js index 84454213e7d..4a69cf32ebc 100644 --- a/packages/datadog-esbuild/index.js +++ b/packages/datadog-esbuild/index.js @@ -4,9 +4,14 @@ const instrumentations = require('../datadog-instrumentations/src/helpers/instrumentations.js') const hooks = require('../datadog-instrumentations/src/helpers/hooks.js') +const extractPackageAndModulePath = require('../datadog-instrumentations/src/utils/src/extract-package-and-module-path') for (const hook of Object.values(hooks)) { - hook() + if (typeof hook === 'object') { + hook.fn() + } else { + hook() + } } const modulesOfInterest = new Set() @@ -21,7 +26,6 @@ for (const instrumentation of Object.values(instrumentations)) { } } -const NM = 'node_modules/' const INSTRUMENTED = Object.keys(instrumentations) const RAW_BUILTINS = require('module').builtinModules const CHANNEL = 'dd-trace:bundler:load' @@ -75,7 +79,10 @@ module.exports.setup = function (build) { try { fullPathToModule = dotFriendlyResolve(args.path, args.resolveDir) } catch (err) { - console.warn(`MISSING: Unable to find "${args.path}". Is the package dead code?`) + if (DEBUG) { + console.warn(`Warning: Unable to find "${args.path}".` + + "Unless it's dead code this could cause a problem at runtime.") + } return } const extracted = extractPackageAndModulePath(fullPathToModule) @@ -89,11 +96,16 @@ module.exports.setup = function (build) { let pathToPackageJson try { - pathToPackageJson = require.resolve(`${extracted.pkg}/package.json`, { paths: [ args.resolveDir ] }) + // we can't use require.resolve('pkg/package.json') as ESM modules don't make the file available + pathToPackageJson = require.resolve(`${extracted.pkg}`, { paths: [args.resolveDir] }) + pathToPackageJson = extractPackageAndModulePath(pathToPackageJson).pkgJson } catch (err) { if (err.code === 'MODULE_NOT_FOUND') { if (!internal) { - console.warn(`MISSING: Unable to find "${extracted.pkg}/package.json". Is the package dead code?`) + if (DEBUG) { + console.warn(`Warning: Unable to find "${extracted.pkg}/package.json".` + + "Unless it's dead code this could cause a problem at runtime.") + } } return } else { @@ -101,7 +113,7 @@ module.exports.setup = function (build) { } } - const packageJson = require(pathToPackageJson) + const packageJson = JSON.parse(fs.readFileSync(pathToPackageJson).toString()) if (DEBUG) console.log(`RESOLVE: ${args.path}@${packageJson.version}`) @@ -173,35 +185,5 @@ function dotFriendlyResolve (path, directory) { path = '../' } - return require.resolve(path, { paths: [ directory ] }) -} - -/** - * For a given full path to a module, - * return the package name it belongs to and the local path to the module - * input: '/foo/node_modules/@co/stuff/foo/bar/baz.js' - * output: { pkg: '@co/stuff', path: 'foo/bar/baz.js' } - */ -function extractPackageAndModulePath (fullPath) { - const nm = fullPath.lastIndexOf(NM) - if (nm < 0) { - return { pkg: null, path: null } - } - - const subPath = fullPath.substring(nm + NM.length) - const firstSlash = subPath.indexOf('/') - - if (subPath[0] === '@') { - const secondSlash = subPath.substring(firstSlash + 1).indexOf('/') - - return { - pkg: subPath.substring(0, firstSlash + 1 + secondSlash), - path: subPath.substring(firstSlash + 1 + secondSlash + 1) - } - } - - return { - pkg: subPath.substring(0, firstSlash), - path: subPath.substring(firstSlash + 1) - } + return require.resolve(path, { paths: [directory] }) } diff --git a/packages/datadog-instrumentations/src/aerospike.js b/packages/datadog-instrumentations/src/aerospike.js index dade4f81895..724c518e050 100644 --- a/packages/datadog-instrumentations/src/aerospike.js +++ b/packages/datadog-instrumentations/src/aerospike.js @@ -43,5 +43,5 @@ addHook({ versions: ['^3.16.2', '4', '5'] }, commandFactory => { - return shimmer.wrap(commandFactory, wrapCreateCommand(commandFactory)) + return shimmer.wrapFunction(commandFactory, f => wrapCreateCommand(f)) }) diff --git a/packages/datadog-instrumentations/src/amqplib.js b/packages/datadog-instrumentations/src/amqplib.js index a5421ff7c93..f0650459a47 100644 --- a/packages/datadog-instrumentations/src/amqplib.js +++ b/packages/datadog-instrumentations/src/amqplib.js @@ -5,16 +5,19 @@ const { addHook, AsyncResource } = require('./helpers/instrument') -const kebabCase = require('lodash.kebabcase') +const kebabCase = require('../../datadog-core/src/utils/src/kebabcase') const shimmer = require('../../datadog-shimmer') +const { NODE_MAJOR, NODE_MINOR } = require('../../../version') +const MIN_VERSION = ((NODE_MAJOR > 22) || (NODE_MAJOR === 22 && NODE_MINOR >= 2)) ? '>=0.5.3' : '>=0.5.0' + const startCh = channel('apm:amqplib:command:start') const finishCh = channel('apm:amqplib:command:finish') const errorCh = channel('apm:amqplib:command:error') let methods = {} -addHook({ name: 'amqplib', file: 'lib/defs.js', versions: ['>=0.5'] }, defs => { +addHook({ name: 'amqplib', file: 'lib/defs.js', versions: [MIN_VERSION] }, defs => { methods = Object.keys(defs) .filter(key => Number.isInteger(defs[key])) .filter(key => isCamelCase(key)) @@ -22,13 +25,13 @@ addHook({ name: 'amqplib', file: 'lib/defs.js', versions: ['>=0.5'] }, defs => { return defs }) -addHook({ name: 'amqplib', file: 'lib/channel.js', versions: ['>=0.5'] }, channel => { +addHook({ name: 'amqplib', file: 'lib/channel.js', versions: [MIN_VERSION] }, channel => { shimmer.wrap(channel.Channel.prototype, 'sendImmediately', sendImmediately => function (method, fields) { return instrument(sendImmediately, this, arguments, methods[method], fields) }) shimmer.wrap(channel.Channel.prototype, 'sendMessage', sendMessage => function (fields) { - return instrument(sendMessage, this, arguments, 'basic.publish', fields) + return instrument(sendMessage, this, arguments, 'basic.publish', fields, arguments[2]) }) shimmer.wrap(channel.BaseChannel.prototype, 'dispatchMessage', dispatchMessage => function (fields, message) { diff --git a/packages/datadog-instrumentations/src/apollo-server-core.js b/packages/datadog-instrumentations/src/apollo-server-core.js new file mode 100644 index 00000000000..3f075ab6938 --- /dev/null +++ b/packages/datadog-instrumentations/src/apollo-server-core.js @@ -0,0 +1,40 @@ +'use strict' + +const { addHook } = require('./helpers/instrument') +const shimmer = require('../../datadog-shimmer') +const dc = require('dc-polyfill') + +const requestChannel = dc.tracingChannel('datadog:apollo-server-core:request') + +addHook({ name: 'apollo-server-core', file: 'dist/runHttpQuery.js', versions: ['>3.0.0'] }, runHttpQueryModule => { + const HttpQueryError = runHttpQueryModule.HttpQueryError + + shimmer.wrap(runHttpQueryModule, 'runHttpQuery', function wrapRunHttpQuery (originalRunHttpQuery) { + return async function runHttpQuery () { + if (!requestChannel.start.hasSubscribers) { + return originalRunHttpQuery.apply(this, arguments) + } + + const abortController = new AbortController() + const abortData = {} + + const runHttpQueryResult = requestChannel.tracePromise( + originalRunHttpQuery, + { abortController, abortData }, + this, + ...arguments) + + const abortPromise = new Promise((resolve, reject) => { + abortController.signal.addEventListener('abort', (event) => { + // runHttpQuery callbacks are writing the response on resolve/reject. + // We should return blocking data in the apollo-server-core HttpQueryError object + reject(new HttpQueryError(abortData.statusCode, abortData.message, true, abortData.headers)) + }, { once: true }) + }) + + return Promise.race([runHttpQueryResult, abortPromise]) + } + }) + + return runHttpQueryModule +}) diff --git a/packages/datadog-instrumentations/src/apollo-server.js b/packages/datadog-instrumentations/src/apollo-server.js new file mode 100644 index 00000000000..5a41903df91 --- /dev/null +++ b/packages/datadog-instrumentations/src/apollo-server.js @@ -0,0 +1,82 @@ +'use strict' + +const dc = require('dc-polyfill') + +const { addHook } = require('./helpers/instrument') +const shimmer = require('../../datadog-shimmer') + +const graphqlMiddlewareChannel = dc.tracingChannel('datadog:apollo:middleware') + +const requestChannel = dc.tracingChannel('datadog:apollo:request') + +let HeaderMap + +function wrapExecuteHTTPGraphQLRequest (originalExecuteHTTPGraphQLRequest) { + return async function executeHTTPGraphQLRequest () { + if (!HeaderMap || !requestChannel.start.hasSubscribers) { + return originalExecuteHTTPGraphQLRequest.apply(this, arguments) + } + + const abortController = new AbortController() + const abortData = {} + + const graphqlResponseData = requestChannel.tracePromise( + originalExecuteHTTPGraphQLRequest, + { abortController, abortData }, + this, + ...arguments) + + const abortPromise = new Promise((resolve, reject) => { + abortController.signal.addEventListener('abort', (event) => { + // This method is expected to return response data + // with headers, status and body + const headers = new HeaderMap() + Object.keys(abortData.headers).forEach(key => { + headers.set(key, abortData.headers[key]) + }) + + resolve({ + headers, + status: abortData.statusCode, + body: { + kind: 'complete', + string: abortData.message + } + }) + }, { once: true }) + }) + + return Promise.race([abortPromise, graphqlResponseData]) + } +} + +function apolloExpress4Hook (express4) { + shimmer.wrap(express4, 'expressMiddleware', function wrapExpressMiddleware (originalExpressMiddleware) { + return function expressMiddleware (server, options) { + const originalMiddleware = originalExpressMiddleware.apply(this, arguments) + + return shimmer.wrapFunction(originalMiddleware, originalMiddleware => function (req, res, next) { + if (!graphqlMiddlewareChannel.start.hasSubscribers) { + return originalMiddleware.apply(this, arguments) + } + + return graphqlMiddlewareChannel.traceSync(originalMiddleware, { req }, this, ...arguments) + }) + } + }) + return express4 +} + +function apolloHeaderMapHook (headerMap) { + HeaderMap = headerMap.HeaderMap + return headerMap +} + +function apolloServerHook (apolloServer) { + shimmer.wrap(apolloServer.ApolloServer.prototype, 'executeHTTPGraphQLRequest', wrapExecuteHTTPGraphQLRequest) + return apolloServer +} + +addHook({ name: '@apollo/server', file: 'dist/cjs/ApolloServer.js', versions: ['>=4.0.0'] }, apolloServerHook) +addHook({ name: '@apollo/server', file: 'dist/cjs/express4/index.js', versions: ['>=4.0.0'] }, apolloExpress4Hook) +addHook({ name: '@apollo/server', file: 'dist/cjs/utils/HeaderMap.js', versions: ['>=4.0.0'] }, apolloHeaderMapHook) diff --git a/packages/datadog-instrumentations/src/apollo.js b/packages/datadog-instrumentations/src/apollo.js new file mode 100644 index 00000000000..e75df6ab046 --- /dev/null +++ b/packages/datadog-instrumentations/src/apollo.js @@ -0,0 +1,103 @@ +const { + addHook, + channel +} = require('./helpers/instrument') +const shimmer = require('../../datadog-shimmer') +const tracingChannel = require('dc-polyfill').tracingChannel + +const CHANNELS = { + 'gateway.request': tracingChannel('apm:apollo:gateway:request'), + 'gateway.plan': tracingChannel('apm:apollo:gateway:plan'), + 'gateway.validate': tracingChannel('apm:apollo:gateway:validate'), + 'gateway.execute': tracingChannel('apm:apollo:gateway:execute'), + 'gateway.fetch': tracingChannel('apm:apollo:gateway:fetch'), + 'gateway.postprocessing': tracingChannel('apm:apollo:gateway:postprocessing') +} + +const generalErrorCh = channel('apm:apollo:gateway:general:error') + +function wrapExecutor (executor) { + return function (...args) { + const channel = CHANNELS['gateway.request'] + const ctx = { requestContext: args[0], gateway: this } + + return channel.tracePromise(executor, ctx, this, ...args) + } +} + +function wrapApolloGateway (ApolloGateway) { + class ApolloGatewayWrapper extends ApolloGateway { + constructor (...args) { + super(...args) + shimmer.wrap(this, 'executor', wrapExecutor) + } + } + return ApolloGatewayWrapper +} + +function wrapRecordExceptions (recordExceptions) { + return function wrappedRecordExceptions (...args) { + const errors = args[1] + // only the last exception in the array of exceptions will be reported on the span, + // this is mimicking apollo-gateways internal instrumentation + // TODO: should we consider a mechanism to report all exceptions? since this method aggregates all exceptions + // where as a span can only have one exception set on it at a time + generalErrorCh.publish({ error: errors[errors.length - 1] }) + return recordExceptions.apply(this, args) + } +} + +function wrapStartActiveSpan (startActiveSpan) { + return function (...args) { + const firstArg = args[0] + const cb = args[args.length - 1] + if (typeof firstArg !== 'string' || typeof cb !== 'function') return startActiveSpan.apply(this, args) + + const method = CHANNELS[firstArg] + let ctx = {} + if (firstArg === 'gateway.fetch') { + ctx = { attributes: args[1].attributes } + } + + switch (firstArg) { + case 'gateway.plan' : + case 'gateway.validate': { + args[args.length - 1] = function (...callbackArgs) { + return method.traceSync(cb, ctx, this, ...callbackArgs) + } + break + } + // Patch `executor` instead so the requestContext can be captured. + case 'gateway.request': + break + case 'gateway.execute': + case 'gateway.postprocessing' : + case 'gateway.fetch': { + args[args.length - 1] = function (...callbackArgs) { + return method.tracePromise(cb, ctx, this, ...callbackArgs) + } + break + } + } + return startActiveSpan.apply(this, args) + } +} + +addHook({ name: '@apollo/gateway', file: 'dist/utilities/opentelemetry.js', versions: ['>=2.3.0'] }, + (obj) => { + const newTracerObj = Object.create(obj.tracer) + shimmer.wrap(newTracerObj, 'startActiveSpan', wrapStartActiveSpan) + obj.tracer = newTracerObj + return obj + }) + +addHook({ name: '@apollo/gateway', file: 'dist/utilities/opentelemetry.js', versions: ['>=2.6.0'] }, + (obj) => { + shimmer.wrap(obj, 'recordExceptions', wrapRecordExceptions) + return obj + }) + +addHook({ name: '@apollo/gateway', versions: ['>=2.3.0'] }, (gateway) => { + shimmer.wrap(gateway, 'ApolloGateway', wrapApolloGateway) + return gateway +}) diff --git a/packages/datadog-instrumentations/src/avsc.js b/packages/datadog-instrumentations/src/avsc.js new file mode 100644 index 00000000000..6d71b1744bf --- /dev/null +++ b/packages/datadog-instrumentations/src/avsc.js @@ -0,0 +1,37 @@ +const shimmer = require('../../datadog-shimmer') +const { addHook } = require('./helpers/instrument') + +const dc = require('dc-polyfill') +const serializeChannel = dc.channel('apm:avsc:serialize-start') +const deserializeChannel = dc.channel('apm:avsc:deserialize-end') + +function wrapSerialization (Type) { + shimmer.wrap(Type.prototype, 'toBuffer', original => function () { + if (!serializeChannel.hasSubscribers) { + return original.apply(this, arguments) + } + serializeChannel.publish({ messageClass: this }) + return original.apply(this, arguments) + }) +} + +function wrapDeserialization (Type) { + shimmer.wrap(Type.prototype, 'fromBuffer', original => function () { + if (!deserializeChannel.hasSubscribers) { + return original.apply(this, arguments) + } + const result = original.apply(this, arguments) + deserializeChannel.publish({ messageClass: result }) + return result + }) +} + +addHook({ + name: 'avsc', + versions: ['>=5.0.0'] +}, avro => { + wrapDeserialization(avro.Type) + wrapSerialization(avro.Type) + + return avro +}) diff --git a/packages/datadog-instrumentations/src/aws-sdk.js b/packages/datadog-instrumentations/src/aws-sdk.js index 68a20d50301..4d9a21db132 100644 --- a/packages/datadog-instrumentations/src/aws-sdk.js +++ b/packages/datadog-instrumentations/src/aws-sdk.js @@ -20,7 +20,8 @@ function wrapRequest (send) { return innerAr.runInAsyncScope(() => { this.on('complete', innerAr.bind(response => { - channel(`apm:aws:request:complete:${channelSuffix}`).publish({ response }) + const cbExists = typeof cb === 'function' + channel(`apm:aws:request:complete:${channelSuffix}`).publish({ response, cbExists }) })) startCh.publish({ @@ -74,7 +75,7 @@ function wrapSmithySend (send) { }) if (typeof cb === 'function') { - args[args.length - 1] = function (err, result) { + args[args.length - 1] = shimmer.wrapFunction(cb, cb => function (err, result) { const message = getMessage(request, err, result) completeChannel.publish(message) @@ -88,7 +89,7 @@ function wrapSmithySend (send) { responseFinishChannel.publish(message.response.error) } }) - } + }) } else { // always a promise return send.call(this, command, ...args) .then( @@ -111,7 +112,8 @@ function wrapSmithySend (send) { } function wrapCb (cb, serviceName, request, ar) { - return function wrappedCb (err, response) { + // eslint-disable-next-line n/handle-callback-err + return shimmer.wrapFunction(cb, cb => function wrappedCb (err, response) { const obj = { request, response } return ar.runInAsyncScope(() => { channel(`apm:aws:response:start:${serviceName}`).publish(obj) @@ -139,7 +141,7 @@ function wrapCb (cb, serviceName, request, ar) { throw e } }) - } + }) } function getMessage (request, error, result) { @@ -161,9 +163,14 @@ function getChannelSuffix (name) { 'lambda', 'redshift', 's3', + 'sfn', 'sns', - 'sqs' - ].includes(name) ? name : 'default' + 'sqs', + 'states', + 'stepfunctions' + ].includes(name) + ? name + : 'default' } addHook({ name: '@smithy/smithy-client', versions: ['>=1.0.3'] }, smithy => { diff --git a/packages/datadog-instrumentations/src/azure-functions.js b/packages/datadog-instrumentations/src/azure-functions.js new file mode 100644 index 00000000000..2527d9afb3f --- /dev/null +++ b/packages/datadog-instrumentations/src/azure-functions.js @@ -0,0 +1,48 @@ +'use strict' + +const { + addHook +} = require('./helpers/instrument') +const shimmer = require('../../datadog-shimmer') +const dc = require('dc-polyfill') + +const azureFunctionsChannel = dc.tracingChannel('datadog:azure-functions:invoke') + +addHook({ name: '@azure/functions', versions: ['>=4'] }, azureFunction => { + const { app } = azureFunction + + shimmer.wrap(app, 'deleteRequest', wrapHandler) + shimmer.wrap(app, 'http', wrapHandler) + shimmer.wrap(app, 'get', wrapHandler) + shimmer.wrap(app, 'patch', wrapHandler) + shimmer.wrap(app, 'post', wrapHandler) + shimmer.wrap(app, 'put', wrapHandler) + + return azureFunction +}) + +// The http methods are overloaded so we need to check which type of argument was passed in order to wrap the handler +// The arguments are either an object with a handler property or the handler function itself +function wrapHandler (method) { + return function (name, arg) { + if (typeof arg === 'object' && arg.hasOwnProperty('handler')) { + const options = arg + shimmer.wrap(options, 'handler', handler => traceHandler(handler, name, method.name)) + } else if (typeof arg === 'function') { + const handler = arg + arguments[1] = shimmer.wrapFunction(handler, handler => traceHandler(handler, name, method.name)) + } + return method.apply(this, arguments) + } +} + +function traceHandler (handler, functionName, methodName) { + return function (...args) { + const httpRequest = args[0] + const invocationContext = args[1] + return azureFunctionsChannel.tracePromise( + handler, + { functionName, httpRequest, invocationContext, methodName }, + this, ...args) + } +} diff --git a/packages/datadog-instrumentations/src/body-parser.js b/packages/datadog-instrumentations/src/body-parser.js index a73c377ba9a..ab51accb44b 100644 --- a/packages/datadog-instrumentations/src/body-parser.js +++ b/packages/datadog-instrumentations/src/body-parser.js @@ -1,13 +1,12 @@ 'use strict' -const { AbortController } = require('node-abort-controller') // AbortController is not available in node <15 const shimmer = require('../../datadog-shimmer') -const { channel, addHook } = require('./helpers/instrument') +const { channel, addHook, AsyncResource } = require('./helpers/instrument') const bodyParserReadCh = channel('datadog:body-parser:read:finish') function publishRequestBodyAndNext (req, res, next) { - return function () { + return shimmer.wrapFunction(next, next => function () { if (bodyParserReadCh.hasSubscribers && req) { const abortController = new AbortController() const body = req.body @@ -18,15 +17,27 @@ function publishRequestBodyAndNext (req, res, next) { } return next.apply(this, arguments) - } + }) } addHook({ name: 'body-parser', file: 'lib/read.js', - versions: ['>=1.4.0'] + versions: ['>=1.4.0 <1.20.0'] +}, read => { + return shimmer.wrapFunction(read, read => function (req, res, next) { + const nextResource = new AsyncResource('bound-anonymous-fn') + arguments[2] = nextResource.bind(publishRequestBodyAndNext(req, res, next)) + return read.apply(this, arguments) + }) +}) + +addHook({ + name: 'body-parser', + file: 'lib/read.js', + versions: ['>=1.20.0'] }, read => { - return shimmer.wrap(read, function (req, res, next) { + return shimmer.wrapFunction(read, read => function (req, res, next) { arguments[2] = publishRequestBodyAndNext(req, res, next) return read.apply(this, arguments) }) diff --git a/packages/datadog-instrumentations/src/cassandra-driver.js b/packages/datadog-instrumentations/src/cassandra-driver.js index 8482ed33870..a4684a01260 100644 --- a/packages/datadog-instrumentations/src/cassandra-driver.js +++ b/packages/datadog-instrumentations/src/cassandra-driver.js @@ -10,7 +10,7 @@ const shimmer = require('../../datadog-shimmer') const startCh = channel('apm:cassandra-driver:query:start') const finishCh = channel('apm:cassandra-driver:query:finish') const errorCh = channel('apm:cassandra-driver:query:error') -const connectCh = channel(`apm:cassandra-driver:query:connect`) +const connectCh = channel('apm:cassandra-driver:query:connect') addHook({ name: 'cassandra-driver', versions: ['>=3.0.0'] }, cassandra => { shimmer.wrap(cassandra.Client.prototype, 'batch', batch => function (queries, options, callback) { @@ -180,12 +180,12 @@ function finish (finishCh, errorCh, error) { } function wrapCallback (finishCh, errorCh, asyncResource, callback) { - return asyncResource.bind(function (err) { + return shimmer.wrapFunction(callback, callback => asyncResource.bind(function (err) { finish(finishCh, errorCh, err) if (callback) { return callback.apply(this, arguments) } - }) + })) } function isRequestValid (exec, args, length) { diff --git a/packages/datadog-instrumentations/src/check_require_cache.js b/packages/datadog-instrumentations/src/check_require_cache.js new file mode 100644 index 00000000000..782cb56e5b2 --- /dev/null +++ b/packages/datadog-instrumentations/src/check_require_cache.js @@ -0,0 +1,103 @@ +'use strict' + +// This code runs before the tracer is configured and before a logger is ready +// For that reason we queue up the messages now and decide what to do with them later +const warnings = [] + +/** + * Here we maintain a list of packages that an application + * may have installed which could potentially conflict with + */ +const potentialConflicts = new Set([ + '@appsignal/javascript', + '@appsignal/nodejs', + '@dynatrace/oneagent', + '@instana/aws-fargate', + '@instana/aws-lambda', + '@instana/azure-container-services', + '@instana/collector', + '@instana/google-cloud-run', + '@sentry/node', + 'appoptics-apm', + 'atatus-nodejs', + 'elastic-apm-node', + 'newrelic', + 'stackify-node-apm', + 'sqreen' +]) + +const extractPackageAndModulePath = require('./utils/src/extract-package-and-module-path') + +/** + * The lowest hanging fruit to debug an app that isn't tracing + * properly is to check that it is loaded before any modules + * that need to be instrumented. This function checks the + * `require.cache` to see if any supported packages have + * already been required and prints a warning. + * + * Note that this only going to work for modules within npm + * packages, like `express`, and not internal modules, like + * `http`. It also only works with CJS, not with ESM imports. + * + * The output isn't necessarily 100% perfect. For example if the + * app loads a package we instrument but outside of an + * unsupported version then a warning would still be displayed. + * This is OK as the tracer should be loaded earlier anyway. + */ +module.exports.checkForRequiredModules = function () { + const packages = require('../../datadog-instrumentations/src/helpers/hooks') + const naughties = new Set() + let didWarn = false + + for (const pathToModule of Object.keys(require.cache)) { + const { pkg } = extractPackageAndModulePath(pathToModule) + + if (naughties.has(pkg)) continue + if (!(pkg in packages)) continue + + warnings.push(`Warning: Package '${pkg}' was loaded before dd-trace! This may break instrumentation.`) + + naughties.add(pkg) + didWarn = true + } + + if (didWarn) warnings.push('Warning: Please ensure dd-trace is loaded before other modules.') +} + +/** + * APM tools, and some other packages in the community, work + * by monkey-patching internal modules and possibly some + * globals. Usually this is done in a conflict-free way by + * wrapping an existing method with a new method that still + * calls the original method. Unfortunately it's possible + * that some of these packages (dd-trace included) may + * wrap methods in a way that make it unsafe for the methods + * to be wrapped again by another library. + * + * When encountered, and when debug mode is on, a warning is + * printed if such a package is discovered. This can help + * when debugging a faulty installation. + */ +module.exports.checkForPotentialConflicts = function () { + const naughties = new Set() + let didWarn = false + + for (const pathToModule of Object.keys(require.cache)) { + const { pkg } = extractPackageAndModulePath(pathToModule) + if (naughties.has(pkg)) continue + if (!potentialConflicts.has(pkg)) continue + + warnings.push(`Warning: Package '${pkg}' may cause conflicts with dd-trace.`) + + naughties.add(pkg) + didWarn = true + } + + if (didWarn) warnings.push('Warning: Packages were loaded that may conflict with dd-trace.') +} + +module.exports.flushStartupLogs = function (log) { + while (warnings.length) { + log.warn(warnings.shift()) + } +} diff --git a/packages/datadog-instrumentations/src/child-process.js b/packages/datadog-instrumentations/src/child-process.js deleted file mode 100644 index ba26dfdf7cf..00000000000 --- a/packages/datadog-instrumentations/src/child-process.js +++ /dev/null @@ -1,30 +0,0 @@ -'use strict' - -const { - channel, - addHook -} = require('./helpers/instrument') -const shimmer = require('../../datadog-shimmer') - -const childProcessChannel = channel('datadog:child_process:execution:start') -const execMethods = ['exec', 'execFile', 'fork', 'spawn', 'execFileSync', 'execSync', 'spawnSync'] -const names = ['child_process', 'node:child_process'] -names.forEach(name => { - addHook({ name }, childProcess => { - shimmer.massWrap(childProcess, execMethods, wrapChildProcessMethod()) - return childProcess - }) -}) - -function wrapChildProcessMethod () { - function wrapMethod (childProcessMethod) { - return function () { - if (childProcessChannel.hasSubscribers && arguments.length > 0) { - const command = arguments[0] - childProcessChannel.publish({ command }) - } - return childProcessMethod.apply(this, arguments) - } - } - return wrapMethod -} diff --git a/packages/datadog-instrumentations/src/child_process.js b/packages/datadog-instrumentations/src/child_process.js new file mode 100644 index 00000000000..8af49788007 --- /dev/null +++ b/packages/datadog-instrumentations/src/child_process.js @@ -0,0 +1,159 @@ +'use strict' + +const util = require('util') + +const { + addHook, + AsyncResource +} = require('./helpers/instrument') +const shimmer = require('../../datadog-shimmer') +const dc = require('dc-polyfill') + +const childProcessChannel = dc.tracingChannel('datadog:child_process:execution') + +// ignored exec method because it calls to execFile directly +const execAsyncMethods = ['execFile', 'spawn'] +const execSyncMethods = ['execFileSync', 'spawnSync'] + +const names = ['child_process', 'node:child_process'] + +// child_process and node:child_process returns the same object instance, we only want to add hooks once +let patched = false +names.forEach(name => { + addHook({ name }, childProcess => { + if (!patched) { + patched = true + shimmer.massWrap(childProcess, execAsyncMethods, wrapChildProcessAsyncMethod()) + shimmer.massWrap(childProcess, execSyncMethods, wrapChildProcessSyncMethod()) + shimmer.wrap(childProcess, 'execSync', wrapChildProcessSyncMethod(true)) + } + + return childProcess + }) +}) + +function normalizeArgs (args, shell) { + const childProcessInfo = { + command: args[0] + } + + if (Array.isArray(args[1])) { + childProcessInfo.command = childProcessInfo.command + ' ' + args[1].join(' ') + if (args[2] !== null && typeof args[2] === 'object') { + childProcessInfo.options = args[2] + } + } else if (args[1] !== null && typeof args[1] === 'object') { + childProcessInfo.options = args[1] + } + childProcessInfo.shell = shell || + childProcessInfo.options?.shell === true || + typeof childProcessInfo.options?.shell === 'string' + + return childProcessInfo +} + +function wrapChildProcessSyncMethod (shell = false) { + return function wrapMethod (childProcessMethod) { + return function () { + if (!childProcessChannel.start.hasSubscribers || arguments.length === 0) { + return childProcessMethod.apply(this, arguments) + } + + const childProcessInfo = normalizeArgs(arguments, shell) + + const innerResource = new AsyncResource('bound-anonymous-fn') + return innerResource.runInAsyncScope(() => { + return childProcessChannel.traceSync( + childProcessMethod, + { + command: childProcessInfo.command, + shell: childProcessInfo.shell + }, + this, + ...arguments) + }) + } + } +} + +function wrapChildProcessCustomPromisifyMethod (customPromisifyMethod, shell) { + return function () { + if (!childProcessChannel.start.hasSubscribers || arguments.length === 0) { + return customPromisifyMethod.apply(this, arguments) + } + + const childProcessInfo = normalizeArgs(arguments, shell) + + return childProcessChannel.tracePromise( + customPromisifyMethod, + { + command: childProcessInfo.command, + shell: childProcessInfo.shell + }, + this, + ...arguments) + } +} + +function wrapChildProcessAsyncMethod (shell = false) { + return function wrapMethod (childProcessMethod) { + function wrappedChildProcessMethod () { + if (!childProcessChannel.start.hasSubscribers || arguments.length === 0) { + return childProcessMethod.apply(this, arguments) + } + + const childProcessInfo = normalizeArgs(arguments, shell) + + const cb = arguments[arguments.length - 1] + if (typeof cb === 'function') { + const callbackResource = new AsyncResource('bound-anonymous-fn') + arguments[arguments.length - 1] = callbackResource.bind(cb) + } + + const innerResource = new AsyncResource('bound-anonymous-fn') + return innerResource.runInAsyncScope(() => { + childProcessChannel.start.publish({ command: childProcessInfo.command, shell: childProcessInfo.shell }) + + const childProcess = childProcessMethod.apply(this, arguments) + if (childProcess) { + let errorExecuted = false + + childProcess.on('error', (e) => { + errorExecuted = true + childProcessChannel.error.publish(e) + }) + + childProcess.on('close', (code) => { + code = code || 0 + if (!errorExecuted && code !== 0) { + childProcessChannel.error.publish() + } + childProcessChannel.asyncEnd.publish({ + command: childProcessInfo.command, + shell: childProcessInfo.shell, + result: code + }) + }) + } + + return childProcess + }) + } + + if (childProcessMethod[util.promisify.custom]) { + const wrapedChildProcessCustomPromisifyMethod = + shimmer.wrapFunction(childProcessMethod[util.promisify.custom], + promisify => wrapChildProcessCustomPromisifyMethod(promisify, shell)) + + // should do it in this way because the original property is readonly + const descriptor = Object.getOwnPropertyDescriptor(childProcessMethod, util.promisify.custom) + Object.defineProperty(wrappedChildProcessMethod, + util.promisify.custom, + { + ...descriptor, + value: wrapedChildProcessCustomPromisifyMethod + }) + } + return wrappedChildProcessMethod + } +} diff --git a/packages/datadog-instrumentations/src/connect.js b/packages/datadog-instrumentations/src/connect.js index fe474a3d89b..507811f6dd3 100644 --- a/packages/datadog-instrumentations/src/connect.js +++ b/packages/datadog-instrumentations/src/connect.js @@ -59,7 +59,7 @@ function wrapLayerHandle (layer) { const original = layer.handle - return shimmer.wrap(original, function () { + return shimmer.wrapFunction(original, original => function () { if (!enterChannel.hasSubscribers) return original.apply(this, arguments) const lastIndex = arguments.length - 1 @@ -90,7 +90,7 @@ function wrapLayerHandle (layer) { } function wrapNext (req, next) { - return function (error) { + return shimmer.wrapFunction(next, next => function (error) { if (error) { errorChannel.publish({ req, error }) } @@ -99,11 +99,11 @@ function wrapNext (req, next) { finishChannel.publish({ req }) next.apply(this, arguments) - } + }) } addHook({ name: 'connect', versions: ['>=3'] }, connect => { - return shimmer.wrap(connect, wrapConnect(connect)) + return shimmer.wrapFunction(connect, connect => wrapConnect(connect)) }) addHook({ name: 'connect', versions: ['2.2.2'] }, connect => { diff --git a/packages/datadog-instrumentations/src/cookie-parser.js b/packages/datadog-instrumentations/src/cookie-parser.js index 94a30818e23..09b3e18b71f 100644 --- a/packages/datadog-instrumentations/src/cookie-parser.js +++ b/packages/datadog-instrumentations/src/cookie-parser.js @@ -1,13 +1,12 @@ 'use strict' -const { AbortController } = require('node-abort-controller') // AbortController is not available in node <15 const shimmer = require('../../datadog-shimmer') const { channel, addHook } = require('./helpers/instrument') const cookieParserReadCh = channel('datadog:cookie-parser:read:finish') function publishRequestCookieAndNext (req, res, next) { - return function cookieParserWrapper () { + return shimmer.wrapFunction(next, next => function cookieParserWrapper () { if (cookieParserReadCh.hasSubscribers && req) { const abortController = new AbortController() @@ -19,17 +18,17 @@ function publishRequestCookieAndNext (req, res, next) { } return next.apply(this, arguments) - } + }) } addHook({ name: 'cookie-parser', versions: ['>=1.0.0'] }, cookieParser => { - return shimmer.wrap(cookieParser, function () { + return shimmer.wrapFunction(cookieParser, cookieParser => function () { const cookieMiddleware = cookieParser.apply(this, arguments) - return shimmer.wrap(cookieMiddleware, function (req, res, next) { + return shimmer.wrapFunction(cookieMiddleware, cookieMiddleware => function (req, res, next) { arguments[2] = publishRequestCookieAndNext(req, res, next) return cookieMiddleware.apply(this, arguments) }) diff --git a/packages/datadog-instrumentations/src/couchbase.js b/packages/datadog-instrumentations/src/couchbase.js index 386a4b676ed..2cc30836738 100644 --- a/packages/datadog-instrumentations/src/couchbase.js +++ b/packages/datadog-instrumentations/src/couchbase.js @@ -37,7 +37,7 @@ function wrapMaybeInvoke (_maybeInvoke) { return _maybeInvoke.apply(this, arguments) } - return shimmer.wrap(_maybeInvoke, wrapped) + return wrapped } function wrapQuery (query) { @@ -51,7 +51,7 @@ function wrapQuery (query) { const res = query.apply(this, arguments) return res } - return shimmer.wrap(query, wrapped) + return wrapped } function wrap (prefix, fn) { @@ -76,13 +76,13 @@ function wrap (prefix, fn) { startCh.publish({ bucket: { name: this.name || this._name }, seedNodes: this._dd_hosts }) - arguments[callbackIndex] = asyncResource.bind(function (error, result) { + arguments[callbackIndex] = shimmer.wrapFunction(cb, cb => asyncResource.bind(function (error, result) { if (error) { errorCh.publish(error) } finishCh.publish(result) return cb.apply(this, arguments) - }) + })) try { return fn.apply(this, arguments) @@ -94,7 +94,7 @@ function wrap (prefix, fn) { } }) } - return shimmer.wrap(fn, wrapped) + return wrapped } // semver >=3 @@ -118,13 +118,13 @@ function wrapCBandPromise (fn, name, startData, thisArg, args) { // v3 offers callback or promises event handling // NOTE: this does not work with v3.2.0-3.2.1 cluster.query, as there is a bug in the couchbase source code const cb = callbackResource.bind(args[cbIndex]) - args[cbIndex] = asyncResource.bind(function (error, result) { + args[cbIndex] = shimmer.wrapFunction(cb, cb => asyncResource.bind(function (error, result) { if (error) { errorCh.publish(error) } finishCh.publish({ result }) return cb.apply(thisArg, arguments) - }) + })) } const res = fn.apply(thisArg, args) @@ -166,8 +166,8 @@ addHook({ name: 'couchbase', file: 'lib/bucket.js', versions: ['^2.6.12'] }, Buc const finishCh = channel('apm:couchbase:query:finish') const errorCh = channel('apm:couchbase:query:error') - Bucket.prototype._maybeInvoke = wrapMaybeInvoke(Bucket.prototype._maybeInvoke) - Bucket.prototype.query = wrapQuery(Bucket.prototype.query) + shimmer.wrap(Bucket.prototype, '_maybeInvoke', maybeInvoke => wrapMaybeInvoke(maybeInvoke)) + shimmer.wrap(Bucket.prototype, 'query', query => wrapQuery(query)) shimmer.wrap(Bucket.prototype, '_n1qlReq', _n1qlReq => function (host, q, adhoc, emitter) { if (!startCh.hasSubscribers) { @@ -203,15 +203,15 @@ addHook({ name: 'couchbase', file: 'lib/bucket.js', versions: ['^2.6.12'] }, Buc }) wrapAllNames(['upsert', 'insert', 'replace', 'append', 'prepend'], name => { - Bucket.prototype[name] = wrap(`apm:couchbase:${name}`, Bucket.prototype[name]) + shimmer.wrap(Bucket.prototype, name, fn => wrap(`apm:couchbase:${name}`, fn)) }) return Bucket }) addHook({ name: 'couchbase', file: 'lib/cluster.js', versions: ['^2.6.12'] }, Cluster => { - Cluster.prototype._maybeInvoke = wrapMaybeInvoke(Cluster.prototype._maybeInvoke) - Cluster.prototype.query = wrapQuery(Cluster.prototype.query) + shimmer.wrap(Cluster.prototype, '_maybeInvoke', maybeInvoke => wrapMaybeInvoke(maybeInvoke)) + shimmer.wrap(Cluster.prototype, 'query', query => wrapQuery(query)) shimmer.wrap(Cluster.prototype, 'openBucket', openBucket => { return function () { @@ -252,9 +252,10 @@ addHook({ name: 'couchbase', file: 'lib/cluster.js', versions: ['^3.0.7', '^3.1. return Cluster }) -// semver >=3.2.0 +// semver >=3.2.2 +// NOTE: <3.2.2 segfaults on cluster.close() https://issues.couchbase.com/browse/JSCBC-936 -addHook({ name: 'couchbase', file: 'dist/collection.js', versions: ['>=3.2.0'] }, collection => { +addHook({ name: 'couchbase', file: 'dist/collection.js', versions: ['>=3.2.2'] }, collection => { const Collection = collection.Collection wrapAllNames(['upsert', 'insert', 'replace'], name => { @@ -264,7 +265,7 @@ addHook({ name: 'couchbase', file: 'dist/collection.js', versions: ['>=3.2.0'] } return collection }) -addHook({ name: 'couchbase', file: 'dist/bucket.js', versions: ['>=3.2.0'] }, bucket => { +addHook({ name: 'couchbase', file: 'dist/bucket.js', versions: ['>=3.2.2'] }, bucket => { const Bucket = bucket.Bucket shimmer.wrap(Bucket.prototype, 'collection', getCollection => { return function () { @@ -278,7 +279,7 @@ addHook({ name: 'couchbase', file: 'dist/bucket.js', versions: ['>=3.2.0'] }, bu return bucket }) -addHook({ name: 'couchbase', file: 'dist/cluster.js', versions: ['3.2.0 - 3.2.1', '>=3.2.2'] }, (cluster) => { +addHook({ name: 'couchbase', file: 'dist/cluster.js', versions: ['>=3.2.2'] }, (cluster) => { const Cluster = cluster.Cluster shimmer.wrap(Cluster.prototype, 'query', wrapV3Query) diff --git a/packages/datadog-instrumentations/src/crypto.js b/packages/datadog-instrumentations/src/crypto.js index 3113c16ef1d..7c95614cee7 100644 --- a/packages/datadog-instrumentations/src/crypto.js +++ b/packages/datadog-instrumentations/src/crypto.js @@ -11,8 +11,9 @@ const cryptoCipherCh = channel('datadog:crypto:cipher:start') const hashMethods = ['createHash', 'createHmac', 'createSign', 'createVerify', 'sign', 'verify'] const cipherMethods = ['createCipheriv', 'createDecipheriv'] +const names = ['crypto', 'node:crypto'] -addHook({ name: 'crypto' }, crypto => { +addHook({ name: names }, crypto => { shimmer.massWrap(crypto, hashMethods, wrapCryptoMethod(cryptoHashCh)) shimmer.massWrap(crypto, cipherMethods, wrapCryptoMethod(cryptoCipherCh)) return crypto diff --git a/packages/datadog-instrumentations/src/cucumber.js b/packages/datadog-instrumentations/src/cucumber.js index b8285cfcda6..0f84d717381 100644 --- a/packages/datadog-instrumentations/src/cucumber.js +++ b/packages/datadog-instrumentations/src/cucumber.js @@ -6,6 +6,7 @@ const shimmer = require('../../datadog-shimmer') const log = require('../../dd-trace/src/log') const testStartCh = channel('ci:cucumber:test:start') +const testRetryCh = channel('ci:cucumber:test:retry') const testFinishCh = channel('ci:cucumber:test:finish') // used for test steps too const testStepStartCh = channel('ci:cucumber:test-step:start') @@ -16,19 +17,26 @@ const testSuiteStartCh = channel('ci:cucumber:test-suite:start') const testSuiteFinishCh = channel('ci:cucumber:test-suite:finish') const testSuiteCodeCoverageCh = channel('ci:cucumber:test-suite:code-coverage') -const itrConfigurationCh = channel('ci:cucumber:itr-configuration') +const libraryConfigurationCh = channel('ci:cucumber:library-configuration') +const knownTestsCh = channel('ci:cucumber:known-tests') const skippableSuitesCh = channel('ci:cucumber:test-suite:skippable') const sessionStartCh = channel('ci:cucumber:session:start') const sessionFinishCh = channel('ci:cucumber:session:finish') +const workerReportTraceCh = channel('ci:cucumber:worker-report:trace') + const itrSkippedSuitesCh = channel('ci:cucumber:itr:skipped-suites') +const getCodeCoverageCh = channel('ci:nyc:get-coverage') + const { getCoveredFilenamesFromCoverage, resetCoverage, mergeCoverage, fromCoverageMapToCoverage, - getTestSuitePath + getTestSuitePath, + CUCUMBER_WORKER_TRACE_PAYLOAD_CODE, + getIsFaultyEarlyFlakeDetection } = require('../../dd-trace/src/plugins/util/test') const isMarkedAsUnskippable = (pickle) => { @@ -41,11 +49,31 @@ const originalCoverageMap = createCoverageMap() // TODO: remove in a later major version const patched = new WeakSet() +const lastStatusByPickleId = new Map() +const numRetriesByPickleId = new Map() +const numAttemptToAsyncResource = new Map() +const newTestsByTestFullname = new Map() + +let eventDataCollector = null let pickleByFile = {} const pickleResultByFile = {} + +const sessionAsyncResource = new AsyncResource('bound-anonymous-fn') + let skippableSuites = [] +let itrCorrelationId = '' let isForcedToRun = false let isUnskippable = false +let isSuitesSkippingEnabled = false +let isEarlyFlakeDetectionEnabled = false +let earlyFlakeDetectionNumRetries = 0 +let earlyFlakeDetectionFaultyThreshold = 0 +let isEarlyFlakeDetectionFaulty = false +let isFlakyTestRetriesEnabled = false +let numTestRetries = 0 +let knownTests = [] +let skippedSuites = [] +let isSuitesSkipped = false function getSuiteStatusFromTestStatuses (testStatuses) { if (testStatuses.some(status => status === 'fail')) { @@ -83,6 +111,90 @@ function getStatusFromResultLatest (result) { return { status: 'fail', errorMessage: result.message } } +function isNewTest (testSuite, testName) { + const testsForSuite = knownTests.cucumber?.[testSuite] || [] + return !testsForSuite.includes(testName) +} + +function getTestStatusFromRetries (testStatuses) { + if (testStatuses.every(status => status === 'fail')) { + return 'fail' + } + if (testStatuses.some(status => status === 'pass')) { + return 'pass' + } + return 'pass' +} + +function getChannelPromise (channelToPublishTo) { + return new Promise(resolve => { + sessionAsyncResource.runInAsyncScope(() => { + channelToPublishTo.publish({ onDone: resolve }) + }) + }) +} + +function getShouldBeSkippedSuite (pickle, suitesToSkip) { + const testSuitePath = getTestSuitePath(pickle.uri, process.cwd()) + const isUnskippable = isMarkedAsUnskippable(pickle) + const isSkipped = suitesToSkip.includes(testSuitePath) + + return [isSkipped && !isUnskippable, testSuitePath] +} + +// From cucumber@>=11 +function getFilteredPicklesNew (coordinator, suitesToSkip) { + return coordinator.sourcedPickles.reduce((acc, sourcedPickle) => { + const { pickle } = sourcedPickle + const [shouldBeSkipped, testSuitePath] = getShouldBeSkippedSuite(pickle, suitesToSkip) + + if (shouldBeSkipped) { + acc.skippedSuites.add(testSuitePath) + } else { + acc.picklesToRun.push(sourcedPickle) + } + return acc + }, { skippedSuites: new Set(), picklesToRun: [] }) +} + +function getFilteredPickles (runtime, suitesToSkip) { + return runtime.pickleIds.reduce((acc, pickleId) => { + const pickle = runtime.eventDataCollector.getPickle(pickleId) + const [shouldBeSkipped, testSuitePath] = getShouldBeSkippedSuite(pickle, suitesToSkip) + + if (shouldBeSkipped) { + acc.skippedSuites.add(testSuitePath) + } else { + acc.picklesToRun.push(pickleId) + } + return acc + }, { skippedSuites: new Set(), picklesToRun: [] }) +} + +// From cucumber@>=11 +function getPickleByFileNew (coordinator) { + return coordinator.sourcedPickles.reduce((acc, { pickle }) => { + if (acc[pickle.uri]) { + acc[pickle.uri].push(pickle) + } else { + acc[pickle.uri] = [pickle] + } + return acc + }, {}) +} + +function getPickleByFile (runtimeOrCoodinator) { + return runtimeOrCoodinator.pickleIds.reduce((acc, pickleId) => { + const test = runtimeOrCoodinator.eventDataCollector.getPickle(pickleId) + if (acc[test.uri]) { + acc[test.uri].push(test) + } else { + acc[test.uri] = [test] + } + return acc + }, {}) +} + function wrapRun (pl, isLatestVersion) { if (patched.has(pl)) return @@ -93,66 +205,80 @@ function wrapRun (pl, isLatestVersion) { return run.apply(this, arguments) } - const asyncResource = new AsyncResource('bound-anonymous-fn') - return asyncResource.runInAsyncScope(() => { - const testSuiteFullPath = this.pickle.uri + let numAttempt = 0 - if (!pickleResultByFile[testSuiteFullPath]) { // first test in suite - isUnskippable = isMarkedAsUnskippable(this.pickle) - const testSuitePath = getTestSuitePath(testSuiteFullPath, process.cwd()) - isForcedToRun = isUnskippable && skippableSuites.includes(testSuitePath) + const asyncResource = new AsyncResource('bound-anonymous-fn') - testSuiteStartCh.publish({ testSuitePath, isUnskippable, isForcedToRun }) - } + numAttemptToAsyncResource.set(numAttempt, asyncResource) - const testSourceLine = this.gherkinDocument && - this.gherkinDocument.feature && - this.gherkinDocument.feature.location && - this.gherkinDocument.feature.location.line + const testFileAbsolutePath = this.pickle.uri - testStartCh.publish({ - testName: this.pickle.name, - fullTestSuite: testSuiteFullPath, - testSourceLine - }) - try { - const promise = run.apply(this, arguments) - promise.finally(() => { - const result = this.getWorstStepResult() - const { status, skipReason, errorMessage } = isLatestVersion - ? getStatusFromResultLatest(result) : getStatusFromResult(result) + const testSourceLine = this.gherkinDocument?.feature?.location?.line - if (!pickleResultByFile[testSuiteFullPath]) { - pickleResultByFile[testSuiteFullPath] = [status] - } else { - pickleResultByFile[testSuiteFullPath].push(status) - } - testFinishCh.publish({ status, skipReason, errorMessage }) - // last test in suite - if (pickleResultByFile[testSuiteFullPath].length === pickleByFile[testSuiteFullPath].length) { - const testSuiteStatus = getSuiteStatusFromTestStatuses(pickleResultByFile[testSuiteFullPath]) - if (global.__coverage__) { - const coverageFiles = getCoveredFilenamesFromCoverage(global.__coverage__) - - testSuiteCodeCoverageCh.publish({ - coverageFiles, - suiteFile: testSuiteFullPath - }) - // We need to reset coverage to get a code coverage per suite - // Before that, we preserve the original coverage - mergeCoverage(global.__coverage__, originalCoverageMap) - resetCoverage(global.__coverage__) - } - - testSuiteFinishCh.publish(testSuiteStatus) + const testStartPayload = { + testName: this.pickle.name, + testFileAbsolutePath, + testSourceLine, + isParallel: !!process.env.CUCUMBER_WORKER_ID + } + asyncResource.runInAsyncScope(() => { + testStartCh.publish(testStartPayload) + }) + try { + this.eventBroadcaster.on('envelope', shimmer.wrapFunction(null, () => (testCase) => { + // Only supported from >=8.0.0 + if (testCase?.testCaseFinished) { + const { testCaseFinished: { willBeRetried } } = testCase + if (willBeRetried) { // test case failed and will be retried + const failedAttemptAsyncResource = numAttemptToAsyncResource.get(numAttempt) + failedAttemptAsyncResource.runInAsyncScope(() => { + testRetryCh.publish(numAttempt++ > 0) // the current span will be finished and a new one will be created + }) + + const newAsyncResource = new AsyncResource('bound-anonymous-fn') + numAttemptToAsyncResource.set(numAttempt, newAsyncResource) + + newAsyncResource.runInAsyncScope(() => { + testStartCh.publish(testStartPayload) // a new span will be created + }) } + } + })) + let promise + + asyncResource.runInAsyncScope(() => { + promise = run.apply(this, arguments) + }) + promise.finally(() => { + const result = this.getWorstStepResult() + const { status, skipReason, errorMessage } = isLatestVersion + ? getStatusFromResultLatest(result) + : getStatusFromResult(result) + + if (lastStatusByPickleId.has(this.pickle.id)) { + lastStatusByPickleId.get(this.pickle.id).push(status) + } else { + lastStatusByPickleId.set(this.pickle.id, [status]) + } + let isNew = false + let isEfdRetry = false + if (isEarlyFlakeDetectionEnabled && status !== 'skip') { + const numRetries = numRetriesByPickleId.get(this.pickle.id) + + isNew = numRetries !== undefined + isEfdRetry = numRetries > 0 + } + const attemptAsyncResource = numAttemptToAsyncResource.get(numAttempt) + + attemptAsyncResource.runInAsyncScope(() => { + testFinishCh.publish({ status, skipReason, errorMessage, isNew, isEfdRetry, isFlakyRetry: numAttempt > 0 }) }) - return promise - } catch (err) { - errorCh.publish(err) - throw err - } - }) + }) + return promise + } catch (err) { + errorCh.publish(err) + throw err + } }) shimmer.wrap(pl.prototype, 'runStep', runStep => function () { if (!testStepStartCh.hasSubscribers) { @@ -175,7 +301,8 @@ function wrapRun (pl, isLatestVersion) { promise.then((result) => { const { status, skipReason, errorMessage } = isLatestVersion - ? getStatusFromResultLatest(result) : getStatusFromResult(result) + ? getStatusFromResultLatest(result) + : getStatusFromResult(result) testFinishCh.publish({ isStep: true, status, skipReason, errorMessage }) }) @@ -189,12 +316,6 @@ function wrapRun (pl, isLatestVersion) { } function pickleHook (PickleRunner) { - if (process.env.CUCUMBER_WORKER_ID) { - // Parallel mode is not supported - log.warn('Unable to initialize CI Visibility because Cucumber is running in parallel mode.') - return PickleRunner - } - const pl = PickleRunner.default wrapRun(pl, false) @@ -203,12 +324,6 @@ function pickleHook (PickleRunner) { } function testCaseHook (TestCaseRunner) { - if (process.env.CUCUMBER_WORKER_ID) { - // Parallel mode is not supported - log.warn('Unable to initialize CI Visibility because Cucumber is running in parallel mode.') - return TestCaseRunner - } - const pl = TestCaseRunner.default wrapRun(pl, true) @@ -216,115 +331,118 @@ function testCaseHook (TestCaseRunner) { return TestCaseRunner } -addHook({ - name: '@cucumber/cucumber', - versions: ['7.0.0 - 7.2.1'], - file: 'lib/runtime/pickle_runner.js' -}, pickleHook) - -addHook({ - name: '@cucumber/cucumber', - versions: ['>=7.3.0'], - file: 'lib/runtime/test_case_runner.js' -}, testCaseHook) - -function getFilteredPickles (runtime, suitesToSkip) { - return runtime.pickleIds.reduce((acc, pickleId) => { - const test = runtime.eventDataCollector.getPickle(pickleId) - const testSuitePath = getTestSuitePath(test.uri, process.cwd()) - - const isUnskippable = isMarkedAsUnskippable(test) - const isSkipped = suitesToSkip.includes(testSuitePath) - - if (isSkipped && !isUnskippable) { - acc.skippedSuites.add(testSuitePath) - } else { - acc.picklesToRun.push(pickleId) - } - return acc - }, { skippedSuites: new Set(), picklesToRun: [] }) +// Valid for old and new cucumber versions +function getCucumberOptions (adapterOrCoordinator) { + if (adapterOrCoordinator.adapter) { + return adapterOrCoordinator.adapter.worker?.options || adapterOrCoordinator.adapter.options + } + return adapterOrCoordinator.options } -function getPickleByFile (runtime) { - return runtime.pickleIds.reduce((acc, pickleId) => { - const test = runtime.eventDataCollector.getPickle(pickleId) - if (acc[test.uri]) { - acc[test.uri].push(test) - } else { - acc[test.uri] = [test] +function getWrappedStart (start, frameworkVersion, isParallel = false, isCoordinator = false) { + return async function () { + if (!libraryConfigurationCh.hasSubscribers) { + return start.apply(this, arguments) } - return acc - }, {}) -} + const options = getCucumberOptions(this) -addHook({ - name: '@cucumber/cucumber', - versions: ['>=7.0.0'], - file: 'lib/runtime/index.js' -}, (runtimePackage, frameworkVersion) => { - shimmer.wrap(runtimePackage.default.prototype, 'start', start => async function () { - const asyncResource = new AsyncResource('bound-anonymous-fn') - let onDone + if (!isParallel && this.adapter?.options) { + isParallel = options.parallel > 0 + } + let errorSkippableRequest + + const configurationResponse = await getChannelPromise(libraryConfigurationCh) + + isEarlyFlakeDetectionEnabled = configurationResponse.libraryConfig?.isEarlyFlakeDetectionEnabled + earlyFlakeDetectionNumRetries = configurationResponse.libraryConfig?.earlyFlakeDetectionNumRetries + earlyFlakeDetectionFaultyThreshold = configurationResponse.libraryConfig?.earlyFlakeDetectionFaultyThreshold + isSuitesSkippingEnabled = configurationResponse.libraryConfig?.isSuitesSkippingEnabled + isFlakyTestRetriesEnabled = configurationResponse.libraryConfig?.isFlakyTestRetriesEnabled + numTestRetries = configurationResponse.libraryConfig?.flakyTestRetriesCount + + if (isEarlyFlakeDetectionEnabled) { + const knownTestsResponse = await getChannelPromise(knownTestsCh) + if (!knownTestsResponse.err) { + knownTests = knownTestsResponse.knownTests + } else { + isEarlyFlakeDetectionEnabled = false + } + } - const configPromise = new Promise(resolve => { - onDone = resolve - }) + if (isSuitesSkippingEnabled) { + const skippableResponse = await getChannelPromise(skippableSuitesCh) - asyncResource.runInAsyncScope(() => { - itrConfigurationCh.publish({ onDone }) - }) + errorSkippableRequest = skippableResponse.err + skippableSuites = skippableResponse.skippableSuites - await configPromise + if (!errorSkippableRequest) { + const filteredPickles = isCoordinator + ? getFilteredPicklesNew(this, skippableSuites) + : getFilteredPickles(this, skippableSuites) - const skippableSuitesPromise = new Promise(resolve => { - onDone = resolve - }) + const { picklesToRun } = filteredPickles + const oldPickles = isCoordinator ? this.sourcedPickles : this.pickleIds - asyncResource.runInAsyncScope(() => { - skippableSuitesCh.publish({ onDone }) - }) + isSuitesSkipped = picklesToRun.length !== oldPickles.length - const skippableResponse = await skippableSuitesPromise + log.debug( + () => `${picklesToRun.length} out of ${oldPickles.length} suites are going to run.` + ) - const err = skippableResponse.err - skippableSuites = skippableResponse.skippableSuites + if (isCoordinator) { + this.sourcedPickles = picklesToRun + } else { + this.pickleIds = picklesToRun + } - let skippedSuites = [] - let isSuitesSkipped = false + skippedSuites = Array.from(filteredPickles.skippedSuites) + itrCorrelationId = skippableResponse.itrCorrelationId + } + } - if (!err) { - const filteredPickles = getFilteredPickles(this, skippableSuites) - const { picklesToRun } = filteredPickles - isSuitesSkipped = picklesToRun.length !== this.pickleIds.length + pickleByFile = isCoordinator ? getPickleByFileNew(this) : getPickleByFile(this) - log.debug( - () => `${picklesToRun.length} out of ${this.pickleIds.length} suites are going to run.` + if (isEarlyFlakeDetectionEnabled) { + const isFaulty = getIsFaultyEarlyFlakeDetection( + Object.keys(pickleByFile), + knownTests.cucumber || {}, + earlyFlakeDetectionFaultyThreshold ) - - this.pickleIds = picklesToRun - - skippedSuites = Array.from(filteredPickles.skippedSuites) + if (isFaulty) { + isEarlyFlakeDetectionEnabled = false + isEarlyFlakeDetectionFaulty = true + } } - pickleByFile = getPickleByFile(this) - const processArgv = process.argv.slice(2).join(' ') const command = process.env.npm_lifecycle_script || `cucumber-js ${processArgv}` - asyncResource.runInAsyncScope(() => { + if (isFlakyTestRetriesEnabled && !options.retry && numTestRetries > 0) { + options.retry = numTestRetries + } + + sessionAsyncResource.runInAsyncScope(() => { sessionStartCh.publish({ command, frameworkVersion }) }) - if (!err && skippedSuites.length) { + if (!errorSkippableRequest && skippedSuites.length) { itrSkippedSuitesCh.publish({ skippedSuites, frameworkVersion }) } const success = await start.apply(this, arguments) + let untestedCoverage + if (getCodeCoverageCh.hasSubscribers) { + untestedCoverage = await getChannelPromise(getCodeCoverageCh) + } + let testCodeCoverageLinesTotal if (global.__coverage__) { try { + if (untestedCoverage) { + originalCoverageMap.merge(fromCoverageMapToCoverage(untestedCoverage)) + } testCodeCoverageLinesTotal = originalCoverageMap.getCoverageSummary().lines.pct } catch (e) { // ignore errors @@ -333,18 +451,388 @@ addHook({ global.__coverage__ = fromCoverageMapToCoverage(originalCoverageMap) } - asyncResource.runInAsyncScope(() => { + sessionAsyncResource.runInAsyncScope(() => { sessionFinishCh.publish({ status: success ? 'pass' : 'fail', isSuitesSkipped, testCodeCoverageLinesTotal, numSkippedSuites: skippedSuites.length, hasUnskippableSuites: isUnskippable, - hasForcedToRunSuites: isForcedToRun + hasForcedToRunSuites: isForcedToRun, + isEarlyFlakeDetectionEnabled, + isEarlyFlakeDetectionFaulty, + isParallel }) }) + eventDataCollector = null return success - }) + } +} + +// Generates suite start and finish events in the main process. +// Handles EFD in both the main process and the worker process. +function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion = false, isWorker = false) { + return async function () { + let pickle + if (isNewerCucumberVersion) { + pickle = arguments[0].pickle + } else { + pickle = this.eventDataCollector.getPickle(arguments[0]) + } + + const testFileAbsolutePath = pickle.uri + const testSuitePath = getTestSuitePath(testFileAbsolutePath, process.cwd()) + + // If it's a worker, suite events are handled in `getWrappedParseWorkerMessage` + if (!isWorker && !pickleResultByFile[testFileAbsolutePath]) { // first test in suite + isUnskippable = isMarkedAsUnskippable(pickle) + isForcedToRun = isUnskippable && skippableSuites.includes(testSuitePath) + + testSuiteStartCh.publish({ + testFileAbsolutePath, + isUnskippable, + isForcedToRun, + itrCorrelationId + }) + } + + let isNew = false + + if (isEarlyFlakeDetectionEnabled) { + isNew = isNewTest(testSuitePath, pickle.name) + if (isNew) { + numRetriesByPickleId.set(pickle.id, 0) + } + } + // TODO: for >=11 we could use `runTestCaseResult` instead of accumulating results in `lastStatusByPickleId` + let runTestCaseResult = await runTestCaseFunction.apply(this, arguments) + + const testStatuses = lastStatusByPickleId.get(pickle.id) + const lastTestStatus = testStatuses[testStatuses.length - 1] + // If it's a new test and it hasn't been skipped, we run it again + if (isEarlyFlakeDetectionEnabled && lastTestStatus !== 'skip' && isNew) { + for (let retryIndex = 0; retryIndex < earlyFlakeDetectionNumRetries; retryIndex++) { + numRetriesByPickleId.set(pickle.id, retryIndex + 1) + runTestCaseResult = await runTestCaseFunction.apply(this, arguments) + } + } + let testStatus = lastTestStatus + let shouldBePassedByEFD = false + if (isNew && isEarlyFlakeDetectionEnabled) { + /** + * If Early Flake Detection (EFD) is enabled the logic is as follows: + * - If all attempts for a test are failing, the test has failed and we will let the test process fail. + * - If just a single attempt passes, we will prevent the test process from failing. + * The rationale behind is the following: you may still be able to block your CI pipeline by gating + * on flakiness (the test will be considered flaky), but you may choose to unblock the pipeline too. + */ + testStatus = getTestStatusFromRetries(testStatuses) + if (testStatus === 'pass') { + // for cucumber@>=11, setting `this.success` does not work, so we have to change the returned value + shouldBePassedByEFD = true + this.success = true + } + } + + if (!pickleResultByFile[testFileAbsolutePath]) { + pickleResultByFile[testFileAbsolutePath] = [testStatus] + } else { + pickleResultByFile[testFileAbsolutePath].push(testStatus) + } + + // If it's a worker, suite events are handled in `getWrappedParseWorkerMessage` + if (!isWorker && pickleResultByFile[testFileAbsolutePath].length === pickleByFile[testFileAbsolutePath].length) { + // last test in suite + const testSuiteStatus = getSuiteStatusFromTestStatuses(pickleResultByFile[testFileAbsolutePath]) + if (global.__coverage__) { + const coverageFiles = getCoveredFilenamesFromCoverage(global.__coverage__) + + testSuiteCodeCoverageCh.publish({ + coverageFiles, + suiteFile: testFileAbsolutePath, + testSuitePath + }) + // We need to reset coverage to get a code coverage per suite + // Before that, we preserve the original coverage + mergeCoverage(global.__coverage__, originalCoverageMap) + resetCoverage(global.__coverage__) + } + + testSuiteFinishCh.publish({ status: testSuiteStatus, testSuitePath }) + } + + if (isNewerCucumberVersion && isEarlyFlakeDetectionEnabled && isNew) { + return shouldBePassedByEFD + } + + return runTestCaseResult + } +} + +function getWrappedParseWorkerMessage (parseWorkerMessageFunction, isNewVersion) { + return function (worker, message) { + // If the message is an array, it's a dd-trace message, so we need to stop cucumber processing, + // or cucumber will throw an error + // TODO: identify the message better + if (Array.isArray(message)) { + const [messageCode, payload] = message + if (messageCode === CUCUMBER_WORKER_TRACE_PAYLOAD_CODE) { + sessionAsyncResource.runInAsyncScope(() => { + workerReportTraceCh.publish(payload) + }) + return + } + } + + let envelope + + if (isNewVersion) { + envelope = message.envelope + } else { + envelope = message.jsonEnvelope + } + + if (!envelope) { + return parseWorkerMessageFunction.apply(this, arguments) + } + let parsed = envelope + + if (typeof parsed === 'string') { + try { + parsed = JSON.parse(envelope) + } catch (e) { + // ignore errors and continue + return parseWorkerMessageFunction.apply(this, arguments) + } + } + let pickle + + if (parsed.testCaseStarted) { + if (isNewVersion) { + pickle = this.inProgress[worker.id].pickle + } else { + const { pickleId } = this.eventDataCollector.testCaseMap[parsed.testCaseStarted.testCaseId] + pickle = this.eventDataCollector.getPickle(pickleId) + } + // THIS FAILS IN PARALLEL MODE + const testFileAbsolutePath = pickle.uri + // First test in suite + if (!pickleResultByFile[testFileAbsolutePath]) { + pickleResultByFile[testFileAbsolutePath] = [] + testSuiteStartCh.publish({ + testFileAbsolutePath + }) + } + } + + const parseWorkerResponse = parseWorkerMessageFunction.apply(this, arguments) + + // after calling `parseWorkerMessageFunction`, the test status can already be read + if (parsed.testCaseFinished) { + let worstTestStepResult + if (isNewVersion && eventDataCollector) { + pickle = this.inProgress[worker.id].pickle + worstTestStepResult = + eventDataCollector.getTestCaseAttempt(parsed.testCaseFinished.testCaseStartedId).worstTestStepResult + } else { + const testCase = this.eventDataCollector.getTestCaseAttempt(parsed.testCaseFinished.testCaseStartedId) + worstTestStepResult = testCase.worstTestStepResult + pickle = testCase.pickle + } + + const { status } = getStatusFromResultLatest(worstTestStepResult) + let isNew = false + + if (isEarlyFlakeDetectionEnabled) { + isNew = isNewTest(pickle.uri, pickle.name) + } + + const testFileAbsolutePath = pickle.uri + const finished = pickleResultByFile[testFileAbsolutePath] + + if (isNew) { + const testFullname = `${pickle.uri}:${pickle.name}` + let testStatuses = newTestsByTestFullname.get(testFullname) + if (!testStatuses) { + testStatuses = [status] + newTestsByTestFullname.set(testFullname, testStatuses) + } else { + testStatuses.push(status) + } + // We have finished all retries + if (testStatuses.length === earlyFlakeDetectionNumRetries + 1) { + const newTestFinalStatus = getTestStatusFromRetries(testStatuses) + // we only push to `finished` if the retries have finished + finished.push(newTestFinalStatus) + } + } else { + // TODO: can we get error message? + const finished = pickleResultByFile[testFileAbsolutePath] + finished.push(status) + } + + if (finished.length === pickleByFile[testFileAbsolutePath].length) { + testSuiteFinishCh.publish({ + status: getSuiteStatusFromTestStatuses(finished), + testSuitePath: getTestSuitePath(testFileAbsolutePath, process.cwd()) + }) + } + } + + return parseWorkerResponse + } +} + +// Test start / finish for older versions. The only hook executed in workers when in parallel mode +addHook({ + name: '@cucumber/cucumber', + versions: ['7.0.0 - 7.2.1'], + file: 'lib/runtime/pickle_runner.js' +}, pickleHook) + +// Test start / finish for newer versions. The only hook executed in workers when in parallel mode +addHook({ + name: '@cucumber/cucumber', + versions: ['>=7.3.0'], + file: 'lib/runtime/test_case_runner.js' +}, testCaseHook) + +// From 7.3.0 onwards, runPickle becomes runTestCase. Not executed in parallel mode. +// `getWrappedStart` generates session start and finish events +// `getWrappedRunTestCase` generates suite start and finish events and handles EFD. +// TODO (fix): there is a lib/runtime/index in >=11.0.0, but we don't instrument it because it's not useful for us +// This causes a info log saying "Found incompatible integration version". +addHook({ + name: '@cucumber/cucumber', + versions: ['>=7.3.0 <11.0.0'], + file: 'lib/runtime/index.js' +}, (runtimePackage, frameworkVersion) => { + shimmer.wrap(runtimePackage.default.prototype, 'runTestCase', runTestCase => getWrappedRunTestCase(runTestCase)) + + shimmer.wrap(runtimePackage.default.prototype, 'start', start => getWrappedStart(start, frameworkVersion)) + + return runtimePackage +}) + +// Not executed in parallel mode. +// `getWrappedStart` generates session start and finish events +// `getWrappedRunTestCase` generates suite start and finish events and handles EFD. +addHook({ + name: '@cucumber/cucumber', + versions: ['>=7.0.0 <7.3.0'], + file: 'lib/runtime/index.js' +}, (runtimePackage, frameworkVersion) => { + shimmer.wrap(runtimePackage.default.prototype, 'runPickle', runPickle => getWrappedRunTestCase(runPickle)) + shimmer.wrap(runtimePackage.default.prototype, 'start', start => getWrappedStart(start, frameworkVersion)) return runtimePackage }) + +// Only executed in parallel mode. +// `getWrappedStart` generates session start and finish events +// `getWrappedParseWorkerMessage` generates suite start and finish events +addHook({ + name: '@cucumber/cucumber', + versions: ['>=8.0.0 <11.0.0'], + file: 'lib/runtime/parallel/coordinator.js' +}, (coordinatorPackage, frameworkVersion) => { + shimmer.wrap(coordinatorPackage.default.prototype, 'start', start => getWrappedStart(start, frameworkVersion, true)) + shimmer.wrap( + coordinatorPackage.default.prototype, + 'parseWorkerMessage', + parseWorkerMessage => getWrappedParseWorkerMessage(parseWorkerMessage) + ) + return coordinatorPackage +}) + +// >=11.0.0 hooks +// `getWrappedRunTestCase` does two things: +// - generates suite start and finish events in the main process, +// - handles EFD in both the main process and the worker process. +addHook({ + name: '@cucumber/cucumber', + versions: ['>=11.0.0'], + file: 'lib/runtime/worker.js' +}, (workerPackage) => { + shimmer.wrap( + workerPackage.Worker.prototype, + 'runTestCase', + runTestCase => getWrappedRunTestCase(runTestCase, true, !!process.env.CUCUMBER_WORKER_ID) + ) + return workerPackage +}) + +// `getWrappedStart` generates session start and finish events +addHook({ + name: '@cucumber/cucumber', + versions: ['>=11.0.0'], + file: 'lib/runtime/coordinator.js' +}, (coordinatorPackage, frameworkVersion) => { + shimmer.wrap( + coordinatorPackage.Coordinator.prototype, + 'run', + run => getWrappedStart(run, frameworkVersion, false, true) + ) + return coordinatorPackage +}) + +// Necessary because `eventDataCollector` is no longer available in the runtime instance +addHook({ + name: '@cucumber/cucumber', + versions: ['>=11.0.0'], + file: 'lib/formatter/helpers/event_data_collector.js' +}, (eventDataCollectorPackage) => { + shimmer.wrap(eventDataCollectorPackage.default.prototype, 'parseEnvelope', parseEnvelope => function () { + eventDataCollector = this + return parseEnvelope.apply(this, arguments) + }) + return eventDataCollectorPackage +}) + +// Only executed in parallel mode for >=11, in the main process. +// `getWrappedParseWorkerMessage` generates suite start and finish events +// In `startWorker` we pass early flake detection info to the worker. +addHook({ + name: '@cucumber/cucumber', + versions: ['>=11.0.0'], + file: 'lib/runtime/parallel/adapter.js' +}, (adapterPackage) => { + shimmer.wrap( + adapterPackage.ChildProcessAdapter.prototype, + 'parseWorkerMessage', + parseWorkerMessage => getWrappedParseWorkerMessage(parseWorkerMessage, true) + ) + // EFD in parallel mode only supported in >=11.0.0 + shimmer.wrap(adapterPackage.ChildProcessAdapter.prototype, 'startWorker', startWorker => function () { + if (isEarlyFlakeDetectionEnabled) { + this.options.worldParameters._ddKnownTests = knownTests + this.options.worldParameters._ddEarlyFlakeDetectionNumRetries = earlyFlakeDetectionNumRetries + } + + return startWorker.apply(this, arguments) + }) + return adapterPackage +}) + +// Hook executed in the worker process when in parallel mode. +// In this hook we read the information passed in `worldParameters` and make it available for +// `getWrappedRunTestCase`. +addHook({ + name: '@cucumber/cucumber', + versions: ['>=11.0.0'], + file: 'lib/runtime/parallel/worker.js' +}, (workerPackage) => { + shimmer.wrap( + workerPackage.ChildProcessWorker.prototype, + 'initialize', + initialize => async function () { + await initialize.apply(this, arguments) + isEarlyFlakeDetectionEnabled = !!this.options.worldParameters._ddKnownTests + if (isEarlyFlakeDetectionEnabled) { + knownTests = this.options.worldParameters._ddKnownTests + earlyFlakeDetectionNumRetries = this.options.worldParameters._ddEarlyFlakeDetectionNumRetries + } + } + ) + return workerPackage +}) diff --git a/packages/datadog-instrumentations/src/dns.js b/packages/datadog-instrumentations/src/dns.js index 7c4f18c22b7..d37ebc44ee4 100644 --- a/packages/datadog-instrumentations/src/dns.js +++ b/packages/datadog-instrumentations/src/dns.js @@ -18,18 +18,19 @@ const rrtypes = { } const rrtypeMap = new WeakMap() +const names = ['dns', 'node:dns'] -addHook({ name: 'dns' }, dns => { - dns.lookup = wrap('apm:dns:lookup', dns.lookup, 2) - dns.lookupService = wrap('apm:dns:lookup_service', dns.lookupService, 3) - dns.resolve = wrap('apm:dns:resolve', dns.resolve, 2) - dns.reverse = wrap('apm:dns:reverse', dns.reverse, 2) +addHook({ name: names }, dns => { + shimmer.wrap(dns, 'lookup', fn => wrap('apm:dns:lookup', fn, 2)) + shimmer.wrap(dns, 'lookupService', fn => wrap('apm:dns:lookup_service', fn, 2)) + shimmer.wrap(dns, 'resolve', fn => wrap('apm:dns:resolve', fn, 2)) + shimmer.wrap(dns, 'reverse', fn => wrap('apm:dns:reverse', fn, 2)) patchResolveShorthands(dns) if (dns.Resolver) { - dns.Resolver.prototype.resolve = wrap('apm:dns:resolve', dns.Resolver.prototype.resolve, 2) - dns.Resolver.prototype.reverse = wrap('apm:dns:reverse', dns.Resolver.prototype.reverse, 2) + shimmer.wrap(dns.Resolver.prototype, 'resolve', fn => wrap('apm:dns:resolve', fn, 2)) + shimmer.wrap(dns.Resolver.prototype, 'reverse', fn => wrap('apm:dns:reverse', fn, 2)) patchResolveShorthands(dns.Resolver.prototype) } @@ -42,7 +43,7 @@ function patchResolveShorthands (prototype) { .filter(method => !!prototype[method]) .forEach(method => { rrtypeMap.set(prototype[method], rrtypes[method]) - prototype[method] = wrap('apm:dns:resolve', prototype[method], 2, rrtypes[method]) + shimmer.wrap(prototype, method, fn => wrap('apm:dns:resolve', fn, 2, rrtypes[method])) }) } @@ -71,13 +72,13 @@ function wrap (prefix, fn, expectedArgs, rrtype) { return asyncResource.runInAsyncScope(() => { startCh.publish(startArgs) - arguments[arguments.length - 1] = asyncResource.bind(function (error, result) { + arguments[arguments.length - 1] = shimmer.wrapFunction(cb, cb => asyncResource.bind(function (error, result) { if (error) { errorCh.publish(error) } finishCh.publish(result) cb.apply(this, arguments) - }) + })) try { return fn.apply(this, arguments) @@ -91,5 +92,5 @@ function wrap (prefix, fn, expectedArgs, rrtype) { }) } - return shimmer.wrap(fn, wrapped) + return wrapped } diff --git a/packages/datadog-instrumentations/src/elasticsearch.js b/packages/datadog-instrumentations/src/elasticsearch.js index dd123a950a1..ef38cd8dd27 100644 --- a/packages/datadog-instrumentations/src/elasticsearch.js +++ b/packages/datadog-instrumentations/src/elasticsearch.js @@ -48,12 +48,12 @@ function createWrapSelect () { return function () { if (arguments.length === 1) { const cb = arguments[0] - arguments[0] = function (err, connection) { + arguments[0] = shimmer.wrapFunction(cb, cb => function (err, connection) { if (connectCh.hasSubscribers && connection && connection.host) { connectCh.publish({ hostname: connection.host.host, port: connection.host.port }) } cb(err, connection) - } + }) } return request.apply(this, arguments) } @@ -86,10 +86,10 @@ function createWrapRequest (name) { if (typeof cb === 'function') { cb = parentResource.bind(cb) - arguments[lastIndex] = asyncResource.bind(function (error) { + arguments[lastIndex] = shimmer.wrapFunction(cb, cb => asyncResource.bind(function (error) { finish(params, error) return cb.apply(null, arguments) - }) + })) return request.apply(this, arguments) } else { const promise = request.apply(this, arguments) diff --git a/packages/datadog-instrumentations/src/express-mongo-sanitize.js b/packages/datadog-instrumentations/src/express-mongo-sanitize.js index 897ab4e32c1..40d208fce4d 100644 --- a/packages/datadog-instrumentations/src/express-mongo-sanitize.js +++ b/packages/datadog-instrumentations/src/express-mongo-sanitize.js @@ -22,15 +22,15 @@ addHook({ name: 'express-mongo-sanitize', versions: ['>=1.0.0'] }, expressMongoS return sanitizedObject }) - return shimmer.wrap(expressMongoSanitize, function () { + return shimmer.wrapFunction(expressMongoSanitize, expressMongoSanitize => function () { const middleware = expressMongoSanitize.apply(this, arguments) - return shimmer.wrap(middleware, function (req, res, next) { + return shimmer.wrapFunction(middleware, middleware => function (req, res, next) { if (!sanitizeMiddlewareFinished.hasSubscribers) { return middleware.apply(this, arguments) } - const wrappedNext = shimmer.wrap(next, function () { + const wrappedNext = shimmer.wrapFunction(next, next => function () { sanitizeMiddlewareFinished.publish({ sanitizedProperties: propertiesToSanitize, req diff --git a/packages/datadog-instrumentations/src/express.js b/packages/datadog-instrumentations/src/express.js index b07c38a42fe..b093eab7830 100644 --- a/packages/datadog-instrumentations/src/express.js +++ b/packages/datadog-instrumentations/src/express.js @@ -3,7 +3,7 @@ const { createWrapRouterMethod } = require('./router') const shimmer = require('../../datadog-shimmer') const { addHook, channel } = require('./helpers/instrument') -const { AbortController } = require('node-abort-controller') +const tracingChannel = require('dc-polyfill').tracingChannel const handleChannel = channel('apm:express:request:handle') @@ -19,18 +19,60 @@ function wrapHandle (handle) { const wrapRouterMethod = createWrapRouterMethod('express') +const responseJsonChannel = channel('datadog:express:response:json:start') + +function wrapResponseJson (json) { + return function wrappedJson (obj) { + if (responseJsonChannel.hasSubscribers) { + // backward compat as express 4.x supports deprecated 3.x signature + if (arguments.length === 2 && typeof arguments[1] !== 'number') { + obj = arguments[1] + } + + responseJsonChannel.publish({ req: this.req, body: obj }) + } + + return json.apply(this, arguments) + } +} + +const responseRenderChannel = tracingChannel('datadog:express:response:render') + +function wrapResponseRender (render) { + return function wrappedRender (view, options, callback) { + if (!responseRenderChannel.start.hasSubscribers) { + return render.apply(this, arguments) + } + + return responseRenderChannel.traceSync( + render, + { + req: this.req, + view, + options + }, + this, + ...arguments + ) + } +} + addHook({ name: 'express', versions: ['>=4'] }, express => { shimmer.wrap(express.application, 'handle', wrapHandle) shimmer.wrap(express.Router, 'use', wrapRouterMethod) shimmer.wrap(express.Router, 'route', wrapRouterMethod) + shimmer.wrap(express.response, 'json', wrapResponseJson) + shimmer.wrap(express.response, 'jsonp', wrapResponseJson) + shimmer.wrap(express.response, 'render', wrapResponseRender) + return express }) const queryParserReadCh = channel('datadog:query:read:finish') function publishQueryParsedAndNext (req, res, next) { - return function () { + return shimmer.wrapFunction(next, next => function () { if (queryParserReadCh.hasSubscribers && req) { const abortController = new AbortController() const query = req.query @@ -41,7 +83,7 @@ function publishQueryParsedAndNext (req, res, next) { } return next.apply(this, arguments) - } + }) } addHook({ @@ -49,10 +91,10 @@ addHook({ versions: ['>=4'], file: 'lib/middleware/query.js' }, query => { - return shimmer.wrap(query, function () { + return shimmer.wrapFunction(query, query => function () { const queryMiddleware = query.apply(this, arguments) - return shimmer.wrap(queryMiddleware, function (req, res, next) { + return shimmer.wrapFunction(queryMiddleware, queryMiddleware => function (req, res, next) { arguments[2] = publishQueryParsedAndNext(req, res, next) return queryMiddleware.apply(this, arguments) }) @@ -60,11 +102,21 @@ addHook({ }) const processParamsStartCh = channel('datadog:express:process_params:start') -const wrapProcessParamsMethod = (requestPositionInArguments) => { - return (original) => { - return function () { +function wrapProcessParamsMethod (requestPositionInArguments) { + return function wrapProcessParams (original) { + return function wrappedProcessParams () { if (processParamsStartCh.hasSubscribers) { - processParamsStartCh.publish({ req: arguments[requestPositionInArguments] }) + const req = arguments[requestPositionInArguments] + const abortController = new AbortController() + + processParamsStartCh.publish({ + req, + res: req?.res, + abortController, + params: req?.params + }) + + if (abortController.signal.aborted) return } return original.apply(this, arguments) diff --git a/packages/datadog-instrumentations/src/fastify.js b/packages/datadog-instrumentations/src/fastify.js index 6e0a5d1cb6b..726e8284f92 100644 --- a/packages/datadog-instrumentations/src/fastify.js +++ b/packages/datadog-instrumentations/src/fastify.js @@ -5,6 +5,7 @@ const { addHook, channel, AsyncResource } = require('./helpers/instrument') const errorChannel = channel('apm:fastify:middleware:error') const handleChannel = channel('apm:fastify:request:handle') +const routeAddedChannel = channel('apm:fastify:route:added') const parsingResources = new WeakMap() @@ -16,6 +17,7 @@ function wrapFastify (fastify, hasParsingEvents) { if (!app || typeof app.addHook !== 'function') return app + app.addHook('onRoute', onRoute) app.addHook('onRequest', onRequest) app.addHook('preHandler', preHandler) @@ -34,12 +36,12 @@ function wrapFastify (fastify, hasParsingEvents) { } function wrapAddHook (addHook) { - return function addHookWithTrace (name, fn) { + return shimmer.wrapFunction(addHook, addHook => function addHookWithTrace (name, fn) { fn = arguments[arguments.length - 1] if (typeof fn !== 'function') return addHook.apply(this, arguments) - arguments[arguments.length - 1] = shimmer.wrap(fn, function (request, reply, done) { + arguments[arguments.length - 1] = shimmer.wrapFunction(fn, fn => function (request, reply, done) { const req = getReq(request) try { @@ -78,7 +80,7 @@ function wrapAddHook (addHook) { }) return addHook.apply(this, arguments) - } + }) } function onRequest (request, reply, done) { @@ -86,8 +88,9 @@ function onRequest (request, reply, done) { const req = getReq(request) const res = getRes(reply) + const routeConfig = getRouteConfig(request) - handleChannel.publish({ req, res }) + handleChannel.publish({ req, res, routeConfig }) return done() } @@ -142,6 +145,10 @@ function getRes (reply) { return reply && (reply.raw || reply.res || reply) } +function getRouteConfig (request) { + return request?.routeOptions?.config +} + function publishError (error, req) { if (error) { errorChannel.publish({ error, req }) @@ -150,8 +157,12 @@ function publishError (error, req) { return error } +function onRoute (routeOptions) { + routeAddedChannel.publish({ routeOptions, onRoute }) +} + addHook({ name: 'fastify', versions: ['>=3'] }, fastify => { - const wrapped = shimmer.wrap(fastify, wrapFastify(fastify, true)) + const wrapped = shimmer.wrapFunction(fastify, fastify => wrapFastify(fastify, true)) wrapped.fastify = wrapped wrapped.default = wrapped @@ -160,9 +171,9 @@ addHook({ name: 'fastify', versions: ['>=3'] }, fastify => { }) addHook({ name: 'fastify', versions: ['2'] }, fastify => { - return shimmer.wrap(fastify, wrapFastify(fastify, true)) + return shimmer.wrapFunction(fastify, fastify => wrapFastify(fastify, true)) }) addHook({ name: 'fastify', versions: ['1'] }, fastify => { - return shimmer.wrap(fastify, wrapFastify(fastify, false)) + return shimmer.wrapFunction(fastify, fastify => wrapFastify(fastify, false)) }) diff --git a/packages/datadog-instrumentations/src/fetch.js b/packages/datadog-instrumentations/src/fetch.js index 0dcdcf5f566..94f40b6a70d 100644 --- a/packages/datadog-instrumentations/src/fetch.js +++ b/packages/datadog-instrumentations/src/fetch.js @@ -1,51 +1,12 @@ 'use strict' const shimmer = require('../../datadog-shimmer') -const { channel } = require('./helpers/instrument') - -const startChannel = channel('apm:fetch:request:start') -const finishChannel = channel('apm:fetch:request:finish') -const errorChannel = channel('apm:fetch:request:error') - -function wrapFetch (fetch, Request) { - if (typeof fetch !== 'function') return fetch - - return function (input, init) { - if (!startChannel.hasSubscribers) return fetch.apply(this, arguments) - - const req = new Request(input, init) - const headers = req.headers - const message = { req, headers } - - return startChannel.runStores(message, () => { - // Request object is read-only so we need new objects to change headers. - arguments[0] = message.req - arguments[1] = { headers: message.headers } - - return fetch.apply(this, arguments) - .then( - res => { - message.res = res - - finishChannel.publish(message) - - return res - }, - err => { - if (err.name !== 'AbortError') { - message.error = err - errorChannel.publish(message) - } - - finishChannel.publish(message) - - throw err - } - ) - }) - } -} +const { tracingChannel } = require('dc-polyfill') +const { createWrapFetch } = require('./helpers/fetch') if (globalThis.fetch) { - globalThis.fetch = shimmer.wrap(fetch, wrapFetch(fetch, globalThis.Request)) + const ch = tracingChannel('apm:fetch:request') + const wrapFetch = createWrapFetch(globalThis.Request, ch) + + globalThis.fetch = shimmer.wrapFunction(fetch, fetch => wrapFetch(fetch)) } diff --git a/packages/datadog-instrumentations/src/find-my-way.js b/packages/datadog-instrumentations/src/find-my-way.js index 8d5f7d4a5a0..694412c96fe 100644 --- a/packages/datadog-instrumentations/src/find-my-way.js +++ b/packages/datadog-instrumentations/src/find-my-way.js @@ -9,11 +9,11 @@ function wrapOn (on) { return function onWithTrace (method, path, opts) { const index = typeof opts === 'function' ? 2 : 3 const handler = arguments[index] - const wrapper = function (req) { + const wrapper = shimmer.wrapFunction(handler, handler => function (req) { routeChannel.publish({ req, route: path }) return handler.apply(this, arguments) - } + }) if (typeof handler === 'function') { arguments[index] = wrapper diff --git a/packages/datadog-instrumentations/src/fs.js b/packages/datadog-instrumentations/src/fs.js index 8d009dddb4a..9ae201b9860 100644 --- a/packages/datadog-instrumentations/src/fs.js +++ b/packages/datadog-instrumentations/src/fs.js @@ -1,4 +1,3 @@ - 'use strict' const { @@ -267,24 +266,44 @@ function createWrapFunction (prefix = '', override = '') { const lastIndex = arguments.length - 1 const cb = typeof arguments[lastIndex] === 'function' && arguments[lastIndex] const innerResource = new AsyncResource('bound-anonymous-fn') - const message = getMessage(method, getMethodParamsRelationByPrefix(prefix)[operation], arguments, this) + const params = getMethodParamsRelationByPrefix(prefix)[operation] + const abortController = new AbortController() + const message = { ...getMessage(method, params, arguments, this), abortController } + + const finish = innerResource.bind(function (error) { + if (error !== null && typeof error === 'object') { // fs.exists receives a boolean + errorChannel.publish(error) + } + finishChannel.publish() + }) if (cb) { const outerResource = new AsyncResource('bound-anonymous-fn') - arguments[lastIndex] = innerResource.bind(function (e) { - if (typeof e === 'object') { // fs.exists receives a boolean - errorChannel.publish(e) - } - - finishChannel.publish() - + arguments[lastIndex] = shimmer.wrapFunction(cb, cb => innerResource.bind(function (e) { + finish(e) return outerResource.runInAsyncScope(() => cb.apply(this, arguments)) - }) + })) } return innerResource.runInAsyncScope(() => { startChannel.publish(message) + + if (abortController.signal.aborted) { + const error = abortController.signal.reason || new Error('Aborted') + + if (prefix === 'promises.') { + finish(error) + return Promise.reject(error) + } else if (name.includes('Sync') || !cb) { + finish(error) + throw error + } else if (cb) { + arguments[lastIndex](error) + return + } + } + try { const result = original.apply(this, arguments) if (cb) return result diff --git a/packages/datadog-instrumentations/src/google-cloud-pubsub.js b/packages/datadog-instrumentations/src/google-cloud-pubsub.js index c0450b93d49..de1bc209f13 100644 --- a/packages/datadog-instrumentations/src/google-cloud-pubsub.js +++ b/packages/datadog-instrumentations/src/google-cloud-pubsub.js @@ -11,7 +11,7 @@ const requestStartCh = channel('apm:google-cloud-pubsub:request:start') const requestFinishCh = channel('apm:google-cloud-pubsub:request:finish') const requestErrorCh = channel('apm:google-cloud-pubsub:request:error') -const receiveStartCh = channel(`apm:google-cloud-pubsub:receive:start`) +const receiveStartCh = channel('apm:google-cloud-pubsub:receive:start') const receiveFinishCh = channel('apm:google-cloud-pubsub:receive:finish') const receiveErrorCh = channel('apm:google-cloud-pubsub:receive:error') @@ -76,7 +76,7 @@ function wrapMethod (method) { if (typeof cb === 'function') { const outerAsyncResource = new AsyncResource('bound-anonymous-fn') - arguments[arguments.length - 1] = innerAsyncResource.bind(function (error) { + arguments[arguments.length - 1] = shimmer.wrapFunction(cb, cb => innerAsyncResource.bind(function (error) { if (error) { requestErrorCh.publish(error) } @@ -84,7 +84,7 @@ function wrapMethod (method) { requestFinishCh.publish() return outerAsyncResource.runInAsyncScope(() => cb.apply(this, arguments)) - }) + })) return method.apply(this, arguments) } else { diff --git a/packages/datadog-instrumentations/src/graphql.js b/packages/datadog-instrumentations/src/graphql.js index 6d4a18f17cb..c776c4f4fa5 100644 --- a/packages/datadog-instrumentations/src/graphql.js +++ b/packages/datadog-instrumentations/src/graphql.js @@ -37,6 +37,13 @@ const validateStartCh = channel('apm:graphql:validate:start') const validateFinishCh = channel('apm:graphql:validate:finish') const validateErrorCh = channel('apm:graphql:validate:error') +class AbortError extends Error { + constructor (message) { + super(message) + this.name = 'AbortError' + } +} + function getOperation (document, operationName) { if (!document || !Array.isArray(document.definitions)) { return @@ -175,11 +182,11 @@ function wrapExecute (execute) { docSource: documentSources.get(document) }) - const context = { source, asyncResource, fields: {} } + const context = { source, asyncResource, fields: {}, abortController: new AbortController() } contexts.set(contextValue, context) - return callInAsyncScope(exe, asyncResource, this, arguments, (err, res) => { + return callInAsyncScope(exe, asyncResource, this, arguments, context.abortController, (err, res) => { if (finishResolveCh.hasSubscribers) finishResolvers(context) const error = err || (res && res.errors && res.errors[0]) @@ -207,7 +214,7 @@ function wrapResolve (resolve) { const field = assertField(context, info, args) - return callInAsyncScope(resolve, field.asyncResource, this, arguments, (err) => { + return callInAsyncScope(resolve, field.asyncResource, this, arguments, context.abortController, (err) => { updateFieldCh.publish({ field, info, err }) }) } @@ -217,10 +224,15 @@ function wrapResolve (resolve) { return resolveAsync } -function callInAsyncScope (fn, aR, thisArg, args, cb) { +function callInAsyncScope (fn, aR, thisArg, args, abortController, cb) { cb = cb || (() => {}) return aR.runInAsyncScope(() => { + if (abortController?.signal.aborted) { + cb(null, null) + throw new AbortError('Aborted') + } + try { const result = fn.apply(thisArg, args) if (result && typeof result.then === 'function') { diff --git a/packages/datadog-instrumentations/src/grpc/client.js b/packages/datadog-instrumentations/src/grpc/client.js index c1a97d96b45..d52355c0dd6 100644 --- a/packages/datadog-instrumentations/src/grpc/client.js +++ b/packages/datadog-instrumentations/src/grpc/client.js @@ -15,54 +15,52 @@ const errorChannel = channel('apm:grpc:client:request:error') const finishChannel = channel('apm:grpc:client:request:finish') const emitChannel = channel('apm:grpc:client:request:emit') -function createWrapMakeRequest (type) { +function createWrapMakeRequest (type, hasPeer = false) { return function wrapMakeRequest (makeRequest) { return function (path) { const args = ensureMetadata(this, arguments, 4) - return callMethod(this, makeRequest, args, path, args[4], type) + return callMethod(this, makeRequest, args, path, args[4], type, hasPeer) } } } -function createWrapLoadPackageDefinition () { +function createWrapLoadPackageDefinition (hasPeer = false) { return function wrapLoadPackageDefinition (loadPackageDefinition) { return function (packageDef) { const result = loadPackageDefinition.apply(this, arguments) if (!result) return result - wrapPackageDefinition(result) + wrapPackageDefinition(result, hasPeer) return result } } } -function createWrapMakeClientConstructor () { +function createWrapMakeClientConstructor (hasPeer = false) { return function wrapMakeClientConstructor (makeClientConstructor) { return function (methods) { const ServiceClient = makeClientConstructor.apply(this, arguments) - - wrapClientConstructor(ServiceClient, methods) - + wrapClientConstructor(ServiceClient, methods, hasPeer) return ServiceClient } } } -function wrapPackageDefinition (def) { +function wrapPackageDefinition (def, hasPeer = false) { for (const name in def) { if (def[name].format) continue if (def[name].service && def[name].prototype) { - wrapClientConstructor(def[name], def[name].service) + wrapClientConstructor(def[name], def[name].service, hasPeer) } else { - wrapPackageDefinition(def[name]) + wrapPackageDefinition(def[name], hasPeer) } } } -function wrapClientConstructor (ServiceClient, methods) { +function wrapClientConstructor (ServiceClient, methods, hasPeer = false) { const proto = ServiceClient.prototype if (typeof methods !== 'object' || 'format' in methods) return @@ -76,27 +74,24 @@ function wrapClientConstructor (ServiceClient, methods) { const type = getType(methods[name]) if (methods[name]) { - proto[name] = wrapMethod(proto[name], path, type) + proto[name] = wrapMethod(proto[name], path, type, hasPeer) } if (originalName) { - proto[originalName] = wrapMethod(proto[originalName], path, type) + proto[originalName] = wrapMethod(proto[originalName], path, type, hasPeer) } }) } -function wrapMethod (method, path, type) { +function wrapMethod (method, path, type, hasPeer) { if (typeof method !== 'function' || patched.has(method)) { return method } - const wrapped = function () { + const wrapped = shimmer.wrapFunction(method, method => function () { const args = ensureMetadata(this, arguments, 1) - - return callMethod(this, method, args, path, args[1], type) - } - - Object.assign(wrapped, method) + return callMethod(this, method, args, path, args[1], type, hasPeer) + }) patched.add(wrapped) @@ -104,7 +99,7 @@ function wrapMethod (method, path, type) { } function wrapCallback (ctx, callback = () => { }) { - return function (err) { + return shimmer.wrapFunction(callback, callback => function (err) { if (err) { ctx.error = err errorChannel.publish(ctx) @@ -114,10 +109,23 @@ function wrapCallback (ctx, callback = () => { }) { return callback.apply(this, arguments) // No async end channel needed }) - } + }) } -function createWrapEmit (ctx) { +function createWrapEmit (ctx, hasPeer = false) { + const onStatusWithPeer = function (ctx, arg1, thisArg) { + ctx.result = arg1 + ctx.peer = thisArg.getPeer() + finishChannel.publish(ctx) + } + + const onStatusWithoutPeer = function (ctx, arg1, thisArg) { + ctx.result = arg1 + finishChannel.publish(ctx) + } + + const onStatus = hasPeer ? onStatusWithPeer : onStatusWithoutPeer + return function wrapEmit (emit) { return function (event, arg1) { switch (event) { @@ -126,8 +134,7 @@ function createWrapEmit (ctx) { errorChannel.publish(ctx) break case 'status': - ctx.result = arg1 - finishChannel.publish(ctx) + onStatus(ctx, arg1, this) break } @@ -138,7 +145,7 @@ function createWrapEmit (ctx) { } } -function callMethod (client, method, args, path, metadata, type) { +function callMethod (client, method, args, path, metadata, type, hasPeer = false) { if (!startChannel.hasSubscribers) return method.apply(client, args) const length = args.length @@ -159,7 +166,7 @@ function callMethod (client, method, args, path, metadata, type) { const call = method.apply(client, args) if (call && typeof call.emit === 'function') { - shimmer.wrap(call, 'emit', createWrapEmit(ctx)) + shimmer.wrap(call, 'emit', createWrapEmit(ctx, hasPeer)) } return call @@ -223,34 +230,45 @@ function getGrpc (client) { } while ((proto = Object.getPrototypeOf(proto))) } -function patch (grpc) { - const proto = grpc.Client.prototype +function patch (hasPeer = false) { + return function patch (grpc) { + const proto = grpc.Client.prototype - instances.set(proto, grpc) + instances.set(proto, grpc) - shimmer.wrap(proto, 'makeBidiStreamRequest', createWrapMakeRequest(types.bidi)) - shimmer.wrap(proto, 'makeClientStreamRequest', createWrapMakeRequest(types.clientStream)) - shimmer.wrap(proto, 'makeServerStreamRequest', createWrapMakeRequest(types.serverStream)) - shimmer.wrap(proto, 'makeUnaryRequest', createWrapMakeRequest(types.unary)) + shimmer.wrap(proto, 'makeBidiStreamRequest', createWrapMakeRequest(types.bidi, hasPeer)) + shimmer.wrap(proto, 'makeClientStreamRequest', createWrapMakeRequest(types.clientStream, hasPeer)) + shimmer.wrap(proto, 'makeServerStreamRequest', createWrapMakeRequest(types.serverStream, hasPeer)) + shimmer.wrap(proto, 'makeUnaryRequest', createWrapMakeRequest(types.unary, hasPeer)) - return grpc + return grpc + } } if (nodeMajor <= 14) { - addHook({ name: 'grpc', versions: ['>=1.24.3'] }, patch) + addHook({ name: 'grpc', versions: ['>=1.24.3'] }, patch(true)) addHook({ name: 'grpc', versions: ['>=1.24.3'], file: 'src/client.js' }, client => { - shimmer.wrap(client, 'makeClientConstructor', createWrapMakeClientConstructor()) + shimmer.wrap(client, 'makeClientConstructor', createWrapMakeClientConstructor(true)) return client }) } -addHook({ name: '@grpc/grpc-js', versions: ['>=1.0.3'] }, patch) +addHook({ name: '@grpc/grpc-js', versions: ['>=1.0.3 <1.1.4'] }, patch(false)) + +addHook({ name: '@grpc/grpc-js', versions: ['>=1.0.3 <1.1.4'], file: 'build/src/make-client.js' }, client => { + shimmer.wrap(client, 'makeClientConstructor', createWrapMakeClientConstructor(false)) + shimmer.wrap(client, 'loadPackageDefinition', createWrapLoadPackageDefinition(false)) + + return client +}) + +addHook({ name: '@grpc/grpc-js', versions: ['>=1.1.4'] }, patch(true)) -addHook({ name: '@grpc/grpc-js', versions: ['>=1.0.3'], file: 'build/src/make-client.js' }, client => { - shimmer.wrap(client, 'makeClientConstructor', createWrapMakeClientConstructor()) - shimmer.wrap(client, 'loadPackageDefinition', createWrapLoadPackageDefinition()) +addHook({ name: '@grpc/grpc-js', versions: ['>=1.1.4'], file: 'build/src/make-client.js' }, client => { + shimmer.wrap(client, 'makeClientConstructor', createWrapMakeClientConstructor(true)) + shimmer.wrap(client, 'loadPackageDefinition', createWrapLoadPackageDefinition(true)) return client }) diff --git a/packages/datadog-instrumentations/src/grpc/server.js b/packages/datadog-instrumentations/src/grpc/server.js index f3fc2c5a1a8..003dd0cb6c7 100644 --- a/packages/datadog-instrumentations/src/grpc/server.js +++ b/packages/datadog-instrumentations/src/grpc/server.js @@ -92,7 +92,9 @@ function createWrapEmit (call, ctx, onCancel) { finishChannel.publish(ctx) call.removeListener('cancelled', onCancel) break - case 'finish': + // Streams are always cancelled before `finish` since 1.10.0 so we have + // to use `prefinish` instead to avoid cancellation false positives. + case 'prefinish': if (call.status) { updateChannel.publish(call.status) } @@ -117,7 +119,7 @@ function wrapStream (call, ctx, onCancel) { } function wrapCallback (callback = () => {}, call, ctx, onCancel) { - return function (err, value, trailer, flags) { + return shimmer.wrapFunction(callback, callback => function (err, value, trailer, flags) { if (err) { ctx.error = err errorChannel.publish(ctx) @@ -134,7 +136,7 @@ function wrapCallback (callback = () => {}, call, ctx, onCancel) { return callback.apply(this, arguments) // No async end channel needed }) - } + }) } function wrapSendStatus (sendStatus, ctx) { diff --git a/packages/datadog-instrumentations/src/hapi.js b/packages/datadog-instrumentations/src/hapi.js index d3186a93656..afecf4c3284 100644 --- a/packages/datadog-instrumentations/src/hapi.js +++ b/packages/datadog-instrumentations/src/hapi.js @@ -1,12 +1,13 @@ 'use strict' +const tracingChannel = require('dc-polyfill').tracingChannel const shimmer = require('../../datadog-shimmer') -const { addHook, channel, AsyncResource } = require('./helpers/instrument') +const { addHook, channel } = require('./helpers/instrument') const handleChannel = channel('apm:hapi:request:handle') const routeChannel = channel('apm:hapi:request:route') const errorChannel = channel('apm:hapi:request:error') -const enterChannel = channel('apm:hapi:extension:enter') +const hapiTracingChannel = tracingChannel('apm:hapi:extension') function wrapServer (server) { return function (options) { @@ -27,25 +28,25 @@ function wrapServer (server) { } function wrapStart (start) { - return function () { + return shimmer.wrapFunction(start, start => function () { if (this && typeof this.ext === 'function') { this.ext('onPreResponse', onPreResponse) } return start.apply(this, arguments) - } + }) } function wrapExt (ext) { - return function (events, method, options) { - if (typeof events === 'object') { + return shimmer.wrapFunction(ext, ext => function (events, method, options) { + if (events !== null && typeof events === 'object') { arguments[0] = wrapEvents(events) } else { arguments[1] = wrapExtension(method) } return ext.apply(this, arguments) - } + }) } function wrapDispatch (dispatch) { @@ -91,19 +92,15 @@ function wrapEvents (events) { function wrapHandler (handler) { if (typeof handler !== 'function') return handler - return function (request, h) { + return shimmer.wrapFunction(handler, handler => function (request, h) { const req = request && request.raw && request.raw.req if (!req) return handler.apply(this, arguments) - const asyncResource = new AsyncResource('bound-anonymous-fn') - - return asyncResource.runInAsyncScope(() => { - enterChannel.publish({ req }) - + return hapiTracingChannel.traceSync(() => { return handler.apply(this, arguments) }) - } + }) } function onPreResponse (request, h) { diff --git a/packages/datadog-instrumentations/src/helpers/fetch.js b/packages/datadog-instrumentations/src/helpers/fetch.js new file mode 100644 index 00000000000..0ae1f821e9e --- /dev/null +++ b/packages/datadog-instrumentations/src/helpers/fetch.js @@ -0,0 +1,22 @@ +'use strict' + +exports.createWrapFetch = function createWrapFetch (Request, ch) { + return function wrapFetch (fetch) { + if (typeof fetch !== 'function') return fetch + + return function (input, init) { + if (!ch.start.hasSubscribers) return fetch.apply(this, arguments) + + if (input instanceof Request) { + const ctx = { req: input } + + return ch.tracePromise(() => fetch.call(this, input, init), ctx) + } else { + const req = new Request(input, init) + const ctx = { req } + + return ch.tracePromise(() => fetch.call(this, req), ctx) + } + } + } +} diff --git a/packages/datadog-instrumentations/src/helpers/hook.js b/packages/datadog-instrumentations/src/helpers/hook.js index 7bec453187a..0177744ea1c 100644 --- a/packages/datadog-instrumentations/src/helpers/hook.js +++ b/packages/datadog-instrumentations/src/helpers/hook.js @@ -11,8 +11,13 @@ const ritm = require('../../../dd-trace/src/ritm') * @param {string[]} modules list of modules to hook into * @param {Function} onrequire callback to be executed upon encountering module */ -function Hook (modules, onrequire) { - if (!(this instanceof Hook)) return new Hook(modules, onrequire) +function Hook (modules, hookOptions, onrequire) { + if (!(this instanceof Hook)) return new Hook(modules, hookOptions, onrequire) + + if (typeof hookOptions === 'function') { + onrequire = hookOptions + hookOptions = {} + } this._patched = Object.create(null) @@ -28,7 +33,7 @@ function Hook (modules, onrequire) { } this._ritmHook = ritm(modules, {}, safeHook) - this._iitmHook = iitm(modules, {}, (moduleExports, moduleName, moduleBaseDir) => { + this._iitmHook = iitm(modules, hookOptions, (moduleExports, moduleName, moduleBaseDir) => { // TODO: Move this logic to import-in-the-middle and only do it for CommonJS // modules and not ESM. In the meantime, all the modules we instrument are // CommonJS modules for which the default export is always moved to diff --git a/packages/datadog-instrumentations/src/helpers/hooks.js b/packages/datadog-instrumentations/src/helpers/hooks.js index bd409dcaa01..62d45e37008 100644 --- a/packages/datadog-instrumentations/src/helpers/hooks.js +++ b/packages/datadog-instrumentations/src/helpers/hooks.js @@ -1,7 +1,11 @@ 'use strict' module.exports = { + '@apollo/server': () => require('../apollo-server'), + '@apollo/gateway': () => require('../apollo'), + 'apollo-server-core': () => require('../apollo-server-core'), '@aws-sdk/smithy-client': () => require('../aws-sdk'), + '@azure/functions': () => require('../azure-functions'), '@cucumber/cucumber': () => require('../cucumber'), '@playwright/test': () => require('../playwright'), '@elastic/elasticsearch': () => require('../elasticsearch'), @@ -20,83 +24,98 @@ module.exports = { '@opentelemetry/sdk-trace-node': () => require('../otel-sdk-trace'), '@redis/client': () => require('../redis'), '@smithy/smithy-client': () => require('../aws-sdk'), - 'aerospike': () => require('../aerospike'), - 'amqp10': () => require('../amqp10'), - 'amqplib': () => require('../amqplib'), + '@vitest/runner': { esmFirst: true, fn: () => require('../vitest') }, + aerospike: () => require('../aerospike'), + amqp10: () => require('../amqp10'), + amqplib: () => require('../amqplib'), + avsc: () => require('../avsc'), 'aws-sdk': () => require('../aws-sdk'), - 'bluebird': () => require('../bluebird'), + bluebird: () => require('../bluebird'), 'body-parser': () => require('../body-parser'), - 'bunyan': () => require('../bunyan'), + bunyan: () => require('../bunyan'), 'cassandra-driver': () => require('../cassandra-driver'), - 'child_process': () => require('../child-process'), - 'node:child_process': () => require('../child-process'), - 'connect': () => require('../connect'), - 'cookie': () => require('../cookie'), + child_process: () => require('../child_process'), + connect: () => require('../connect'), + cookie: () => require('../cookie'), 'cookie-parser': () => require('../cookie-parser'), - 'couchbase': () => require('../couchbase'), - 'crypto': () => require('../crypto'), - 'cypress': () => require('../cypress'), - 'dns': () => require('../dns'), - 'elasticsearch': () => require('../elasticsearch'), - 'express': () => require('../express'), + couchbase: () => require('../couchbase'), + crypto: () => require('../crypto'), + cypress: () => require('../cypress'), + dns: () => require('../dns'), + elasticsearch: () => require('../elasticsearch'), + express: () => require('../express'), 'express-mongo-sanitize': () => require('../express-mongo-sanitize'), - 'fastify': () => require('../fastify'), + fastify: () => require('../fastify'), 'find-my-way': () => require('../find-my-way'), - 'fs': () => require('../fs'), - 'node:fs': () => require('../fs'), + fs: () => require('../fs'), 'generic-pool': () => require('../generic-pool'), - 'graphql': () => require('../graphql'), - 'grpc': () => require('../grpc'), - 'hapi': () => require('../hapi'), - 'http': () => require('../http'), - 'http2': () => require('../http2'), - 'https': () => require('../http'), - 'ioredis': () => require('../ioredis'), + graphql: () => require('../graphql'), + grpc: () => require('../grpc'), + hapi: () => require('../hapi'), + http: () => require('../http'), + http2: () => require('../http2'), + https: () => require('../http'), + ioredis: () => require('../ioredis'), 'jest-circus': () => require('../jest'), 'jest-config': () => require('../jest'), 'jest-environment-node': () => require('../jest'), 'jest-environment-jsdom': () => require('../jest'), - 'jest-jasmine2': () => require('../jest'), + 'jest-runtime': () => require('../jest'), 'jest-worker': () => require('../jest'), - 'knex': () => require('../knex'), - 'koa': () => require('../koa'), + knex: () => require('../knex'), + koa: () => require('../koa'), 'koa-router': () => require('../koa'), - 'kafkajs': () => require('../kafkajs'), - 'ldapjs': () => require('../ldapjs'), + kafkajs: () => require('../kafkajs'), + ldapjs: () => require('../ldapjs'), 'limitd-client': () => require('../limitd-client'), - 'mariadb': () => require('../mariadb'), - 'memcached': () => require('../memcached'), + lodash: () => require('../lodash'), + mariadb: () => require('../mariadb'), + memcached: () => require('../memcached'), 'microgateway-core': () => require('../microgateway-core'), - 'mocha': () => require('../mocha'), + mocha: () => require('../mocha'), 'mocha-each': () => require('../mocha'), - 'moleculer': () => require('../moleculer'), - 'mongodb': () => require('../mongodb'), + moleculer: () => require('../moleculer'), + mongodb: () => require('../mongodb'), 'mongodb-core': () => require('../mongodb-core'), - 'mongoose': () => require('../mongoose'), - 'mysql': () => require('../mysql'), - 'mysql2': () => require('../mysql2'), - 'net': () => require('../net'), - 'next': () => require('../next'), - 'oracledb': () => require('../oracledb'), - 'openai': () => require('../openai'), - 'paperplane': () => require('../paperplane'), + mongoose: () => require('../mongoose'), + mquery: () => require('../mquery'), + mysql: () => require('../mysql'), + mysql2: () => require('../mysql2'), + net: () => require('../net'), + next: () => require('../next'), + 'node:child_process': () => require('../child_process'), + 'node:crypto': () => require('../crypto'), + 'node:dns': () => require('../dns'), + 'node:http': () => require('../http'), + 'node:http2': () => require('../http2'), + 'node:https': () => require('../http'), + 'node:net': () => require('../net'), + nyc: () => require('../nyc'), + oracledb: () => require('../oracledb'), + openai: () => require('../openai'), + paperplane: () => require('../paperplane'), 'passport-http': () => require('../passport-http'), 'passport-local': () => require('../passport-local'), - 'pg': () => require('../pg'), - 'pino': () => require('../pino'), + pg: () => require('../pg'), + pino: () => require('../pino'), 'pino-pretty': () => require('../pino'), - 'playwright': () => require('../playwright'), + playwright: () => require('../playwright'), 'promise-js': () => require('../promise-js'), - 'promise': () => require('../promise'), - 'q': () => require('../q'), - 'qs': () => require('../qs'), - 'redis': () => require('../redis'), - 'restify': () => require('../restify'), - 'rhea': () => require('../rhea'), - 'router': () => require('../router'), - 'sharedb': () => require('../sharedb'), - 'sequelize': () => require('../sequelize'), - 'tedious': () => require('../tedious'), - 'when': () => require('../when'), - 'winston': () => require('../winston') + promise: () => require('../promise'), + protobufjs: () => require('../protobufjs'), + q: () => require('../q'), + qs: () => require('../qs'), + redis: () => require('../redis'), + restify: () => require('../restify'), + rhea: () => require('../rhea'), + router: () => require('../router'), + 'selenium-webdriver': () => require('../selenium'), + sequelize: () => require('../sequelize'), + sharedb: () => require('../sharedb'), + tedious: () => require('../tedious'), + undici: () => require('../undici'), + vitest: { esmFirst: true, fn: () => require('../vitest') }, + when: () => require('../when'), + winston: () => require('../winston'), + workerpool: () => require('../mocha') } diff --git a/packages/datadog-instrumentations/src/helpers/instrument.js b/packages/datadog-instrumentations/src/helpers/instrument.js index 323c6b01624..20657335044 100644 --- a/packages/datadog-instrumentations/src/helpers/instrument.js +++ b/packages/datadog-instrumentations/src/helpers/instrument.js @@ -17,15 +17,21 @@ exports.channel = function (name) { /** * @param {string} args.name module name * @param {string[]} args.versions array of semver range strings - * @param {string} args.file path to file within package to instrument? + * @param {string} args.file path to file within package to instrument + * @param {string} args.filePattern pattern to match files within package to instrument * @param Function hook */ -exports.addHook = function addHook ({ name, versions, file }, hook) { - if (!instrumentations[name]) { - instrumentations[name] = [] +exports.addHook = function addHook ({ name, versions, file, filePattern }, hook) { + if (typeof name === 'string') { + name = [name] } - instrumentations[name].push({ name, versions, file, hook }) + for (const val of name) { + if (!instrumentations[val]) { + instrumentations[val] = [] + } + instrumentations[val].push({ name: val, versions, file, filePattern, hook }) + } } // AsyncResource.bind exists and binds `this` properly only from 17.8.0 and up. @@ -51,13 +57,13 @@ if (semver.satisfies(process.versions.node, '>=17.8.0')) { bound = this.runInAsyncScope.bind(this, fn, thisArg) } Object.defineProperties(bound, { - 'length': { + length: { configurable: true, enumerable: false, value: fn.length, writable: false }, - 'asyncResource': { + asyncResource: { configurable: true, enumerable: true, value: this, diff --git a/packages/datadog-instrumentations/src/helpers/register.js b/packages/datadog-instrumentations/src/helpers/register.js index e89a91b55f2..4b4185423c0 100644 --- a/packages/datadog-instrumentations/src/helpers/register.js +++ b/packages/datadog-instrumentations/src/helpers/register.js @@ -6,8 +6,13 @@ const semver = require('semver') const Hook = require('./hook') const requirePackageJson = require('../../../dd-trace/src/require-package-json') const log = require('../../../dd-trace/src/log') +const checkRequireCache = require('../check_require_cache') +const telemetry = require('../../../dd-trace/src/telemetry/init-telemetry') -const { DD_TRACE_DISABLED_INSTRUMENTATIONS = '' } = process.env +const { + DD_TRACE_DISABLED_INSTRUMENTATIONS = '', + DD_TRACE_DEBUG = '' +} = process.env const hooks = require('./hooks') const instrumentations = require('./instrumentations') @@ -17,6 +22,15 @@ const disabledInstrumentations = new Set( DD_TRACE_DISABLED_INSTRUMENTATIONS ? DD_TRACE_DISABLED_INSTRUMENTATIONS.split(',') : [] ) +// Check for DD_TRACE__ENABLED environment variables +for (const [key, value] of Object.entries(process.env)) { + const match = key.match(/^DD_TRACE_(.+)_ENABLED$/) + if (match && (value.toLowerCase() === 'false' || value === '0')) { + const integration = match[1].toLowerCase() + disabledInstrumentations.add(integration) + } +} + const loadChannel = channel('dd-trace:instrumentation:load') // Globals @@ -24,38 +38,118 @@ if (!disabledInstrumentations.has('fetch')) { require('../fetch') } -// TODO: make this more efficient +if (!disabledInstrumentations.has('process')) { + require('../process') +} + +const HOOK_SYMBOL = Symbol('hookExportsMap') + +if (DD_TRACE_DEBUG && DD_TRACE_DEBUG.toLowerCase() !== 'false') { + checkRequireCache.checkForRequiredModules() + setImmediate(checkRequireCache.checkForPotentialConflicts) +} + +const seenCombo = new Set() +// TODO: make this more efficient for (const packageName of names) { if (disabledInstrumentations.has(packageName)) continue - Hook([packageName], (moduleExports, moduleName, moduleBaseDir, moduleVersion) => { + const hookOptions = {} + + let hook = hooks[packageName] + + if (typeof hook === 'object') { + hookOptions.internals = hook.esmFirst + hook = hook.fn + } + + Hook([packageName], hookOptions, (moduleExports, moduleName, moduleBaseDir, moduleVersion) => { moduleName = moduleName.replace(pathSepExpr, '/') // This executes the integration file thus adding its entries to `instrumentations` - hooks[packageName]() + hook() if (!instrumentations[packageName]) { return moduleExports } - for (const { name, file, versions, hook } of instrumentations[packageName]) { + const namesAndSuccesses = {} + for (const { name, file, versions, hook, filePattern } of instrumentations[packageName]) { + let fullFilePattern = filePattern const fullFilename = filename(name, file) + if (fullFilePattern) { + fullFilePattern = filename(name, fullFilePattern) + } + + // Create a WeakMap associated with the hook function so that patches on the same moduleExport only happens once + // for example by instrumenting both dns and node:dns double the spans would be created + // since they both patch the same moduleExport, this WeakMap is used to mitigate that + if (!hook[HOOK_SYMBOL]) { + hook[HOOK_SYMBOL] = new WeakMap() + } + let matchesFile = false - if (moduleName === fullFilename) { - const version = moduleVersion || getVersion(moduleBaseDir) + matchesFile = moduleName === fullFilename + + if (fullFilePattern) { + // Some libraries include a hash in their filenames when installed, + // so our instrumentation has to include a '.*' to match them for more than a single version. + matchesFile = matchesFile || new RegExp(fullFilePattern).test(moduleName) + } + + if (matchesFile) { + let version = moduleVersion + try { + version = version || getVersion(moduleBaseDir) + } catch (e) { + log.error(`Error getting version for "${name}": ${e.message}`) + log.error(e) + continue + } + if (typeof namesAndSuccesses[`${name}@${version}`] === 'undefined') { + namesAndSuccesses[`${name}@${version}`] = false + } if (matchVersion(version, versions)) { + // Check if the hook already has a set moduleExport + if (hook[HOOK_SYMBOL].has(moduleExports)) { + namesAndSuccesses[`${name}@${version}`] = true + return moduleExports + } + try { loadChannel.publish({ name, version, file }) - - moduleExports = hook(moduleExports, version) + // Send the name and version of the module back to the callback because now addHook + // takes in an array of names so by passing the name the callback will know which module name is being used + moduleExports = hook(moduleExports, version, name) + // Set the moduleExports in the hooks weakmap + hook[HOOK_SYMBOL].set(moduleExports, name) } catch (e) { - log.error(e) + log.info('Error during ddtrace instrumentation of application, aborting.') + log.info(e) + telemetry('error', [ + `error_type:${e.constructor.name}`, + `integration:${name}`, + `integration_version:${version}` + ]) } + namesAndSuccesses[`${name}@${version}`] = true } } } + for (const nameVersion of Object.keys(namesAndSuccesses)) { + const [name, version] = nameVersion.split('@') + const success = namesAndSuccesses[nameVersion] + if (!success && !seenCombo.has(nameVersion)) { + telemetry('abort.integration', [ + `integration:${name}`, + `integration_version:${version}` + ]) + log.info(`Found incompatible integration version: ${nameVersion}`) + seenCombo.add(nameVersion) + } + } return moduleExports }) diff --git a/packages/datadog-instrumentations/src/http/client.js b/packages/datadog-instrumentations/src/http/client.js index fcf5cc05f0a..29547df61dc 100644 --- a/packages/datadog-instrumentations/src/http/client.js +++ b/packages/datadog-instrumentations/src/http/client.js @@ -14,9 +14,9 @@ const endChannel = channel('apm:http:client:request:end') const asyncStartChannel = channel('apm:http:client:request:asyncStart') const errorChannel = channel('apm:http:client:request:error') -addHook({ name: 'https' }, hookFn) +const names = ['http', 'https', 'node:http', 'node:https'] -addHook({ name: 'http' }, hookFn) +addHook({ name: names }, hookFn) function hookFn (http) { patch(http, 'request') @@ -43,18 +43,20 @@ function patch (http, methodName) { return request.apply(this, arguments) } - const ctx = { args, http } + const abortController = new AbortController() + + const ctx = { args, http, abortController } return startChannel.runStores(ctx, () => { let finished = false let callback = args.callback if (callback) { - callback = function () { + callback = shimmer.wrapFunction(args.callback, cb => function () { return asyncStartChannel.runStores(ctx, () => { - return args.callback.apply(this, arguments) + return cb.apply(this, arguments) }) - } + }) } const options = args.options @@ -107,6 +109,10 @@ function patch (http, methodName) { return emit.apply(this, arguments) } + if (abortController.signal.aborted) { + req.destroy(abortController.signal.reason || new Error('Aborted')) + } + return req } catch (e) { ctx.error = e @@ -132,7 +138,7 @@ function patch (http, methodName) { } function combineOptions (inputURL, inputOptions) { - if (typeof inputOptions === 'object') { + if (inputOptions !== null && typeof inputOptions === 'object') { return Object.assign(inputURL || {}, inputOptions) } else { return inputURL @@ -155,6 +161,7 @@ function patch (http, methodName) { try { return urlToOptions(new url.URL(inputURL)) } catch (e) { + // eslint-disable-next-line n/no-deprecated-api return url.parse(inputURL) } } else if (inputURL instanceof url.URL) { diff --git a/packages/datadog-instrumentations/src/http/server.js b/packages/datadog-instrumentations/src/http/server.js index f3eb528214f..0624c886787 100644 --- a/packages/datadog-instrumentations/src/http/server.js +++ b/packages/datadog-instrumentations/src/http/server.js @@ -1,6 +1,5 @@ 'use strict' -const { AbortController } = require('node-abort-controller') // AbortController is not available in node <15 const { channel, addHook @@ -11,18 +10,32 @@ const startServerCh = channel('apm:http:server:request:start') const exitServerCh = channel('apm:http:server:request:exit') const errorServerCh = channel('apm:http:server:request:error') const finishServerCh = channel('apm:http:server:request:finish') +const startWriteHeadCh = channel('apm:http:server:response:writeHead:start') const finishSetHeaderCh = channel('datadog:http:server:response:set-header:finish') +const startSetHeaderCh = channel('datadog:http:server:response:set-header:start') const requestFinishedSet = new WeakSet() -addHook({ name: 'https' }, http => { - // http.ServerResponse not present on https +const httpNames = ['http', 'node:http'] +const httpsNames = ['https', 'node:https'] + +addHook({ name: httpNames }, http => { + shimmer.wrap(http.ServerResponse.prototype, 'emit', wrapResponseEmit) shimmer.wrap(http.Server.prototype, 'emit', wrapEmit) + shimmer.wrap(http.ServerResponse.prototype, 'writeHead', wrapWriteHead) + shimmer.wrap(http.ServerResponse.prototype, 'write', wrapWrite) + shimmer.wrap(http.ServerResponse.prototype, 'end', wrapEnd) + shimmer.wrap(http.ServerResponse.prototype, 'setHeader', wrapSetHeader) + shimmer.wrap(http.ServerResponse.prototype, 'removeHeader', wrapAppendOrRemoveHeader) + // Added in node v16.17.0 + if (http.ServerResponse.prototype.appendHeader) { + shimmer.wrap(http.ServerResponse.prototype, 'appendHeader', wrapAppendOrRemoveHeader) + } return http }) -addHook({ name: 'http' }, http => { - shimmer.wrap(http.ServerResponse.prototype, 'emit', wrapResponseEmit) +addHook({ name: httpsNames }, http => { + // http.ServerResponse not present on https shimmer.wrap(http.Server.prototype, 'emit', wrapEmit) return http }) @@ -59,9 +72,7 @@ function wrapEmit (emit) { // TODO: should this always return true ? return this.listenerCount(eventName) > 0 } - if (finishSetHeaderCh.hasSubscribers) { - wrapSetHeader(res) - } + return emit.apply(this, arguments) } catch (err) { errorServerCh.publish(err) @@ -75,12 +86,138 @@ function wrapEmit (emit) { } } -function wrapSetHeader (res) { - shimmer.wrap(res, 'setHeader', setHeader => { - return function (name, value) { - const setHeaderResult = setHeader.apply(this, arguments) - finishSetHeaderCh.publish({ name, value, res }) - return setHeaderResult +function wrapWriteHead (writeHead) { + return function wrappedWriteHead (statusCode, reason, obj) { + if (!startWriteHeadCh.hasSubscribers) { + return writeHead.apply(this, arguments) + } + + const abortController = new AbortController() + + if (typeof reason !== 'string') { + obj ??= reason + } + + // support writeHead(200, ['key1', 'val1', 'key2', 'val2']) + if (Array.isArray(obj)) { + const headers = {} + + for (let i = 0; i < obj.length; i += 2) { + headers[obj[i]] = obj[i + 1] + } + + obj = headers + } + + // this doesn't support explicit duplicate headers, but it's an edge case + const responseHeaders = Object.assign(this.getHeaders(), obj) + + startWriteHeadCh.publish({ + req: this.req, + res: this, + abortController, + statusCode, + responseHeaders + }) + + if (abortController.signal.aborted) { + return this + } + + return writeHead.apply(this, arguments) + } +} + +function wrapWrite (write) { + return function wrappedWrite () { + if (!startWriteHeadCh.hasSubscribers) { + return write.apply(this, arguments) + } + + const abortController = new AbortController() + + const responseHeaders = this.getHeaders() + + startWriteHeadCh.publish({ + req: this.req, + res: this, + abortController, + statusCode: this.statusCode, + responseHeaders + }) + + if (abortController.signal.aborted) { + return true } - }) + + return write.apply(this, arguments) + } +} + +function wrapSetHeader (setHeader) { + return function wrappedSetHeader (name, value) { + if (!startSetHeaderCh.hasSubscribers && !finishSetHeaderCh.hasSubscribers) { + return setHeader.apply(this, arguments) + } + + if (startSetHeaderCh.hasSubscribers) { + const abortController = new AbortController() + startSetHeaderCh.publish({ res: this, abortController }) + + if (abortController.signal.aborted) { + return + } + } + + const setHeaderResult = setHeader.apply(this, arguments) + + if (finishSetHeaderCh.hasSubscribers) { + finishSetHeaderCh.publish({ name, value, res: this }) + } + + return setHeaderResult + } +} + +function wrapAppendOrRemoveHeader (originalMethod) { + return function wrappedAppendOrRemoveHeader () { + if (!startSetHeaderCh.hasSubscribers) { + return originalMethod.apply(this, arguments) + } + + const abortController = new AbortController() + startSetHeaderCh.publish({ res: this, abortController }) + + if (abortController.signal.aborted) { + return this + } + + return originalMethod.apply(this, arguments) + } +} + +function wrapEnd (end) { + return function wrappedEnd () { + if (!startWriteHeadCh.hasSubscribers) { + return end.apply(this, arguments) + } + + const abortController = new AbortController() + + const responseHeaders = this.getHeaders() + + startWriteHeadCh.publish({ + req: this.req, + res: this, + abortController, + statusCode: this.statusCode, + responseHeaders + }) + + if (abortController.signal.aborted) { + return this + } + + return end.apply(this, arguments) + } } diff --git a/packages/datadog-instrumentations/src/http2/client.js b/packages/datadog-instrumentations/src/http2/client.js index de4957318ae..651c9ed6edd 100644 --- a/packages/datadog-instrumentations/src/http2/client.js +++ b/packages/datadog-instrumentations/src/http2/client.js @@ -10,6 +10,8 @@ const asyncStartChannel = channel('apm:http2:client:request:asyncStart') const asyncEndChannel = channel('apm:http2:client:request:asyncEnd') const errorChannel = channel('apm:http2:client:request:error') +const names = ['http2', 'node:http2'] + function createWrapEmit (ctx) { return function wrapEmit (emit) { return function (event, arg1) { @@ -66,7 +68,7 @@ function wrapConnect (connect) { } } -addHook({ name: 'http2' }, http2 => { +addHook({ name: names }, http2 => { shimmer.wrap(http2, 'connect', wrapConnect) return http2 diff --git a/packages/datadog-instrumentations/src/http2/server.js b/packages/datadog-instrumentations/src/http2/server.js index 6c9a290c7a7..07bfa11e453 100644 --- a/packages/datadog-instrumentations/src/http2/server.js +++ b/packages/datadog-instrumentations/src/http2/server.js @@ -14,7 +14,9 @@ const startServerCh = channel('apm:http2:server:request:start') const errorServerCh = channel('apm:http2:server:request:error') const finishServerCh = channel('apm:http2:server:request:finish') -addHook({ name: 'http2' }, http2 => { +const names = ['http2', 'node:http2'] + +addHook({ name: names }, http2 => { shimmer.wrap(http2, 'createSecureServer', wrapCreateServer) shimmer.wrap(http2, 'createServer', wrapCreateServer) return http2 diff --git a/packages/datadog-instrumentations/src/jest.js b/packages/datadog-instrumentations/src/jest.js index f62f0c9fac9..e006f311dc3 100644 --- a/packages/datadog-instrumentations/src/jest.js +++ b/packages/datadog-instrumentations/src/jest.js @@ -9,14 +9,16 @@ const { JEST_WORKER_COVERAGE_PAYLOAD_CODE, getTestLineStart, getTestSuitePath, - getTestParametersString + getTestParametersString, + addEfdStringToTestName, + removeEfdStringFromTestName, + getIsFaultyEarlyFlakeDetection } = require('../../dd-trace/src/plugins/util/test') const { getFormattedJestTestParameters, getJestTestName, getJestSuitesToRun } = require('../../datadog-plugin-jest/src/util') -const { DD_MAJOR } = require('../../../version') const testSessionStartCh = channel('ci:jest:session:start') const testSessionFinishCh = channel('ci:jest:session:finish') @@ -37,11 +39,22 @@ const testRunFinishCh = channel('ci:jest:test:finish') const testErrCh = channel('ci:jest:test:err') const skippableSuitesCh = channel('ci:jest:test-suite:skippable') -const jestItrConfigurationCh = channel('ci:jest:itr-configuration') +const libraryConfigurationCh = channel('ci:jest:library-configuration') +const knownTestsCh = channel('ci:jest:known-tests') const itrSkippedSuitesCh = channel('ci:jest:itr:skipped-suites') +// Message sent by jest's main process to workers to run a test suite (=test file) +// https://github.com/jestjs/jest/blob/1d682f21c7a35da4d3ab3a1436a357b980ebd0fa/packages/jest-worker/src/types.ts#L37 +const CHILD_MESSAGE_CALL = 1 +// Maximum time we'll wait for the tracer to flush +const FLUSH_TIMEOUT = 10000 +// eslint-disable-next-line +// https://github.com/jestjs/jest/blob/41f842a46bb2691f828c3a5f27fc1d6290495b82/packages/jest-circus/src/types.ts#L9C8-L9C54 +const RETRY_TIMES = Symbol.for('RETRY_TIMES') + let skippableSuites = [] +let knownTests = {} let isCodeCoverageEnabled = false let isSuitesSkippingEnabled = false let isUserCodeCoverageEnabled = false @@ -49,19 +62,18 @@ let isSuitesSkipped = false let numSkippedSuites = 0 let hasUnskippableSuites = false let hasForcedToRunSuites = false +let isEarlyFlakeDetectionEnabled = false +let earlyFlakeDetectionNumRetries = 0 +let earlyFlakeDetectionFaultyThreshold = 30 +let isEarlyFlakeDetectionFaulty = false +let hasFilteredSkippableSuites = false const sessionAsyncResource = new AsyncResource('bound-anonymous-fn') -const specStatusToTestStatus = { - 'pending': 'skip', - 'disabled': 'skip', - 'todo': 'skip', - 'passed': 'pass', - 'failed': 'fail' -} - const asyncResources = new WeakMap() const originalTestFns = new WeakMap() +const retriedTestsToNumAttempts = new Map() +const newTestsTestStatuses = new Map() // based on https://github.com/facebook/jest/blob/main/packages/jest-circus/src/formatNodeAssertErrors.ts#L41 function formatJestError (errors) { @@ -90,6 +102,13 @@ function getTestEnvironmentOptions (config) { return {} } +function getEfdStats (testStatuses) { + return testStatuses.reduce((acc, testStatus) => { + acc[testStatus]++ + return acc + }, { pass: 0, fail: 0 }) +} + function getWrappedEnvironment (BaseEnvironment, jestVersion) { return class DatadogEnvironment extends BaseEnvironment { constructor (config, context) { @@ -99,8 +118,78 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { this.testSuite = getTestSuitePath(context.testPath, rootDir) this.nameToParams = {} this.global._ddtrace = global._ddtrace + this.hasSnapshotTests = undefined + this.displayName = config.projectConfig?.displayName?.name this.testEnvironmentOptions = getTestEnvironmentOptions(config) + + const repositoryRoot = this.testEnvironmentOptions._ddRepositoryRoot + + if (repositoryRoot) { + this.testSourceFile = getTestSuitePath(context.testPath, repositoryRoot) + } + + this.isEarlyFlakeDetectionEnabled = this.testEnvironmentOptions._ddIsEarlyFlakeDetectionEnabled + this.isFlakyTestRetriesEnabled = this.testEnvironmentOptions._ddIsFlakyTestRetriesEnabled + this.flakyTestRetriesCount = this.testEnvironmentOptions._ddFlakyTestRetriesCount + + if (this.isEarlyFlakeDetectionEnabled) { + const hasKnownTests = !!knownTests.jest + earlyFlakeDetectionNumRetries = this.testEnvironmentOptions._ddEarlyFlakeDetectionNumRetries + try { + this.knownTestsForThisSuite = hasKnownTests + ? (knownTests.jest[this.testSuite] || []) + : this.getKnownTestsForSuite(this.testEnvironmentOptions._ddKnownTests) + } catch (e) { + // If there has been an error parsing the tests, we'll disable Early Flake Deteciton + this.isEarlyFlakeDetectionEnabled = false + } + } + + if (this.isFlakyTestRetriesEnabled) { + const currentNumRetries = this.global[RETRY_TIMES] + if (!currentNumRetries) { + this.global[RETRY_TIMES] = this.flakyTestRetriesCount + } + } + } + + getHasSnapshotTests () { + if (this.hasSnapshotTests !== undefined) { + return this.hasSnapshotTests + } + let hasSnapshotTests = true + try { + const { _snapshotData } = this.getVmContext().expect.getState().snapshotState + hasSnapshotTests = Object.keys(_snapshotData).length > 0 + } catch (e) { + // if we can't be sure, we'll err on the side of caution and assume it has snapshots + } + this.hasSnapshotTests = hasSnapshotTests + return hasSnapshotTests + } + + // Function that receives a list of known tests for a test service and + // returns the ones that belong to the current suite + getKnownTestsForSuite (knownTests) { + if (this.knownTestsForThisSuite) { + return this.knownTestsForThisSuite + } + let knownTestsForSuite = knownTests + // If jest is using workers, known tests are serialized to json. + // If jest runs in band, they are not. + if (typeof knownTestsForSuite === 'string') { + knownTestsForSuite = JSON.parse(knownTestsForSuite) + } + return knownTestsForSuite + } + + // Add the `add_test` event we don't have the test object yet, so + // we use its describe block to get the full name + getTestNameFromAddTestEvent (event, state) { + const describeSuffix = getJestTestName(state.currentDescribeBlock) + const fullTestName = describeSuffix ? `${describeSuffix} ${event.testName}` : event.testName + return removeEfdStringFromTestName(fullTestName) } async handleTestEvent (event, state) { @@ -124,23 +213,65 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { } } if (event.name === 'test_start') { + let isNewTest = false + let numEfdRetry = null const testParameters = getTestParametersString(this.nameToParams, event.test.name) // Async resource for this test is created here // It is used later on by the test_done handler const asyncResource = new AsyncResource('bound-anonymous-fn') asyncResources.set(event.test, asyncResource) + const testName = getJestTestName(event.test) + + if (this.isEarlyFlakeDetectionEnabled) { + const originalTestName = removeEfdStringFromTestName(testName) + isNewTest = retriedTestsToNumAttempts.has(originalTestName) + if (isNewTest) { + numEfdRetry = retriedTestsToNumAttempts.get(originalTestName) + retriedTestsToNumAttempts.set(originalTestName, numEfdRetry + 1) + } + } + const isJestRetry = event.test?.invocations > 1 asyncResource.runInAsyncScope(() => { testStartCh.publish({ - name: getJestTestName(event.test), + name: removeEfdStringFromTestName(testName), suite: this.testSuite, + testSourceFile: this.testSourceFile, runner: 'jest-circus', + displayName: this.displayName, testParameters, - frameworkVersion: jestVersion + frameworkVersion: jestVersion, + isNew: isNewTest, + isEfdRetry: numEfdRetry > 0, + isJestRetry }) originalTestFns.set(event.test, event.test.fn) event.test.fn = asyncResource.bind(event.test.fn) }) } + if (event.name === 'add_test') { + if (this.isEarlyFlakeDetectionEnabled) { + const testName = this.getTestNameFromAddTestEvent(event, state) + const isNew = !this.knownTestsForThisSuite?.includes(testName) + const isSkipped = event.mode === 'todo' || event.mode === 'skip' + if (isNew && !isSkipped && !retriedTestsToNumAttempts.has(testName)) { + retriedTestsToNumAttempts.set(testName, 0) + // Retrying snapshots has proven to be problematic, so we'll skip them for now + // We'll still detect new tests, but we won't retry them. + // TODO: do not bail out of EFD with the whole test suite + if (this.getHasSnapshotTests()) { + log.warn('Early flake detection is disabled for suites with snapshots') + return + } + for (let retryIndex = 0; retryIndex < earlyFlakeDetectionNumRetries; retryIndex++) { + if (this.global.test) { + this.global.test(addEfdStringToTestName(event.testName, retryIndex), event.fn, event.timeout) + } else { + log.error('Early flake detection could not retry test because global.test is undefined') + } + } + } + } + } if (event.name === 'test_done') { const asyncResource = asyncResources.get(event.test) asyncResource.runInAsyncScope(() => { @@ -156,6 +287,19 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { }) // restore in case it is retried event.test.fn = originalTestFns.get(event.test) + // We'll store the test statuses of the retries + if (this.isEarlyFlakeDetectionEnabled) { + const testName = getJestTestName(event.test) + const originalTestName = removeEfdStringFromTestName(testName) + const isNewTest = retriedTestsToNumAttempts.has(originalTestName) + if (isNewTest) { + if (newTestsTestStatuses.has(originalTestName)) { + newTestsTestStatuses.get(originalTestName).push(status) + } else { + newTestsTestStatuses.set(originalTestName, [status]) + } + } + } }) } if (event.name === 'test_skip' || event.name === 'test_todo') { @@ -164,7 +308,9 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { testSkippedCh.publish({ name: getJestTestName(event.test), suite: this.testSuite, + testSourceFile: this.testSourceFile, runner: 'jest-circus', + displayName: this.displayName, frameworkVersion: jestVersion, testStartLine: getTestLineStart(event.test.asyncError, this.testSuite) }) @@ -184,6 +330,23 @@ function getTestEnvironment (pkg, jestVersion) { return getWrappedEnvironment(pkg, jestVersion) } +function applySuiteSkipping (originalTests, rootDir, frameworkVersion) { + const jestSuitesToRun = getJestSuitesToRun(skippableSuites, originalTests, rootDir || process.cwd()) + hasFilteredSkippableSuites = true + log.debug( + () => `${jestSuitesToRun.suitesToRun.length} out of ${originalTests.length} suites are going to run.` + ) + hasUnskippableSuites = jestSuitesToRun.hasUnskippableSuites + hasForcedToRunSuites = jestSuitesToRun.hasForcedToRunSuites + + isSuitesSkipped = jestSuitesToRun.suitesToRun.length !== originalTests.length + numSkippedSuites = jestSuitesToRun.skippedSuites.length + + itrSkippedSuitesCh.publish({ skippedSuites: jestSuitesToRun.skippedSuites, frameworkVersion }) + + return jestSuitesToRun.suitesToRun +} + addHook({ name: 'jest-environment-node', versions: ['>=24.8.0'] @@ -194,36 +357,65 @@ addHook({ versions: ['>=24.8.0'] }, getTestEnvironment) +function getWrappedScheduleTests (scheduleTests, frameworkVersion) { + return async function (tests) { + if (!isSuitesSkippingEnabled || hasFilteredSkippableSuites) { + return scheduleTests.apply(this, arguments) + } + const [test] = tests + const rootDir = test?.context?.config?.rootDir + + arguments[0] = applySuiteSkipping(tests, rootDir, frameworkVersion) + + return scheduleTests.apply(this, arguments) + } +} + +addHook({ + name: '@jest/core', + file: 'build/TestScheduler.js', + versions: ['>=27.0.0'] +}, (testSchedulerPackage, frameworkVersion) => { + const oldCreateTestScheduler = testSchedulerPackage.createTestScheduler + const newCreateTestScheduler = async function () { + if (!isSuitesSkippingEnabled || hasFilteredSkippableSuites) { + return oldCreateTestScheduler.apply(this, arguments) + } + // If suite skipping is enabled and has not filtered skippable suites yet, we'll attempt to do it + const scheduler = await oldCreateTestScheduler.apply(this, arguments) + shimmer.wrap(scheduler, 'scheduleTests', scheduleTests => getWrappedScheduleTests(scheduleTests, frameworkVersion)) + return scheduler + } + testSchedulerPackage.createTestScheduler = newCreateTestScheduler + return testSchedulerPackage +}) + +addHook({ + name: '@jest/core', + file: 'build/TestScheduler.js', + versions: ['>=24.8.0 <27.0.0'] +}, (testSchedulerPackage, frameworkVersion) => { + shimmer.wrap( + testSchedulerPackage.default.prototype, + 'scheduleTests', scheduleTests => getWrappedScheduleTests(scheduleTests, frameworkVersion) + ) + return testSchedulerPackage +}) + addHook({ name: '@jest/test-sequencer', - versions: ['>=24.8.0'] + versions: ['>=28'] }, (sequencerPackage, frameworkVersion) => { shimmer.wrap(sequencerPackage.default.prototype, 'shard', shard => function () { const shardedTests = shard.apply(this, arguments) - if (!shardedTests.length) { + if (!shardedTests.length || !isSuitesSkippingEnabled || !skippableSuites.length) { return shardedTests } - // TODO: could we get the rootDir from each test? const [test] = shardedTests - const rootDir = test && test.context && test.context.config && test.context.config.rootDir - - const jestSuitesToRun = getJestSuitesToRun(skippableSuites, shardedTests, rootDir || process.cwd()) - - log.debug( - () => `${jestSuitesToRun.suitesToRun.length} out of ${shardedTests.length} suites are going to run.` - ) - - hasUnskippableSuites = jestSuitesToRun.hasUnskippableSuites - hasForcedToRunSuites = jestSuitesToRun.hasForcedToRunSuites + const rootDir = test?.context?.config?.rootDir - isSuitesSkipped = jestSuitesToRun.suitesToRun.length !== shardedTests.length - numSkippedSuites = jestSuitesToRun.skippedSuites.length - - itrSkippedSuitesCh.publish({ skippedSuites: jestSuitesToRun.skippedSuites, frameworkVersion }) - - skippableSuites = [] - return jestSuitesToRun.suitesToRun + return applySuiteSkipping(shardedTests, rootDir, frameworkVersion) }) return sequencerPackage }) @@ -234,24 +426,49 @@ function cliWrapper (cli, jestVersion) { const configurationPromise = new Promise((resolve) => { onDone = resolve }) - if (!jestItrConfigurationCh.hasSubscribers) { + if (!libraryConfigurationCh.hasSubscribers) { return runCLI.apply(this, arguments) } sessionAsyncResource.runInAsyncScope(() => { - jestItrConfigurationCh.publish({ onDone }) + libraryConfigurationCh.publish({ onDone }) }) try { - const { err, itrConfig } = await configurationPromise + const { err, libraryConfig } = await configurationPromise if (!err) { - isCodeCoverageEnabled = itrConfig.isCodeCoverageEnabled - isSuitesSkippingEnabled = itrConfig.isSuitesSkippingEnabled + isCodeCoverageEnabled = libraryConfig.isCodeCoverageEnabled + isSuitesSkippingEnabled = libraryConfig.isSuitesSkippingEnabled + isEarlyFlakeDetectionEnabled = libraryConfig.isEarlyFlakeDetectionEnabled + earlyFlakeDetectionNumRetries = libraryConfig.earlyFlakeDetectionNumRetries + earlyFlakeDetectionFaultyThreshold = libraryConfig.earlyFlakeDetectionFaultyThreshold } } catch (err) { log.error(err) } + if (isEarlyFlakeDetectionEnabled) { + const knownTestsPromise = new Promise((resolve) => { + onDone = resolve + }) + + sessionAsyncResource.runInAsyncScope(() => { + knownTestsCh.publish({ onDone }) + }) + + try { + const { err, knownTests: receivedKnownTests } = await knownTestsPromise + if (!err) { + knownTests = receivedKnownTests + } else { + // We disable EFD if there has been an error in the known tests request + isEarlyFlakeDetectionEnabled = false + } + } catch (err) { + log.error(err) + } + } + if (isSuitesSkippingEnabled) { const skippableSuitesPromise = new Promise((resolve) => { onDone = resolve @@ -311,6 +528,21 @@ function cliWrapper (cli, jestVersion) { status = 'fail' error = new Error(`Failed test suites: ${numFailedTestSuites}. Failed tests: ${numFailedTests}`) } + let timeoutId + + // Pass the resolve callback to defer it to DC listener + const flushPromise = new Promise((resolve) => { + onDone = () => { + clearTimeout(timeoutId) + resolve() + } + }) + + const timeoutPromise = new Promise((resolve) => { + timeoutId = setTimeout(() => { + resolve('timeout') + }, FLUSH_TIMEOUT).unref() + }) sessionAsyncResource.runInAsyncScope(() => { testSessionFinishCh.publish({ @@ -322,12 +554,42 @@ function cliWrapper (cli, jestVersion) { numSkippedSuites, hasUnskippableSuites, hasForcedToRunSuites, - error + error, + isEarlyFlakeDetectionEnabled, + isEarlyFlakeDetectionFaulty, + onDone }) }) + const waitingResult = await Promise.race([flushPromise, timeoutPromise]) + + if (waitingResult === 'timeout') { + log.error('Timeout waiting for the tracer to flush') + } numSkippedSuites = 0 + /** + * If Early Flake Detection (EFD) is enabled the logic is as follows: + * - If all attempts for a test are failing, the test has failed and we will let the test process fail. + * - If just a single attempt passes, we will prevent the test process from failing. + * The rationale behind is the following: you may still be able to block your CI pipeline by gating + * on flakiness (the test will be considered flaky), but you may choose to unblock the pipeline too. + */ + + if (isEarlyFlakeDetectionEnabled) { + let numFailedTestsToIgnore = 0 + for (const testStatuses of newTestsTestStatuses.values()) { + const { pass, fail } = getEfdStats(testStatuses) + if (pass > 0) { // as long as one passes, we'll consider the test passed + numFailedTestsToIgnore += fail + } + } + // If every test that failed was an EFD retry, we'll consider the suite passed + if (numFailedTestsToIgnore !== 0 && result.results.numFailedTests === numFailedTestsToIgnore) { + result.results.success = true + } + } + return result }) @@ -341,10 +603,13 @@ function coverageReporterWrapper (coverageReporter) { /** * If ITR is active, we're running fewer tests, so of course the total code coverage is reduced. - * This calculation adds no value, so we'll skip it. + * This calculation adds no value, so we'll skip it, as long as the user has not manually opted in to code coverage, + * in which case we'll leave it. */ shimmer.wrap(CoverageReporter.prototype, '_addUntestedFiles', addUntestedFiles => async function () { - if (isSuitesSkippingEnabled) { + // If the user has added coverage manually, they're willing to pay the price of this execution, so + // we will not skip it. + if (isSuitesSkippingEnabled && !isUserCodeCoverageEnabled) { return Promise.resolve() } return addUntestedFiles.apply(this, arguments) @@ -373,7 +638,7 @@ addHook({ function jestAdapterWrapper (jestAdapter, jestVersion) { const adapter = jestAdapter.default ? jestAdapter.default : jestAdapter - const newAdapter = shimmer.wrap(adapter, function () { + const newAdapter = shimmer.wrapFunction(adapter, adapter => function () { const environment = arguments[2] if (!environment) { return adapter.apply(this, arguments) @@ -383,6 +648,8 @@ function jestAdapterWrapper (jestAdapter, jestVersion) { testSuiteStartCh.publish({ testSuite: environment.testSuite, testEnvironmentOptions: environment.testEnvironmentOptions, + testSourceFile: environment.testSourceFile, + displayName: environment.displayName, frameworkVersion: jestVersion }) return adapter.apply(this, arguments).then(suiteResults => { @@ -403,7 +670,7 @@ function jestAdapterWrapper (jestAdapter, jestVersion) { const coverageFiles = getCoveredFilenamesFromCoverage(environment.global.__coverage__) .map(filename => getTestSuitePath(filename, environment.rootDir)) asyncResource.runInAsyncScope(() => { - testSuiteCodeCoverageCh.publish([...coverageFiles, environment.testSuite]) + testSuiteCodeCoverageCh.publish({ coverageFiles, testSuite: environment.testSuite }) }) } testSuiteFinishCh.publish({ status, errorMessage }) @@ -442,6 +709,10 @@ function configureTestEnvironment (readConfigsResult) { isUserCodeCoverageEnabled = !!readConfigsResult.globalConfig.collectCoverage + if (readConfigsResult.globalConfig.forceExit) { + log.warn("Jest's '--forceExit' flag has been passed. This may cause loss of data.") + } + if (isCodeCoverageEnabled) { const globalConfig = { ...readConfigsResult.globalConfig, @@ -495,6 +766,16 @@ addHook({ _ddTestModuleId, _ddTestSessionId, _ddTestCommand, + _ddTestSessionName, + _ddForcedToRun, + _ddUnskippable, + _ddItrCorrelationId, + _ddKnownTests, + _ddIsEarlyFlakeDetectionEnabled, + _ddEarlyFlakeDetectionNumRetries, + _ddRepositoryRoot, + _ddIsFlakyTestRetriesEnabled, + _ddFlakyTestRetriesCount, ...restOfTestEnvironmentOptions } = testEnvironmentOptions @@ -519,13 +800,26 @@ addHook({ const SearchSource = searchSourcePackage.default ? searchSourcePackage.default : searchSourcePackage shimmer.wrap(SearchSource.prototype, 'getTestPaths', getTestPaths => async function () { - if (!skippableSuites.length) { - return getTestPaths.apply(this, arguments) - } - + const testPaths = await getTestPaths.apply(this, arguments) const [{ rootDir, shard }] = arguments - if (shard && shard.shardIndex) { + if (isEarlyFlakeDetectionEnabled) { + const projectSuites = testPaths.tests.map(test => getTestSuitePath(test.path, test.context.config.rootDir)) + const isFaulty = + getIsFaultyEarlyFlakeDetection(projectSuites, knownTests.jest || {}, earlyFlakeDetectionFaultyThreshold) + if (isFaulty) { + log.error('Early flake detection is disabled because the number of new suites is too high.') + isEarlyFlakeDetectionEnabled = false + const testEnvironmentOptions = testPaths.tests[0]?.context?.config?.testEnvironmentOptions + // Project config is shared among all tests, so we can modify it here + if (testEnvironmentOptions) { + testEnvironmentOptions._ddIsEarlyFlakeDetectionEnabled = false + } + isEarlyFlakeDetectionFaulty = true + } + } + + if (shard?.shardCount > 1 || !isSuitesSkippingEnabled || !skippableSuites.length) { // If the user is using jest sharding, we want to apply the filtering of tests in the shard process. // The reason for this is the following: // The tests for different shards are likely being run in different CI jobs so @@ -533,27 +827,12 @@ addHook({ // If the skippable endpoint is returning different suites and we filter the list of tests here, // the base list of tests that is used for sharding might be different, // causing the shards to potentially run the same suite. - return getTestPaths.apply(this, arguments) + return testPaths } - - const testPaths = await getTestPaths.apply(this, arguments) const { tests } = testPaths - const jestSuitesToRun = getJestSuitesToRun(skippableSuites, tests, rootDir) - - log.debug(() => `${jestSuitesToRun.suitesToRun.length} out of ${tests.length} suites are going to run.`) - - hasUnskippableSuites = jestSuitesToRun.hasUnskippableSuites - hasForcedToRunSuites = jestSuitesToRun.hasForcedToRunSuites - - isSuitesSkipped = jestSuitesToRun.suitesToRun.length !== tests.length - numSkippedSuites = jestSuitesToRun.skippedSuites.length - - itrSkippedSuitesCh.publish({ skippedSuites: jestSuitesToRun.skippedSuites, frameworkVersion }) - - skippableSuites = [] - - return { ...testPaths, tests: jestSuitesToRun.suitesToRun } + const suitesToRun = applySuiteSkipping(tests, rootDir, frameworkVersion) + return { ...testPaths, tests: suitesToRun } }) return searchSourcePackage @@ -570,51 +849,73 @@ addHook({ versions: ['24.8.0 - 24.9.0'] }, jestConfigSyncWrapper) -function jasmineAsyncInstallWraper (jasmineAsyncInstallExport, jestVersion) { - log.warn('jest-jasmine2 support is removed from dd-trace@v4. Consider changing to jest-circus as `testRunner`.') - return function (globalConfig, globalInput) { - globalInput._ddtrace = global._ddtrace - shimmer.wrap(globalInput.jasmine.Spec.prototype, 'execute', execute => function (onComplete) { - const asyncResource = new AsyncResource('bound-anonymous-fn') - asyncResource.runInAsyncScope(() => { - const testSuite = getTestSuitePath(this.result.testPath, globalConfig.rootDir) - testStartCh.publish({ - name: this.getFullName(), - suite: testSuite, - runner: 'jest-jasmine2', - frameworkVersion: jestVersion - }) - const spec = this - const callback = asyncResource.bind(function () { - if (spec.result.failedExpectations && spec.result.failedExpectations.length) { - const formattedError = formatJestError(spec.result.failedExpectations[0].error) - testErrCh.publish(formattedError) - } - testRunFinishCh.publish({ status: specStatusToTestStatus[spec.result.status] }) - onComplete.apply(this, arguments) - }) - arguments[0] = callback - execute.apply(this, arguments) - }) - }) - return jasmineAsyncInstallExport.default(globalConfig, globalInput) - } -} +const LIBRARIES_BYPASSING_JEST_REQUIRE_ENGINE = [ + 'selenium-webdriver', + 'winston' +] -if (DD_MAJOR < 4) { - addHook({ - name: 'jest-jasmine2', - versions: ['>=24.8.0'], - file: 'build/jasmineAsyncInstall.js' - }, jasmineAsyncInstallWraper) +function shouldBypassJestRequireEngine (moduleName) { + return ( + LIBRARIES_BYPASSING_JEST_REQUIRE_ENGINE.some(library => moduleName.includes(library)) + ) } +addHook({ + name: 'jest-runtime', + versions: ['>=24.8.0'] +}, (runtimePackage) => { + const Runtime = runtimePackage.default ? runtimePackage.default : runtimePackage + + shimmer.wrap(Runtime.prototype, 'requireModuleOrMock', requireModuleOrMock => function (from, moduleName) { + // TODO: do this for every library that we instrument + if (shouldBypassJestRequireEngine(moduleName)) { + // To bypass jest's own require engine + return this._requireCoreModule(moduleName) + } + return requireModuleOrMock.apply(this, arguments) + }) + + return runtimePackage +}) + addHook({ name: 'jest-worker', versions: ['>=24.9.0'], file: 'build/workers/ChildProcessWorker.js' }, (childProcessWorker) => { const ChildProcessWorker = childProcessWorker.default + shimmer.wrap(ChildProcessWorker.prototype, 'send', send => function (request) { + if (!isEarlyFlakeDetectionEnabled) { + return send.apply(this, arguments) + } + const [type] = request + // eslint-disable-next-line + // https://github.com/jestjs/jest/blob/1d682f21c7a35da4d3ab3a1436a357b980ebd0fa/packages/jest-worker/src/workers/ChildProcessWorker.ts#L424 + if (type === CHILD_MESSAGE_CALL) { + // This is the message that the main process sends to the worker to run a test suite (=test file). + // In here we modify the config.testEnvironmentOptions to include the known tests for the suite. + // This way the suite only knows about the tests that are part of it. + const args = request[request.length - 1] + if (args.length > 1) { + return send.apply(this, arguments) + } + if (!args[0]?.config) { + return send.apply(this, arguments) + } + const [{ globalConfig, config, path: testSuiteAbsolutePath }] = args + const testSuite = getTestSuitePath(testSuiteAbsolutePath, globalConfig.rootDir || process.cwd()) + const suiteKnownTests = knownTests.jest?.[testSuite] || [] + args[0].config = { + ...config, + testEnvironmentOptions: { + ...config.testEnvironmentOptions, + _ddKnownTests: suiteKnownTests + } + } + } + + return send.apply(this, arguments) + }) shimmer.wrap(ChildProcessWorker.prototype, '_onMessage', _onMessage => function () { const [code, data] = arguments[0] if (code === JEST_WORKER_TRACE_PAYLOAD_CODE) { // datadog trace payload diff --git a/packages/datadog-instrumentations/src/kafkajs.js b/packages/datadog-instrumentations/src/kafkajs.js index beaba513097..e75c03e7e64 100644 --- a/packages/datadog-instrumentations/src/kafkajs.js +++ b/packages/datadog-instrumentations/src/kafkajs.js @@ -8,19 +8,42 @@ const { const shimmer = require('../../datadog-shimmer') const producerStartCh = channel('apm:kafkajs:produce:start') +const producerCommitCh = channel('apm:kafkajs:produce:commit') const producerFinishCh = channel('apm:kafkajs:produce:finish') const producerErrorCh = channel('apm:kafkajs:produce:error') const consumerStartCh = channel('apm:kafkajs:consume:start') +const consumerCommitCh = channel('apm:kafkajs:consume:commit') const consumerFinishCh = channel('apm:kafkajs:consume:finish') const consumerErrorCh = channel('apm:kafkajs:consume:error') +const batchConsumerStartCh = channel('apm:kafkajs:consume-batch:start') +const batchConsumerFinishCh = channel('apm:kafkajs:consume-batch:finish') +const batchConsumerErrorCh = channel('apm:kafkajs:consume-batch:error') + +function commitsFromEvent (event) { + const { payload: { groupId, topics } } = event + const commitList = [] + for (const { topic, partitions } of topics) { + for (const { partition, offset } of partitions) { + commitList.push({ + groupId, + partition, + offset, + topic + }) + } + } + consumerCommitCh.publish(commitList) +} + addHook({ name: 'kafkajs', file: 'src/index.js', versions: ['>=1.4'] }, (BaseKafka) => { class Kafka extends BaseKafka { constructor (options) { super(options) this._brokers = (options.brokers && typeof options.brokers !== 'function') - ? options.brokers.join(',') : undefined + ? options.brokers.join(',') + : undefined } } @@ -29,42 +52,59 @@ addHook({ name: 'kafkajs', file: 'src/index.js', versions: ['>=1.4'] }, (BaseKaf const send = producer.send const bootstrapServers = this._brokers - producer.send = function () { - const innerAsyncResource = new AsyncResource('bound-anonymous-fn') + const kafkaClusterIdPromise = getKafkaClusterId(this) - return innerAsyncResource.runInAsyncScope(() => { - if (!producerStartCh.hasSubscribers) { - return send.apply(this, arguments) - } + producer.send = function () { + const wrappedSend = (clusterId) => { + const innerAsyncResource = new AsyncResource('bound-anonymous-fn') - try { - const { topic, messages = [] } = arguments[0] - for (const message of messages) { - if (typeof message === 'object') { - message.headers = message.headers || {} - } + return innerAsyncResource.runInAsyncScope(() => { + if (!producerStartCh.hasSubscribers) { + return send.apply(this, arguments) } - producerStartCh.publish({ topic, messages, bootstrapServers }) - - const result = send.apply(this, arguments) - result.then( - innerAsyncResource.bind(() => producerFinishCh.publish(undefined)), - innerAsyncResource.bind(err => { - if (err) { - producerErrorCh.publish(err) + try { + const { topic, messages = [] } = arguments[0] + for (const message of messages) { + if (message !== null && typeof message === 'object') { + message.headers = message.headers || {} } - producerFinishCh.publish(undefined) - }) - ) - - return result - } catch (e) { - producerErrorCh.publish(e) - producerFinishCh.publish(undefined) - throw e - } - }) + } + producerStartCh.publish({ topic, messages, bootstrapServers, clusterId }) + + const result = send.apply(this, arguments) + + result.then( + innerAsyncResource.bind(res => { + producerFinishCh.publish(undefined) + producerCommitCh.publish(res) + }), + innerAsyncResource.bind(err => { + if (err) { + producerErrorCh.publish(err) + } + producerFinishCh.publish(undefined) + }) + ) + + return result + } catch (e) { + producerErrorCh.publish(e) + producerFinishCh.publish(undefined) + throw e + } + }) + } + + if (!isPromise(kafkaClusterIdPromise)) { + // promise is already resolved + return wrappedSend(kafkaClusterIdPromise) + } else { + // promise is not resolved + return kafkaClusterIdPromise.then((clusterId) => { + return wrappedSend(clusterId) + }) + } } return producer }) @@ -74,47 +114,128 @@ addHook({ name: 'kafkajs', file: 'src/index.js', versions: ['>=1.4'] }, (BaseKaf return createConsumer.apply(this, arguments) } + const kafkaClusterIdPromise = getKafkaClusterId(this) + + const eachMessageExtractor = (args, clusterId) => { + const { topic, partition, message } = args[0] + return { topic, partition, message, groupId, clusterId } + } + + const eachBatchExtractor = (args, clusterId) => { + const { batch } = args[0] + const { topic, partition, messages } = batch + return { topic, partition, messages, groupId, clusterId } + } + const consumer = createConsumer.apply(this, arguments) - const run = consumer.run + consumer.on(consumer.events.COMMIT_OFFSETS, commitsFromEvent) + + const run = consumer.run const groupId = arguments[0].groupId - consumer.run = function ({ eachMessage, ...runArgs }) { - if (typeof eachMessage !== 'function') return run({ eachMessage, ...runArgs }) - - return run({ - eachMessage: function (...eachMessageArgs) { - const innerAsyncResource = new AsyncResource('bound-anonymous-fn') - return innerAsyncResource.runInAsyncScope(() => { - const { topic, partition, message } = eachMessageArgs[0] - consumerStartCh.publish({ topic, partition, message, groupId }) - try { - const result = eachMessage.apply(this, eachMessageArgs) - if (result && typeof result.then === 'function') { - result.then( - innerAsyncResource.bind(() => consumerFinishCh.publish(undefined)), - innerAsyncResource.bind(err => { - if (err) { - consumerErrorCh.publish(err) - } - consumerFinishCh.publish(undefined) - }) - ) - } else { - consumerFinishCh.publish(undefined) - } - return result - } catch (e) { - consumerErrorCh.publish(e) - consumerFinishCh.publish(undefined) - throw e - } - }) - }, - ...runArgs - }) + consumer.run = function ({ eachMessage, eachBatch, ...runArgs }) { + const wrapConsume = (clusterId) => { + return run({ + eachMessage: wrappedCallback( + eachMessage, + consumerStartCh, + consumerFinishCh, + consumerErrorCh, + eachMessageExtractor, + clusterId + ), + eachBatch: wrappedCallback( + eachBatch, + batchConsumerStartCh, + batchConsumerFinishCh, + batchConsumerErrorCh, + eachBatchExtractor, + clusterId + ), + ...runArgs + }) + } + + if (!isPromise(kafkaClusterIdPromise)) { + // promise is already resolved + return wrapConsume(kafkaClusterIdPromise) + } else { + // promise is not resolved + return kafkaClusterIdPromise.then((clusterId) => { + return wrapConsume(clusterId) + }) + } } return consumer }) return Kafka }) + +const wrappedCallback = (fn, startCh, finishCh, errorCh, extractArgs, clusterId) => { + return typeof fn === 'function' + ? function (...args) { + const innerAsyncResource = new AsyncResource('bound-anonymous-fn') + return innerAsyncResource.runInAsyncScope(() => { + const extractedArgs = extractArgs(args, clusterId) + + startCh.publish(extractedArgs) + try { + const result = fn.apply(this, args) + if (result && typeof result.then === 'function') { + result.then( + innerAsyncResource.bind(() => finishCh.publish(undefined)), + innerAsyncResource.bind(err => { + if (err) { + errorCh.publish(err) + } + finishCh.publish(undefined) + }) + ) + } else { + finishCh.publish(undefined) + } + return result + } catch (e) { + errorCh.publish(e) + finishCh.publish(undefined) + throw e + } + }) + } + : fn +} + +const getKafkaClusterId = (kafka) => { + if (kafka._ddKafkaClusterId) { + return kafka._ddKafkaClusterId + } + + if (!kafka.admin) { + return null + } + + const admin = kafka.admin() + + if (!admin.describeCluster) { + return null + } + + return admin.connect() + .then(() => { + return admin.describeCluster() + }) + .then((clusterInfo) => { + const clusterId = clusterInfo?.clusterId + kafka._ddKafkaClusterId = clusterId + admin.disconnect() + return clusterId + }) + .catch((error) => { + throw error + }) +} + +function isPromise (obj) { + return !!obj && (typeof obj === 'object' || typeof obj === 'function') && typeof obj.then === 'function' +} diff --git a/packages/datadog-instrumentations/src/knex.js b/packages/datadog-instrumentations/src/knex.js index 6c684a3e0d5..43d5db38181 100644 --- a/packages/datadog-instrumentations/src/knex.js +++ b/packages/datadog-instrumentations/src/knex.js @@ -81,8 +81,8 @@ addHook({ function wrapCallbackWithFinish (callback, finish) { if (typeof callback !== 'function') return callback - return function () { + return shimmer.wrapFunction(callback, callback => function () { finish() callback.apply(this, arguments) - } + }) } diff --git a/packages/datadog-instrumentations/src/koa.js b/packages/datadog-instrumentations/src/koa.js index bb281503f22..f139eb1c494 100644 --- a/packages/datadog-instrumentations/src/koa.js +++ b/packages/datadog-instrumentations/src/koa.js @@ -71,7 +71,7 @@ function wrapStack (layer) { middleware = original || middleware - const handler = shimmer.wrap(middleware, wrapMiddleware(middleware, layer)) + const handler = shimmer.wrapFunction(middleware, middleware => wrapMiddleware(middleware, layer)) originals.set(handler, middleware) @@ -84,7 +84,7 @@ function wrapMiddleware (fn, layer) { const name = fn.name - return function (ctx, next) { + return shimmer.wrapFunction(fn, fn => function (ctx, next) { if (!ctx || !enterChannel.hasSubscribers) return fn.apply(this, arguments) const req = ctx.req @@ -122,7 +122,7 @@ function wrapMiddleware (fn, layer) { } finally { exitChannel.publish({ req }) } - } + }) } function fulfill (ctx, error) { @@ -142,11 +142,11 @@ function fulfill (ctx, error) { } function wrapNext (req, next) { - return function () { + return shimmer.wrapFunction(next, next => function () { nextChannel.publish({ req }) return next.apply(this, arguments) - } + }) } addHook({ name: 'koa', versions: ['>=2'] }, Koa => { diff --git a/packages/datadog-instrumentations/src/ldapjs.js b/packages/datadog-instrumentations/src/ldapjs.js index b4df501dfb5..da70642aee9 100644 --- a/packages/datadog-instrumentations/src/ldapjs.js +++ b/packages/datadog-instrumentations/src/ldapjs.js @@ -61,7 +61,7 @@ addHook({ name: 'ldapjs', versions: ['>=2'] }, ldapjs => { let filter if (isString(options)) { filter = options - } else if (typeof options === 'object' && options.filter) { + } else if (options !== null && typeof options === 'object' && options.filter) { if (isString(options.filter)) { filter = options.filter } @@ -76,8 +76,9 @@ addHook({ name: 'ldapjs', versions: ['>=2'] }, ldapjs => { const callbackIndex = getCallbackArgIndex(arguments) if (callbackIndex > -1) { const callback = arguments[callbackIndex] - arguments[callbackIndex] = shimmer.wrap(callback, function (err, corkedEmitter) { - if (typeof corkedEmitter === 'object' && typeof corkedEmitter['on'] === 'function') { + // eslint-disable-next-line n/handle-callback-err + arguments[callbackIndex] = shimmer.wrapFunction(callback, callback => function (err, corkedEmitter) { + if (corkedEmitter !== null && typeof corkedEmitter === 'object' && typeof corkedEmitter.on === 'function') { wrapEmitter(corkedEmitter) } callback.apply(this, arguments) diff --git a/packages/datadog-instrumentations/src/lodash.js b/packages/datadog-instrumentations/src/lodash.js new file mode 100644 index 00000000000..2ac9a1afd64 --- /dev/null +++ b/packages/datadog-instrumentations/src/lodash.js @@ -0,0 +1,31 @@ +'use strict' + +const { channel, addHook } = require('./helpers/instrument') + +const shimmer = require('../../datadog-shimmer') + +addHook({ name: 'lodash', versions: ['>=4'] }, lodash => { + const lodashOperationCh = channel('datadog:lodash:operation') + + const instrumentedLodashFn = ['trim', 'trimStart', 'trimEnd', 'toLower', 'toUpper', 'join'] + + shimmer.massWrap( + lodash, + instrumentedLodashFn, + lodashFn => { + return function () { + if (!lodashOperationCh.hasSubscribers) { + return lodashFn.apply(this, arguments) + } + + const result = lodashFn.apply(this, arguments) + const message = { operation: lodashFn.name, arguments, result } + lodashOperationCh.publish(message) + + return message.result + } + } + ) + + return lodash +}) diff --git a/packages/datadog-instrumentations/src/mariadb.js b/packages/datadog-instrumentations/src/mariadb.js index 2afe5928c8f..9a6891bfa20 100644 --- a/packages/datadog-instrumentations/src/mariadb.js +++ b/packages/datadog-instrumentations/src/mariadb.js @@ -11,7 +11,7 @@ const skipCh = channel('apm:mariadb:pool:skip') const unskipCh = channel('apm:mariadb:pool:unskip') function wrapCommandStart (start, callbackResource) { - return function () { + return shimmer.wrapFunction(start, start => function () { if (!startCh.hasSubscribers) return start.apply(this, arguments) const resolve = callbackResource.bind(this.resolve) @@ -44,7 +44,7 @@ function wrapCommandStart (start, callbackResource) { startCh.publish({ sql: this.sql, conf: this.opts }) return start.apply(this, arguments) }) - } + }) } function wrapCommand (Command) { @@ -98,7 +98,7 @@ function createWrapQueryCallback (options) { arguments.length = arguments.length + 1 } - arguments[arguments.length - 1] = asyncResource.bind(function (err) { + arguments[arguments.length - 1] = shimmer.wrapFunction(cb, cb => asyncResource.bind(function (err) { if (err) { errorCh.publish(err) } @@ -108,7 +108,7 @@ function createWrapQueryCallback (options) { if (typeof cb === 'function') { return callbackResource.runInAsyncScope(() => cb.apply(this, arguments)) } - }) + })) return asyncResource.runInAsyncScope(() => { startCh.publish({ sql, conf: options }) @@ -119,7 +119,7 @@ function createWrapQueryCallback (options) { } } -function wrapConnection (Connection, promiseMethod) { +function wrapConnection (promiseMethod, Connection) { return function (options) { Connection.apply(this, arguments) @@ -170,13 +170,13 @@ addHook({ name, file: 'lib/pool.js', versions: ['>=3'] }, (Pool) => { }) addHook({ name, file: 'lib/connection.js', versions: ['>=2.5.2 <3'] }, (Connection) => { - return shimmer.wrap(Connection, wrapConnection(Connection, '_queryPromise')) + return shimmer.wrapFunction(Connection, wrapConnection.bind(null, '_queryPromise')) }) addHook({ name, file: 'lib/connection.js', versions: ['>=2.0.4 <=2.5.1'] }, (Connection) => { - return shimmer.wrap(Connection, wrapConnection(Connection, 'query')) + return shimmer.wrapFunction(Connection, wrapConnection.bind(null, 'query')) }) addHook({ name, file: 'lib/pool-base.js', versions: ['>=2.0.4 <3'] }, (PoolBase) => { - return shimmer.wrap(PoolBase, wrapPoolBase(PoolBase)) + return shimmer.wrapFunction(PoolBase, wrapPoolBase) }) diff --git a/packages/datadog-instrumentations/src/memcached.js b/packages/datadog-instrumentations/src/memcached.js index 3190cec0032..672f5a6a6e4 100644 --- a/packages/datadog-instrumentations/src/memcached.js +++ b/packages/datadog-instrumentations/src/memcached.js @@ -26,14 +26,14 @@ addHook({ name: 'memcached', versions: ['>=2.2'] }, Memcached => { const query = queryCompiler.apply(this, arguments) const callback = callbackResource.bind(query.callback) - query.callback = asyncResource.bind(function (err) { + query.callback = shimmer.wrapFunction(callback, callback => asyncResource.bind(function (err) { if (err) { errorCh.publish(err) } finishCh.publish() return callback.apply(this, arguments) - }) + })) startCh.publish({ client, server, query }) return query diff --git a/packages/datadog-instrumentations/src/microgateway-core.js b/packages/datadog-instrumentations/src/microgateway-core.js index f0bcd8aa4ad..f96a769ae85 100644 --- a/packages/datadog-instrumentations/src/microgateway-core.js +++ b/packages/datadog-instrumentations/src/microgateway-core.js @@ -8,7 +8,9 @@ const routeChannel = channel('apm:microgateway-core:request:route') const errorChannel = channel('apm:microgateway-core:request:error') const name = 'microgateway-core' -const versions = ['>=2.1'] + +// TODO Remove " <=3.0.0" when "volos-util-apigee" module is fixed +const versions = ['>=2.1 <=3.0.0'] const requestResources = new WeakMap() function wrapConfigProxyFactory (configProxyFactory) { @@ -40,7 +42,7 @@ function wrapPluginsFactory (pluginsFactory) { } function wrapNext (req, res, next) { - return function nextWithTrace (err) { + return shimmer.wrapFunction(next, next => function nextWithTrace (err) { const requestResource = requestResources.get(req) requestResource.runInAsyncScope(() => { @@ -54,13 +56,13 @@ function wrapNext (req, res, next) { }) return next.apply(this, arguments) - } + }) } addHook({ name, versions, file: 'lib/config-proxy-middleware.js' }, configProxyFactory => { - return shimmer.wrap(configProxyFactory, wrapConfigProxyFactory(configProxyFactory)) + return shimmer.wrapFunction(configProxyFactory, wrapConfigProxyFactory) }) addHook({ name, versions, file: 'lib/plugins-middleware.js' }, pluginsFactory => { - return shimmer.wrap(pluginsFactory, wrapPluginsFactory(pluginsFactory)) + return shimmer.wrapFunction(pluginsFactory, wrapPluginsFactory) }) diff --git a/packages/datadog-instrumentations/src/mocha.js b/packages/datadog-instrumentations/src/mocha.js index 462fb42d20f..fec27587bc7 100644 --- a/packages/datadog-instrumentations/src/mocha.js +++ b/packages/datadog-instrumentations/src/mocha.js @@ -1,538 +1,9 @@ -const { createCoverageMap } = require('istanbul-lib-coverage') - -const { isMarkedAsUnskippable } = require('../../datadog-plugin-jest/src/util') - -const { addHook, channel, AsyncResource } = require('./helpers/instrument') -const shimmer = require('../../datadog-shimmer') -const log = require('../../dd-trace/src/log') -const { - getCoveredFilenamesFromCoverage, - resetCoverage, - mergeCoverage, - getTestSuitePath, - fromCoverageMapToCoverage, - getCallSites -} = require('../../dd-trace/src/plugins/util/test') - -const testStartCh = channel('ci:mocha:test:start') -const errorCh = channel('ci:mocha:test:error') -const skipCh = channel('ci:mocha:test:skip') -const testFinishCh = channel('ci:mocha:test:finish') -const parameterizedTestCh = channel('ci:mocha:test:parameterize') - -const itrConfigurationCh = channel('ci:mocha:itr-configuration') -const skippableSuitesCh = channel('ci:mocha:test-suite:skippable') - -const testSessionStartCh = channel('ci:mocha:session:start') -const testSessionFinishCh = channel('ci:mocha:session:finish') - -const testSuiteStartCh = channel('ci:mocha:test-suite:start') -const testSuiteFinishCh = channel('ci:mocha:test-suite:finish') -const testSuiteErrorCh = channel('ci:mocha:test-suite:error') -const testSuiteCodeCoverageCh = channel('ci:mocha:test-suite:code-coverage') - -const itrSkippedSuitesCh = channel('ci:mocha:itr:skipped-suites') - -// TODO: remove when root hooks and fixtures are implemented -const patched = new WeakSet() - -const testToAr = new WeakMap() -const originalFns = new WeakMap() -const testFileToSuiteAr = new Map() -const testToStartLine = new WeakMap() - -// `isWorker` is true if it's a Mocha worker -let isWorker = false - -// We'll preserve the original coverage here -const originalCoverageMap = createCoverageMap() - -let suitesToSkip = [] -let frameworkVersion -let isSuitesSkipped = false -let skippedSuites = [] -const unskippableSuites = [] -let isForcedToRun = false - -function getSuitesByTestFile (root) { - const suitesByTestFile = {} - function getSuites (suite) { - if (suite.file) { - if (suitesByTestFile[suite.file]) { - suitesByTestFile[suite.file].push(suite) - } else { - suitesByTestFile[suite.file] = [suite] - } - } - suite.suites.forEach(suite => { - getSuites(suite) - }) - } - getSuites(root) - - const numSuitesByTestFile = Object.keys(suitesByTestFile).reduce((acc, testFile) => { - acc[testFile] = suitesByTestFile[testFile].length - return acc - }, {}) - - return { suitesByTestFile, numSuitesByTestFile } -} - -function getTestStatus (test) { - if (test.isPending()) { - return 'skip' - } - if (test.isFailed() || test.timedOut) { - return 'fail' - } - return 'pass' -} - -function isRetry (test) { - return test._currentRetry !== undefined && test._currentRetry !== 0 -} - -function getTestAsyncResource (test) { - if (!test.fn) { - return testToAr.get(test) - } - if (!test.fn.asyncResource) { - return testToAr.get(test.fn) - } - const originalFn = originalFns.get(test.fn) - return testToAr.get(originalFn) -} - -function getFilteredSuites (originalSuites) { - return originalSuites.reduce((acc, suite) => { - const testPath = getTestSuitePath(suite.file, process.cwd()) - const shouldSkip = suitesToSkip.includes(testPath) - const isUnskippable = unskippableSuites.includes(suite.file) - if (shouldSkip && !isUnskippable) { - acc.skippedSuites.add(testPath) - } else { - acc.suitesToRun.push(suite) - } - return acc - }, { suitesToRun: [], skippedSuites: new Set() }) +if (process.env.MOCHA_WORKER_ID) { + require('./mocha/worker') +} else { + require('./mocha/main') } -function mochaHook (Runner) { - if (patched.has(Runner)) return Runner - - patched.add(Runner) - - shimmer.wrap(Runner.prototype, 'run', run => function () { - if (!testStartCh.hasSubscribers || isWorker) { - return run.apply(this, arguments) - } - - const { suitesByTestFile, numSuitesByTestFile } = getSuitesByTestFile(this.suite) - - const testRunAsyncResource = new AsyncResource('bound-anonymous-fn') - - this.once('end', testRunAsyncResource.bind(function () { - let status = 'pass' - let error - if (this.stats) { - status = this.stats.failures === 0 ? 'pass' : 'fail' - if (this.stats.tests === 0) { - status = 'skip' - } - } else if (this.failures !== 0) { - status = 'fail' - } - - if (status === 'fail') { - error = new Error(`Failed tests: ${this.failures}.`) - } - - testFileToSuiteAr.clear() - - let testCodeCoverageLinesTotal - if (global.__coverage__) { - try { - testCodeCoverageLinesTotal = originalCoverageMap.getCoverageSummary().lines.pct - } catch (e) { - // ignore errors - } - // restore the original coverage - global.__coverage__ = fromCoverageMapToCoverage(originalCoverageMap) - } - - testSessionFinishCh.publish({ - status, - isSuitesSkipped, - testCodeCoverageLinesTotal, - numSkippedSuites: skippedSuites.length, - hasForcedToRunSuites: isForcedToRun, - hasUnskippableSuites: !!unskippableSuites.length, - error - }) - })) - - this.once('start', testRunAsyncResource.bind(function () { - const processArgv = process.argv.slice(2).join(' ') - const command = `mocha ${processArgv}` - testSessionStartCh.publish({ command, frameworkVersion }) - if (skippedSuites.length) { - itrSkippedSuitesCh.publish({ skippedSuites, frameworkVersion }) - } - })) - - this.on('suite', function (suite) { - if (suite.root || !suite.tests.length) { - return - } - let asyncResource = testFileToSuiteAr.get(suite.file) - if (!asyncResource) { - asyncResource = new AsyncResource('bound-anonymous-fn') - testFileToSuiteAr.set(suite.file, asyncResource) - const isUnskippable = unskippableSuites.includes(suite.file) - isForcedToRun = isUnskippable && suitesToSkip.includes(getTestSuitePath(suite.file, process.cwd())) - asyncResource.runInAsyncScope(() => { - testSuiteStartCh.publish({ testSuite: suite.file, isUnskippable, isForcedToRun }) - }) - } - }) - - this.on('suite end', function (suite) { - if (suite.root) { - return - } - const suitesInTestFile = suitesByTestFile[suite.file] - - const isLastSuite = --numSuitesByTestFile[suite.file] === 0 - if (!isLastSuite) { - return - } - - let status = 'pass' - if (suitesInTestFile.every(suite => suite.pending)) { - status = 'skip' - } else { - // has to check every test in the test file - suitesInTestFile.forEach(suite => { - suite.eachTest(test => { - if (test.state === 'failed' || test.timedOut) { - status = 'fail' - } - }) - }) - } - - if (global.__coverage__) { - const coverageFiles = getCoveredFilenamesFromCoverage(global.__coverage__) - - testSuiteCodeCoverageCh.publish({ - coverageFiles, - suiteFile: suite.file - }) - // We need to reset coverage to get a code coverage per suite - // Before that, we preserve the original coverage - mergeCoverage(global.__coverage__, originalCoverageMap) - resetCoverage(global.__coverage__) - } - - const asyncResource = testFileToSuiteAr.get(suite.file) - asyncResource.runInAsyncScope(() => { - testSuiteFinishCh.publish(status) - }) - }) - - this.on('test', (test) => { - if (isRetry(test)) { - return - } - const testStartLine = testToStartLine.get(test) - const asyncResource = new AsyncResource('bound-anonymous-fn') - testToAr.set(test.fn, asyncResource) - asyncResource.runInAsyncScope(() => { - testStartCh.publish({ test, testStartLine }) - }) - }) - - this.on('test end', (test) => { - const asyncResource = getTestAsyncResource(test) - const status = getTestStatus(test) - - // if there are afterEach to be run, we don't finish the test yet - if (asyncResource && !test.parent._afterEach.length) { - asyncResource.runInAsyncScope(() => { - testFinishCh.publish(status) - }) - } - }) - - // If the hook passes, 'hook end' will be emitted. Otherwise, 'fail' will be emitted - this.on('hook end', (hook) => { - const test = hook.ctx.currentTest - if (test && hook.parent._afterEach.includes(hook)) { // only if it's an afterEach - const isLastAfterEach = hook.parent._afterEach.indexOf(hook) === hook.parent._afterEach.length - 1 - if (isLastAfterEach) { - const status = getTestStatus(test) - const asyncResource = getTestAsyncResource(test) - asyncResource.runInAsyncScope(() => { - testFinishCh.publish(status) - }) - } - } - }) - - this.on('fail', (testOrHook, err) => { - const testFile = testOrHook.file - let test = testOrHook - const isHook = testOrHook.type === 'hook' - if (isHook && testOrHook.ctx) { - test = testOrHook.ctx.currentTest - } - let testAsyncResource - if (test) { - testAsyncResource = getTestAsyncResource(test) - } - if (testAsyncResource) { - testAsyncResource.runInAsyncScope(() => { - if (isHook) { - err.message = `${testOrHook.fullTitle()}: ${err.message}` - errorCh.publish(err) - // if it's a hook and it has failed, 'test end' will not be called - testFinishCh.publish('fail') - } else { - errorCh.publish(err) - } - }) - } - const testSuiteAsyncResource = testFileToSuiteAr.get(testFile) - - if (testSuiteAsyncResource) { - // we propagate the error to the suite - const testSuiteError = new Error( - `"${testOrHook.parent.fullTitle()}" failed with message "${err.message}"` - ) - testSuiteError.stack = err.stack - testSuiteAsyncResource.runInAsyncScope(() => { - testSuiteErrorCh.publish(testSuiteError) - }) - } - }) - - this.on('pending', (test) => { - const asyncResource = getTestAsyncResource(test) - if (asyncResource) { - asyncResource.runInAsyncScope(() => { - skipCh.publish(test) - }) - } else { - // if there is no async resource, the test has been skipped through `test.skip` - // or the parent suite is skipped - const skippedTestAsyncResource = new AsyncResource('bound-anonymous-fn') - if (test.fn) { - testToAr.set(test.fn, skippedTestAsyncResource) - } else { - testToAr.set(test, skippedTestAsyncResource) - } - skippedTestAsyncResource.runInAsyncScope(() => { - skipCh.publish(test) - }) - } - }) - - return run.apply(this, arguments) - }) - - return Runner -} - -function mochaEachHook (mochaEach) { - if (patched.has(mochaEach)) return mochaEach - - patched.add(mochaEach) - - return shimmer.wrap(mochaEach, function () { - const [params] = arguments - const { it, ...rest } = mochaEach.apply(this, arguments) - return { - it: function (name) { - parameterizedTestCh.publish({ name, params }) - it.apply(this, arguments) - }, - ...rest - } - }) -} - -addHook({ - name: 'mocha', - versions: ['>=5.2.0'], - file: 'lib/mocha.js' -}, (Mocha, mochaVersion) => { - frameworkVersion = mochaVersion - const mochaRunAsyncResource = new AsyncResource('bound-anonymous-fn') - /** - * Get ITR configuration and skippable suites - * If ITR is disabled, `onDone` is called immediately on the subscriber - */ - shimmer.wrap(Mocha.prototype, 'run', run => function () { - if (this.options.parallel) { - log.warn(`Unable to initialize CI Visibility because Mocha is running in parallel mode.`) - return run.apply(this, arguments) - } - - if (!itrConfigurationCh.hasSubscribers || this.isWorker) { - if (this.isWorker) { - isWorker = true - } - return run.apply(this, arguments) - } - this.options.delay = true - - const runner = run.apply(this, arguments) - - this.files.forEach(path => { - const isUnskippable = isMarkedAsUnskippable({ path }) - if (isUnskippable) { - unskippableSuites.push(path) - } - }) - - const onReceivedSkippableSuites = ({ err, skippableSuites }) => { - if (err) { - suitesToSkip = [] - } else { - suitesToSkip = skippableSuites - } - // We remove the suites that we skip through ITR - const filteredSuites = getFilteredSuites(runner.suite.suites) - const { suitesToRun } = filteredSuites - - isSuitesSkipped = suitesToRun.length !== runner.suite.suites.length - - log.debug( - () => `${suitesToRun.length} out of ${runner.suite.suites.length} suites are going to run.` - ) - - runner.suite.suites = suitesToRun - - skippedSuites = Array.from(filteredSuites.skippedSuites) - - global.run() - } - - const onReceivedConfiguration = ({ err }) => { - if (err) { - return global.run() - } - if (!skippableSuitesCh.hasSubscribers) { - return global.run() - } - - skippableSuitesCh.publish({ - onDone: mochaRunAsyncResource.bind(onReceivedSkippableSuites) - }) - } - - mochaRunAsyncResource.runInAsyncScope(() => { - itrConfigurationCh.publish({ - onDone: mochaRunAsyncResource.bind(onReceivedConfiguration) - }) - }) - return runner - }) - return Mocha -}) - -addHook({ - name: 'mocha', - versions: ['>=5.2.0'], - file: 'lib/suite.js' -}, (Suite) => { - shimmer.wrap(Suite.prototype, 'addTest', addTest => function (test) { - const callSites = getCallSites() - let startLine - const testCallSite = callSites.find(site => site.getFileName() === test.file) - if (testCallSite) { - startLine = testCallSite.getLineNumber() - testToStartLine.set(test, startLine) - } - return addTest.apply(this, arguments) - }) - return Suite -}) - -addHook({ - name: 'mocha', - versions: ['>=5.2.0'], - file: 'lib/runner.js' -}, mochaHook) - -addHook({ - name: 'mocha', - versions: ['>=5.2.0'], - file: 'lib/cli/run-helpers.js' -}, (run) => { - shimmer.wrap(run, 'runMocha', runMocha => async function () { - if (!testStartCh.hasSubscribers) { - return runMocha.apply(this, arguments) - } - const mocha = arguments[0] - /** - * This attaches `run` to the global context, which we'll call after - * our configuration and skippable suites requests - */ - if (!mocha.options.parallel) { - mocha.options.delay = true - } - return runMocha.apply(this, arguments) - }) - return run -}) - -addHook({ - name: 'mocha', - versions: ['>=5.2.0'], - file: 'lib/runnable.js' -}, (Runnable) => { - shimmer.wrap(Runnable.prototype, 'run', run => function () { - if (!testStartCh.hasSubscribers) { - return run.apply(this, arguments) - } - const isBeforeEach = this.parent._beforeEach.includes(this) - const isAfterEach = this.parent._afterEach.includes(this) - - const isTestHook = isBeforeEach || isAfterEach - - // we restore the original user defined function - if (this.fn.asyncResource) { - const originalFn = originalFns.get(this.fn) - this.fn = originalFn - } - - if (isTestHook || this.type === 'test') { - const test = isTestHook ? this.ctx.currentTest : this - const asyncResource = getTestAsyncResource(test) - - if (asyncResource) { - // we bind the test fn to the correct async resource - const newFn = asyncResource.bind(this.fn) - - // we store the original function, not to lose it - originalFns.set(newFn, this.fn) - this.fn = newFn - - // Temporarily keep functionality when .asyncResource is removed from node - // in https://github.com/nodejs/node/pull/46432 - if (!this.fn.asyncResource) { - this.fn.asyncResource = asyncResource - } - } - } - - return run.apply(this, arguments) - }) - return Runnable -}) - -addHook({ - name: 'mocha-each', - versions: ['>=2.0.1'] -}, mochaEachHook) +// TODO add appropriate calls to wrapFunction whenever we're adding a callback +// wrapper. Right now this is less of an issue since that only has effect in +// SSI, where CI Vis isn't supported. diff --git a/packages/datadog-instrumentations/src/mocha/common.js b/packages/datadog-instrumentations/src/mocha/common.js new file mode 100644 index 00000000000..c25ab2fdb21 --- /dev/null +++ b/packages/datadog-instrumentations/src/mocha/common.js @@ -0,0 +1,48 @@ +const { addHook, channel } = require('../helpers/instrument') +const shimmer = require('../../../datadog-shimmer') +const { getCallSites } = require('../../../dd-trace/src/plugins/util/stacktrace') +const { testToStartLine } = require('./utils') + +const parameterizedTestCh = channel('ci:mocha:test:parameterize') +const patched = new WeakSet() + +// mocha-each support +addHook({ + name: 'mocha-each', + versions: ['>=2.0.1'] +}, mochaEach => { + if (patched.has(mochaEach)) return mochaEach + + patched.add(mochaEach) + + return shimmer.wrapFunction(mochaEach, mochaEach => function () { + const [params] = arguments + const { it, ...rest } = mochaEach.apply(this, arguments) + return { + it: function (title) { + parameterizedTestCh.publish({ title, params }) + it.apply(this, arguments) + }, + ...rest + } + }) +}) + +// support for start line +addHook({ + name: 'mocha', + versions: ['>=5.2.0'], + file: 'lib/suite.js' +}, (Suite) => { + shimmer.wrap(Suite.prototype, 'addTest', addTest => function (test) { + const callSites = getCallSites() + let startLine + const testCallSite = callSites.find(site => site.getFileName() === test.file) + if (testCallSite) { + startLine = testCallSite.getLineNumber() + testToStartLine.set(test, startLine) + } + return addTest.apply(this, arguments) + }) + return Suite +}) diff --git a/packages/datadog-instrumentations/src/mocha/main.js b/packages/datadog-instrumentations/src/mocha/main.js new file mode 100644 index 00000000000..2e796a71371 --- /dev/null +++ b/packages/datadog-instrumentations/src/mocha/main.js @@ -0,0 +1,617 @@ +'use strict' + +const { createCoverageMap } = require('istanbul-lib-coverage') +const { addHook, channel, AsyncResource } = require('../helpers/instrument') +const shimmer = require('../../../datadog-shimmer') +const { isMarkedAsUnskippable } = require('../../../datadog-plugin-jest/src/util') +const log = require('../../../dd-trace/src/log') +const { + getTestSuitePath, + MOCHA_WORKER_TRACE_PAYLOAD_CODE, + fromCoverageMapToCoverage, + getCoveredFilenamesFromCoverage, + mergeCoverage, + resetCoverage, + getIsFaultyEarlyFlakeDetection +} = require('../../../dd-trace/src/plugins/util/test') + +const { + isNewTest, + getSuitesByTestFile, + runnableWrapper, + getOnTestHandler, + getOnTestEndHandler, + getOnTestRetryHandler, + getOnHookEndHandler, + getOnFailHandler, + getOnPendingHandler, + testFileToSuiteAr, + newTests, + getTestFullName, + getRunTestsWrapper +} = require('./utils') + +require('./common') + +const testSessionAsyncResource = new AsyncResource('bound-anonymous-fn') +const patched = new WeakSet() + +const unskippableSuites = [] +let suitesToSkip = [] +let isSuitesSkipped = false +let skippedSuites = [] +let itrCorrelationId = '' +let isForcedToRun = false +const config = {} + +// We'll preserve the original coverage here +const originalCoverageMap = createCoverageMap() +let untestedCoverage + +// test channels +const testStartCh = channel('ci:mocha:test:start') + +// test suite channels +const testSuiteStartCh = channel('ci:mocha:test-suite:start') +const testSuiteFinishCh = channel('ci:mocha:test-suite:finish') +const testSuiteErrorCh = channel('ci:mocha:test-suite:error') +const testSuiteCodeCoverageCh = channel('ci:mocha:test-suite:code-coverage') + +// session channels +const libraryConfigurationCh = channel('ci:mocha:library-configuration') +const knownTestsCh = channel('ci:mocha:known-tests') +const skippableSuitesCh = channel('ci:mocha:test-suite:skippable') +const workerReportTraceCh = channel('ci:mocha:worker-report:trace') +const testSessionStartCh = channel('ci:mocha:session:start') +const testSessionFinishCh = channel('ci:mocha:session:finish') +const itrSkippedSuitesCh = channel('ci:mocha:itr:skipped-suites') + +const getCodeCoverageCh = channel('ci:nyc:get-coverage') + +// Tests from workers do not come with `isFailed` method +function isTestFailed (test) { + if (test.isFailed) { + return test.isFailed() + } + if (test.isPending) { + return !test.isPending() && test.state !== 'failed' + } + return false +} + +function getFilteredSuites (originalSuites) { + return originalSuites.reduce((acc, suite) => { + const testPath = getTestSuitePath(suite.file, process.cwd()) + const shouldSkip = suitesToSkip.includes(testPath) + const isUnskippable = unskippableSuites.includes(suite.file) + if (shouldSkip && !isUnskippable) { + acc.skippedSuites.add(testPath) + } else { + acc.suitesToRun.push(suite) + } + return acc + }, { suitesToRun: [], skippedSuites: new Set() }) +} + +function getOnStartHandler (isParallel, frameworkVersion) { + return testSessionAsyncResource.bind(function () { + const processArgv = process.argv.slice(2).join(' ') + const command = `mocha ${processArgv}` + testSessionStartCh.publish({ command, frameworkVersion }) + if (!isParallel && skippedSuites.length) { + itrSkippedSuitesCh.publish({ skippedSuites, frameworkVersion }) + } + }) +} + +function getOnEndHandler (isParallel) { + return testSessionAsyncResource.bind(function () { + let status = 'pass' + let error + if (this.stats) { + status = this.stats.failures === 0 ? 'pass' : 'fail' + if (this.stats.tests === 0) { + status = 'skip' + } + } else if (this.failures !== 0) { + status = 'fail' + } + + if (config.isEarlyFlakeDetectionEnabled) { + /** + * If Early Flake Detection (EFD) is enabled the logic is as follows: + * - If all attempts for a test are failing, the test has failed and we will let the test process fail. + * - If just a single attempt passes, we will prevent the test process from failing. + * The rationale behind is the following: you may still be able to block your CI pipeline by gating + * on flakiness (the test will be considered flaky), but you may choose to unblock the pipeline too. + */ + for (const tests of Object.values(newTests)) { + const failingNewTests = tests.filter(test => isTestFailed(test)) + const areAllNewTestsFailing = failingNewTests.length === tests.length + if (failingNewTests.length && !areAllNewTestsFailing) { + this.stats.failures -= failingNewTests.length + this.failures -= failingNewTests.length + } + } + } + + if (status === 'fail') { + error = new Error(`Failed tests: ${this.failures}.`) + } + + testFileToSuiteAr.clear() + + let testCodeCoverageLinesTotal + if (global.__coverage__) { + try { + if (untestedCoverage) { + originalCoverageMap.merge(fromCoverageMapToCoverage(untestedCoverage)) + } + testCodeCoverageLinesTotal = originalCoverageMap.getCoverageSummary().lines.pct + } catch (e) { + // ignore errors + } + // restore the original coverage + global.__coverage__ = fromCoverageMapToCoverage(originalCoverageMap) + } + + testSessionFinishCh.publish({ + status, + isSuitesSkipped, + testCodeCoverageLinesTotal, + numSkippedSuites: skippedSuites.length, + hasForcedToRunSuites: isForcedToRun, + hasUnskippableSuites: !!unskippableSuites.length, + error, + isEarlyFlakeDetectionEnabled: config.isEarlyFlakeDetectionEnabled, + isEarlyFlakeDetectionFaulty: config.isEarlyFlakeDetectionFaulty, + isParallel + }) + }) +} + +function getExecutionConfiguration (runner, isParallel, onFinishRequest) { + const mochaRunAsyncResource = new AsyncResource('bound-anonymous-fn') + + const onReceivedSkippableSuites = ({ err, skippableSuites, itrCorrelationId: responseItrCorrelationId }) => { + if (err) { + suitesToSkip = [] + } else { + suitesToSkip = skippableSuites + itrCorrelationId = responseItrCorrelationId + } + // We remove the suites that we skip through ITR + const filteredSuites = getFilteredSuites(runner.suite.suites) + const { suitesToRun } = filteredSuites + + isSuitesSkipped = suitesToRun.length !== runner.suite.suites.length + + log.debug( + () => `${suitesToRun.length} out of ${runner.suite.suites.length} suites are going to run.` + ) + + runner.suite.suites = suitesToRun + + skippedSuites = Array.from(filteredSuites.skippedSuites) + + onFinishRequest() + } + + const onReceivedKnownTests = ({ err, knownTests }) => { + if (err) { + config.knownTests = [] + config.isEarlyFlakeDetectionEnabled = false + } else { + config.knownTests = knownTests + } + + if (config.isSuitesSkippingEnabled) { + skippableSuitesCh.publish({ + onDone: mochaRunAsyncResource.bind(onReceivedSkippableSuites) + }) + } else { + onFinishRequest() + } + } + + const onReceivedConfiguration = ({ err, libraryConfig }) => { + if (err || !skippableSuitesCh.hasSubscribers || !knownTestsCh.hasSubscribers) { + return onFinishRequest() + } + + config.isEarlyFlakeDetectionEnabled = libraryConfig.isEarlyFlakeDetectionEnabled + config.earlyFlakeDetectionNumRetries = libraryConfig.earlyFlakeDetectionNumRetries + config.earlyFlakeDetectionFaultyThreshold = libraryConfig.earlyFlakeDetectionFaultyThreshold + // ITR and auto test retries are not supported in parallel mode yet + config.isSuitesSkippingEnabled = !isParallel && libraryConfig.isSuitesSkippingEnabled + config.isFlakyTestRetriesEnabled = !isParallel && libraryConfig.isFlakyTestRetriesEnabled + config.flakyTestRetriesCount = !isParallel && libraryConfig.flakyTestRetriesCount + + if (config.isEarlyFlakeDetectionEnabled) { + knownTestsCh.publish({ + onDone: mochaRunAsyncResource.bind(onReceivedKnownTests) + }) + } else if (config.isSuitesSkippingEnabled) { + skippableSuitesCh.publish({ + onDone: mochaRunAsyncResource.bind(onReceivedSkippableSuites) + }) + } else { + onFinishRequest() + } + } + + libraryConfigurationCh.publish({ + onDone: mochaRunAsyncResource.bind(onReceivedConfiguration) + }) +} + +// In this hook we delay the execution with options.delay to grab library configuration, +// skippable and known tests. +// It is called but skipped in parallel mode. +addHook({ + name: 'mocha', + versions: ['>=5.2.0'], + file: 'lib/mocha.js' +}, (Mocha) => { + shimmer.wrap(Mocha.prototype, 'run', run => function () { + // Workers do not need to request any data, just run the tests + if (!testStartCh.hasSubscribers || process.env.MOCHA_WORKER_ID || this.options.parallel) { + return run.apply(this, arguments) + } + + // `options.delay` does not work in parallel mode, so we can't delay the execution this way + // This needs to be both here and in `runMocha` hook. Read the comment in `runMocha` hook for more info. + this.options.delay = true + + const runner = run.apply(this, arguments) + + this.files.forEach(path => { + const isUnskippable = isMarkedAsUnskippable({ path }) + if (isUnskippable) { + unskippableSuites.push(path) + } + }) + + getExecutionConfiguration(runner, false, () => { + if (config.isEarlyFlakeDetectionEnabled) { + const testSuites = this.files.map(file => getTestSuitePath(file, process.cwd())) + const isFaulty = getIsFaultyEarlyFlakeDetection( + testSuites, + config.knownTests?.mocha || {}, + config.earlyFlakeDetectionFaultyThreshold + ) + if (isFaulty) { + config.isEarlyFlakeDetectionEnabled = false + config.isEarlyFlakeDetectionFaulty = true + } + } + if (getCodeCoverageCh.hasSubscribers) { + getCodeCoverageCh.publish({ + onDone: (receivedCodeCoverage) => { + untestedCoverage = receivedCodeCoverage + global.run() + } + }) + } else { + global.run() + } + }) + + return runner + }) + return Mocha +}) + +addHook({ + name: 'mocha', + versions: ['>=5.2.0'], + file: 'lib/cli/run-helpers.js' +}, (run) => { + shimmer.wrap(run, 'runMocha', runMocha => async function () { + if (!testStartCh.hasSubscribers) { + return runMocha.apply(this, arguments) + } + const mocha = arguments[0] + + /** + * This attaches `run` to the global context, which we'll call after + * our configuration and skippable suites requests. + * You need this both here and in Mocha#run hook: the programmatic API + * does not call `runMocha`, so it needs to be in Mocha#run. When using + * the CLI, modifying `options.delay` in Mocha#run is not enough (it's too late), + * so it also needs to be here. + */ + if (!mocha.options.parallel) { + mocha.options.delay = true + } + + return runMocha.apply(this, arguments) + }) + return run +}) + +// Only used in serial mode (no --parallel flag is passed) +// This hook is used to generate session, module, suite and test events +addHook({ + name: 'mocha', + versions: ['>=5.2.0'], + file: 'lib/runner.js' +}, function (Runner, frameworkVersion) { + if (patched.has(Runner)) return Runner + + patched.add(Runner) + + shimmer.wrap(Runner.prototype, 'runTests', runTests => getRunTestsWrapper(runTests, config)) + + shimmer.wrap(Runner.prototype, 'run', run => function () { + if (!testStartCh.hasSubscribers) { + return run.apply(this, arguments) + } + + const { suitesByTestFile, numSuitesByTestFile } = getSuitesByTestFile(this.suite) + + this.once('start', getOnStartHandler(false, frameworkVersion)) + + this.once('end', getOnEndHandler(false)) + + this.on('test', getOnTestHandler(true, newTests)) + + this.on('test end', getOnTestEndHandler()) + + this.on('retry', getOnTestRetryHandler()) + + // If the hook passes, 'hook end' will be emitted. Otherwise, 'fail' will be emitted + this.on('hook end', getOnHookEndHandler()) + + this.on('fail', getOnFailHandler(true)) + + this.on('pending', getOnPendingHandler()) + + this.on('suite', function (suite) { + if (suite.root || !suite.tests.length) { + return + } + let asyncResource = testFileToSuiteAr.get(suite.file) + if (!asyncResource) { + asyncResource = new AsyncResource('bound-anonymous-fn') + testFileToSuiteAr.set(suite.file, asyncResource) + const isUnskippable = unskippableSuites.includes(suite.file) + isForcedToRun = isUnskippable && suitesToSkip.includes(getTestSuitePath(suite.file, process.cwd())) + asyncResource.runInAsyncScope(() => { + testSuiteStartCh.publish({ + testSuiteAbsolutePath: suite.file, + isUnskippable, + isForcedToRun, + itrCorrelationId + }) + }) + } + }) + + this.on('suite end', function (suite) { + if (suite.root) { + return + } + const suitesInTestFile = suitesByTestFile[suite.file] + + const isLastSuite = --numSuitesByTestFile[suite.file] === 0 + if (!isLastSuite) { + return + } + + let status = 'pass' + if (suitesInTestFile.every(suite => suite.pending)) { + status = 'skip' + } else { + // has to check every test in the test file + suitesInTestFile.forEach(suite => { + suite.eachTest(test => { + if (test.state === 'failed' || test.timedOut) { + status = 'fail' + } + }) + }) + } + + if (global.__coverage__) { + const coverageFiles = getCoveredFilenamesFromCoverage(global.__coverage__) + + testSuiteCodeCoverageCh.publish({ + coverageFiles, + suiteFile: suite.file + }) + // We need to reset coverage to get a code coverage per suite + // Before that, we preserve the original coverage + mergeCoverage(global.__coverage__, originalCoverageMap) + resetCoverage(global.__coverage__) + } + + const asyncResource = testFileToSuiteAr.get(suite.file) + if (asyncResource) { + asyncResource.runInAsyncScope(() => { + testSuiteFinishCh.publish(status) + }) + } else { + log.warn(() => `No AsyncResource found for suite ${suite.file}`) + } + }) + + return run.apply(this, arguments) + }) + + return Runner +}) + +// Used both in serial and parallel mode, and by both the main process and the workers +// Used to set the correct async resource to the test. +addHook({ + name: 'mocha', + versions: ['>=5.2.0'], + file: 'lib/runnable.js' +}, (runnablePackage) => runnableWrapper(runnablePackage, config)) + +// Only used in parallel mode (--parallel flag is passed) +// Used to generate suite events and receive test payloads from workers +addHook({ + name: 'workerpool', + // mocha@8.0.0 added parallel support and uses workerpool for it + // The version they use is 6.0.0: + // https://github.com/mochajs/mocha/blob/612fa31228c695f16173ac675f40ccdf26b4cfb5/package.json#L75 + versions: ['>=6.0.0'], + file: 'src/WorkerHandler.js' +}, (workerHandlerPackage) => { + shimmer.wrap(workerHandlerPackage.prototype, 'exec', exec => function (_, path) { + if (!testStartCh.hasSubscribers) { + return exec.apply(this, arguments) + } + if (!path?.length) { + return exec.apply(this, arguments) + } + const [testSuiteAbsolutePath] = path + const testSuiteAsyncResource = new AsyncResource('bound-anonymous-fn') + + function onMessage (message) { + if (Array.isArray(message)) { + const [messageCode, payload] = message + if (messageCode === MOCHA_WORKER_TRACE_PAYLOAD_CODE) { + testSuiteAsyncResource.runInAsyncScope(() => { + workerReportTraceCh.publish(payload) + }) + } + } + } + + this.worker.on('message', onMessage) + + testSuiteAsyncResource.runInAsyncScope(() => { + testSuiteStartCh.publish({ + testSuiteAbsolutePath + }) + }) + + try { + const promise = exec.apply(this, arguments) + promise.then( + (result) => { + const status = result.failureCount === 0 ? 'pass' : 'fail' + testSuiteAsyncResource.runInAsyncScope(() => { + testSuiteFinishCh.publish(status) + }) + this.worker.off('message', onMessage) + }, + (err) => { + testSuiteAsyncResource.runInAsyncScope(() => { + testSuiteErrorCh.publish(err) + testSuiteFinishCh.publish('fail') + }) + this.worker.off('message', onMessage) + } + ) + return promise + } catch (err) { + testSuiteAsyncResource.runInAsyncScope(() => { + testSuiteErrorCh.publish(err) + testSuiteFinishCh.publish('fail') + }) + this.worker.off('message', onMessage) + throw err + } + }) + + return workerHandlerPackage +}) + +// Only used in parallel mode (--parallel flag is passed) +// Used to start and finish test session and test module +addHook({ + name: 'mocha', + versions: ['>=8.0.0'], + file: 'lib/nodejs/parallel-buffered-runner.js' +}, (ParallelBufferedRunner, frameworkVersion) => { + shimmer.wrap(ParallelBufferedRunner.prototype, 'run', run => function (cb, { files }) { + if (!testStartCh.hasSubscribers) { + return run.apply(this, arguments) + } + + this.once('start', getOnStartHandler(true, frameworkVersion)) + this.once('end', getOnEndHandler(true)) + + getExecutionConfiguration(this, true, () => { + if (config.isEarlyFlakeDetectionEnabled) { + const testSuites = files.map(file => getTestSuitePath(file, process.cwd())) + const isFaulty = getIsFaultyEarlyFlakeDetection( + testSuites, + config.knownTests?.mocha || {}, + config.earlyFlakeDetectionFaultyThreshold + ) + if (isFaulty) { + config.isEarlyFlakeDetectionEnabled = false + config.isEarlyFlakeDetectionFaulty = true + } + } + run.apply(this, arguments) + }) + + return this + }) + + return ParallelBufferedRunner +}) + +// Only in parallel mode: BufferedWorkerPool#run is used to run a test file in a worker +// If Early Flake Detection is enabled, +// In this hook we pass the known tests to the worker and collect the new tests that run +addHook({ + name: 'mocha', + versions: ['>=8.0.0'], + file: 'lib/nodejs/buffered-worker-pool.js' +}, (BufferedWorkerPoolPackage) => { + const { BufferedWorkerPool } = BufferedWorkerPoolPackage + + shimmer.wrap(BufferedWorkerPool.prototype, 'run', run => async function (testSuiteAbsolutePath, workerArgs) { + if (!testStartCh.hasSubscribers || !config.isEarlyFlakeDetectionEnabled) { + return run.apply(this, arguments) + } + + const testPath = getTestSuitePath(testSuiteAbsolutePath, process.cwd()) + const testSuiteKnownTests = config.knownTests.mocha?.[testPath] || [] + + // We pass the known tests for the test file to the worker + const testFileResult = await run.apply( + this, + [ + testSuiteAbsolutePath, + { + ...workerArgs, + _ddEfdNumRetries: config.earlyFlakeDetectionNumRetries, + _ddKnownTests: { + mocha: { + [testPath]: testSuiteKnownTests + } + } + } + ] + ) + const tests = testFileResult + .events + .filter(event => event.eventName === 'test end') + .map(event => event.data) + + // `newTests` is filled in the worker process, so we need to use the test results to fill it here too. + for (const test of tests) { + if (isNewTest(test, config.knownTests)) { + const testFullName = getTestFullName(test) + const tests = newTests[testFullName] + + if (!tests) { + newTests[testFullName] = [test] + } else { + tests.push(test) + } + } + } + return testFileResult + }) + + return BufferedWorkerPoolPackage +}) diff --git a/packages/datadog-instrumentations/src/mocha/utils.js b/packages/datadog-instrumentations/src/mocha/utils.js new file mode 100644 index 00000000000..2b51fd6e73b --- /dev/null +++ b/packages/datadog-instrumentations/src/mocha/utils.js @@ -0,0 +1,369 @@ +'use strict' + +const { + getTestSuitePath, + removeEfdStringFromTestName, + addEfdStringToTestName +} = require('../../../dd-trace/src/plugins/util/test') +const { channel, AsyncResource } = require('../helpers/instrument') +const shimmer = require('../../../datadog-shimmer') + +// test channels +const testStartCh = channel('ci:mocha:test:start') +const testFinishCh = channel('ci:mocha:test:finish') +// after a test has failed, we'll publish to this channel +const testRetryCh = channel('ci:mocha:test:retry') +const errorCh = channel('ci:mocha:test:error') +const skipCh = channel('ci:mocha:test:skip') + +// suite channels +const testSuiteErrorCh = channel('ci:mocha:test-suite:error') + +const testToAr = new WeakMap() +const originalFns = new WeakMap() +const testToStartLine = new WeakMap() +const testFileToSuiteAr = new Map() +const wrappedFunctions = new WeakSet() +const newTests = {} + +function isNewTest (test, knownTests) { + const testSuite = getTestSuitePath(test.file, process.cwd()) + const testName = removeEfdStringFromTestName(test.fullTitle()) + const testsForSuite = knownTests.mocha?.[testSuite] || [] + return !testsForSuite.includes(testName) +} + +function retryTest (test, earlyFlakeDetectionNumRetries) { + const originalTestName = test.title + const suite = test.parent + for (let retryIndex = 0; retryIndex < earlyFlakeDetectionNumRetries; retryIndex++) { + const clonedTest = test.clone() + clonedTest.title = addEfdStringToTestName(originalTestName, retryIndex + 1) + suite.addTest(clonedTest) + clonedTest._ddIsNew = true + clonedTest._ddIsEfdRetry = true + } +} + +function getSuitesByTestFile (root) { + const suitesByTestFile = {} + function getSuites (suite) { + if (suite.file) { + if (suitesByTestFile[suite.file]) { + suitesByTestFile[suite.file].push(suite) + } else { + suitesByTestFile[suite.file] = [suite] + } + } + suite.suites.forEach(suite => { + getSuites(suite) + }) + } + getSuites(root) + + const numSuitesByTestFile = Object.keys(suitesByTestFile).reduce((acc, testFile) => { + acc[testFile] = suitesByTestFile[testFile].length + return acc + }, {}) + + return { suitesByTestFile, numSuitesByTestFile } +} + +function isMochaRetry (test) { + return test._currentRetry !== undefined && test._currentRetry !== 0 +} + +function isLastRetry (test) { + return test._currentRetry === test._retries +} + +function getTestFullName (test) { + return `mocha.${getTestSuitePath(test.file, process.cwd())}.${removeEfdStringFromTestName(test.fullTitle())}` +} + +function getTestStatus (test) { + if (test.isPending()) { + return 'skip' + } + if (test.isFailed() || test.timedOut) { + return 'fail' + } + return 'pass' +} + +function getTestToArKey (test) { + if (!test.fn) { + return test + } + if (!wrappedFunctions.has(test.fn)) { + return test.fn + } + const originalFn = originalFns.get(test.fn) + return originalFn +} + +function getTestAsyncResource (test) { + const key = getTestToArKey(test) + return testToAr.get(key) +} + +function runnableWrapper (RunnablePackage, libraryConfig) { + shimmer.wrap(RunnablePackage.prototype, 'run', run => function () { + if (!testStartCh.hasSubscribers) { + return run.apply(this, arguments) + } + // Flaky test retries does not work in parallel mode + if (libraryConfig?.isFlakyTestRetriesEnabled) { + this.retries(libraryConfig?.flakyTestRetriesCount) + } + // The reason why the wrapping logic is here is because we need to cover + // `afterEach` and `beforeEach` hooks as well. + // It can't be done in `getOnTestHandler` because it's only called for tests. + const isBeforeEach = this.parent._beforeEach.includes(this) + const isAfterEach = this.parent._afterEach.includes(this) + + const isTestHook = isBeforeEach || isAfterEach + + // we restore the original user defined function + if (wrappedFunctions.has(this.fn)) { + const originalFn = originalFns.get(this.fn) + this.fn = originalFn + wrappedFunctions.delete(this.fn) + } + + if (isTestHook || this.type === 'test') { + const test = isTestHook ? this.ctx.currentTest : this + const asyncResource = getTestAsyncResource(test) + + if (asyncResource) { + // we bind the test fn to the correct async resource + const newFn = asyncResource.bind(this.fn) + + // we store the original function, not to lose it + originalFns.set(newFn, this.fn) + this.fn = newFn + + wrappedFunctions.add(this.fn) + } + } + + return run.apply(this, arguments) + }) + return RunnablePackage +} + +function getOnTestHandler (isMain) { + return function (test) { + const testStartLine = testToStartLine.get(test) + const asyncResource = new AsyncResource('bound-anonymous-fn') + + // This may be a retry. If this is the case, `test.fn` is already wrapped, + // so we need to restore it. + if (wrappedFunctions.has(test.fn)) { + const originalFn = originalFns.get(test.fn) + test.fn = originalFn + wrappedFunctions.delete(test.fn) + } + testToAr.set(test.fn, asyncResource) + + const { + file: testSuiteAbsolutePath, + title, + _ddIsNew: isNew, + _ddIsEfdRetry: isEfdRetry + } = test + + const testInfo = { + testName: test.fullTitle(), + testSuiteAbsolutePath, + title, + testStartLine + } + + if (!isMain) { + testInfo.isParallel = true + } + + testInfo.isNew = isNew + testInfo.isEfdRetry = isEfdRetry + // We want to store the result of the new tests + if (isNew) { + const testFullName = getTestFullName(test) + if (newTests[testFullName]) { + newTests[testFullName].push(test) + } else { + newTests[testFullName] = [test] + } + } + + asyncResource.runInAsyncScope(() => { + testStartCh.publish(testInfo) + }) + } +} + +function getOnTestEndHandler () { + return function (test) { + const asyncResource = getTestAsyncResource(test) + const status = getTestStatus(test) + + // if there are afterEach to be run, we don't finish the test yet + if (asyncResource && !test.parent._afterEach.length) { + asyncResource.runInAsyncScope(() => { + testFinishCh.publish({ status, hasBeenRetried: isMochaRetry(test) }) + }) + } + } +} + +function getOnHookEndHandler () { + return function (hook) { + const test = hook.ctx.currentTest + if (test && hook.parent._afterEach.includes(hook)) { // only if it's an afterEach + const isLastAfterEach = hook.parent._afterEach.indexOf(hook) === hook.parent._afterEach.length - 1 + if (test._retries > 0 && !isLastRetry(test)) { + return + } + if (isLastAfterEach) { + const status = getTestStatus(test) + const asyncResource = getTestAsyncResource(test) + if (asyncResource) { + asyncResource.runInAsyncScope(() => { + testFinishCh.publish({ status, hasBeenRetried: isMochaRetry(test) }) + }) + } + } + } + } +} + +function getOnFailHandler (isMain) { + return function (testOrHook, err) { + const testFile = testOrHook.file + let test = testOrHook + const isHook = testOrHook.type === 'hook' + if (isHook && testOrHook.ctx) { + test = testOrHook.ctx.currentTest + } + let testAsyncResource + if (test) { + testAsyncResource = getTestAsyncResource(test) + } + if (testAsyncResource) { + testAsyncResource.runInAsyncScope(() => { + if (isHook) { + err.message = `${testOrHook.fullTitle()}: ${err.message}` + errorCh.publish(err) + // if it's a hook and it has failed, 'test end' will not be called + testFinishCh.publish({ status: 'fail', hasBeenRetried: isMochaRetry(test) }) + } else { + errorCh.publish(err) + } + }) + } + + if (isMain) { + const testSuiteAsyncResource = testFileToSuiteAr.get(testFile) + + if (testSuiteAsyncResource) { + // we propagate the error to the suite + const testSuiteError = new Error( + `"${testOrHook.parent.fullTitle()}" failed with message "${err.message}"` + ) + testSuiteError.stack = err.stack + testSuiteAsyncResource.runInAsyncScope(() => { + testSuiteErrorCh.publish(testSuiteError) + }) + } + } + } +} + +function getOnTestRetryHandler () { + return function (test, err) { + const asyncResource = getTestAsyncResource(test) + if (asyncResource) { + const isFirstAttempt = test._currentRetry === 0 + asyncResource.runInAsyncScope(() => { + testRetryCh.publish({ isFirstAttempt, err }) + }) + } + const key = getTestToArKey(test) + testToAr.delete(key) + } +} + +function getOnPendingHandler () { + return function (test) { + const testStartLine = testToStartLine.get(test) + const { + file: testSuiteAbsolutePath, + title + } = test + + const testInfo = { + testName: test.fullTitle(), + testSuiteAbsolutePath, + title, + testStartLine + } + + const asyncResource = getTestAsyncResource(test) + if (asyncResource) { + asyncResource.runInAsyncScope(() => { + skipCh.publish(testInfo) + }) + } else { + // if there is no async resource, the test has been skipped through `test.skip` + // or the parent suite is skipped + const skippedTestAsyncResource = new AsyncResource('bound-anonymous-fn') + if (test.fn) { + testToAr.set(test.fn, skippedTestAsyncResource) + } else { + testToAr.set(test, skippedTestAsyncResource) + } + skippedTestAsyncResource.runInAsyncScope(() => { + skipCh.publish(testInfo) + }) + } + } +} + +// Hook to add retries to tests if EFD is enabled +function getRunTestsWrapper (runTests, config) { + return function (suite, fn) { + if (config.isEarlyFlakeDetectionEnabled) { + // by the time we reach `this.on('test')`, it is too late. We need to add retries here + suite.tests.forEach(test => { + if (!test.isPending() && isNewTest(test, config.knownTests)) { + test._ddIsNew = true + retryTest(test, config.earlyFlakeDetectionNumRetries) + } + }) + } + return runTests.apply(this, arguments) + } +} + +module.exports = { + isNewTest, + retryTest, + getSuitesByTestFile, + isMochaRetry, + getTestFullName, + getTestStatus, + runnableWrapper, + testToAr, + originalFns, + getTestAsyncResource, + testToStartLine, + getOnTestHandler, + getOnTestEndHandler, + getOnTestRetryHandler, + getOnHookEndHandler, + getOnFailHandler, + getOnPendingHandler, + testFileToSuiteAr, + getRunTestsWrapper, + newTests +} diff --git a/packages/datadog-instrumentations/src/mocha/worker.js b/packages/datadog-instrumentations/src/mocha/worker.js new file mode 100644 index 00000000000..63670ba5db2 --- /dev/null +++ b/packages/datadog-instrumentations/src/mocha/worker.js @@ -0,0 +1,80 @@ +'use strict' + +const { addHook, channel } = require('../helpers/instrument') +const shimmer = require('../../../datadog-shimmer') + +const { + runnableWrapper, + getOnTestHandler, + getOnTestEndHandler, + getOnHookEndHandler, + getOnFailHandler, + getOnPendingHandler, + getRunTestsWrapper +} = require('./utils') +require('./common') + +const workerFinishCh = channel('ci:mocha:worker:finish') + +const config = {} + +addHook({ + name: 'mocha', + versions: ['>=8.0.0'], + file: 'lib/mocha.js' +}, (Mocha) => { + shimmer.wrap(Mocha.prototype, 'run', run => function () { + if (this.options._ddKnownTests) { + // EFD is enabled if there's a list of known tests + config.isEarlyFlakeDetectionEnabled = true + config.knownTests = this.options._ddKnownTests + config.earlyFlakeDetectionNumRetries = this.options._ddEfdNumRetries + delete this.options._ddKnownTests + delete this.options._ddEfdNumRetries + } + return run.apply(this, arguments) + }) + + return Mocha +}) + +// Runner is also hooked in mocha/main.js, but in here we only generate test events. +addHook({ + name: 'mocha', + versions: ['>=5.2.0'], + file: 'lib/runner.js' +}, function (Runner) { + shimmer.wrap(Runner.prototype, 'runTests', runTests => getRunTestsWrapper(runTests, config)) + + shimmer.wrap(Runner.prototype, 'run', run => function () { + if (!workerFinishCh.hasSubscribers) { + return run.apply(this, arguments) + } + // We flush when the worker ends with its test file (a mocha instance in a worker runs a single test file) + this.on('end', () => { + workerFinishCh.publish() + }) + this.on('test', getOnTestHandler(false)) + + this.on('test end', getOnTestEndHandler()) + + // If the hook passes, 'hook end' will be emitted. Otherwise, 'fail' will be emitted + this.on('hook end', getOnHookEndHandler()) + + this.on('fail', getOnFailHandler(false)) + + this.on('pending', getOnPendingHandler()) + + return run.apply(this, arguments) + }) + return Runner +}) + +// Used both in serial and parallel mode, and by both the main process and the workers +// Used to set the correct async resource to the test. +addHook({ + name: 'mocha', + versions: ['>=5.2.0'], + file: 'lib/runnable.js' +}, runnableWrapper) +// TODO: parallel mode does not support flaky test retries, so no library config is passed. diff --git a/packages/datadog-instrumentations/src/moleculer/server.js b/packages/datadog-instrumentations/src/moleculer/server.js index 5a586f777b7..425d1cf05f4 100644 --- a/packages/datadog-instrumentations/src/moleculer/server.js +++ b/packages/datadog-instrumentations/src/moleculer/server.js @@ -24,7 +24,7 @@ function createMiddleware () { localAction (next, action) { const broker = this - return function datadogMiddleware (ctx) { + return shimmer.wrapFunction(next, next => function datadogMiddleware (ctx) { const actionResource = new AsyncResource('bound-anonymous-fn') return actionResource.runInAsyncScope(() => { @@ -47,7 +47,7 @@ function createMiddleware () { finishChannel.publish() } }) - } + }) } } } diff --git a/packages/datadog-instrumentations/src/mongodb-core.js b/packages/datadog-instrumentations/src/mongodb-core.js index 1e8be322189..0441d95d5f0 100644 --- a/packages/datadog-instrumentations/src/mongodb-core.js +++ b/packages/datadog-instrumentations/src/mongodb-core.js @@ -7,9 +7,9 @@ const { } = require('./helpers/instrument') const shimmer = require('../../datadog-shimmer') -const startCh = channel(`apm:mongodb:query:start`) -const finishCh = channel(`apm:mongodb:query:finish`) -const errorCh = channel(`apm:mongodb:query:error`) +const startCh = channel('apm:mongodb:query:start') +const finishCh = channel('apm:mongodb:query:finish') +const errorCh = channel('apm:mongodb:query:error') addHook({ name: 'mongodb-core', versions: ['2 - 3.1.9'] }, Server => { const serverProto = Server.Server.prototype @@ -32,12 +32,18 @@ addHook({ name: 'mongodb', versions: ['>=4 <4.6.0'], file: 'lib/cmap/connection. return Connection }) -addHook({ name: 'mongodb', versions: ['>=4.6.0'], file: 'lib/cmap/connection.js' }, Connection => { +addHook({ name: 'mongodb', versions: ['>=4.6.0 <6.4.0'], file: 'lib/cmap/connection.js' }, Connection => { const proto = Connection.Connection.prototype shimmer.wrap(proto, 'command', command => wrapConnectionCommand(command, 'command')) return Connection }) +addHook({ name: 'mongodb', versions: ['>=6.4.0'], file: 'lib/cmap/connection.js' }, Connection => { + const proto = Connection.Connection.prototype + shimmer.wrap(proto, 'command', command => wrapConnectionCommand(command, 'command', undefined, instrumentPromise)) + return Connection +}) + addHook({ name: 'mongodb', versions: ['>=3.3 <4'], file: 'lib/core/wireprotocol/index.js' }, wp => wrapWp(wp)) addHook({ name: 'mongodb-core', versions: ['>=3.2'], file: 'lib/wireprotocol/index.js' }, wp => wrapWp(wp)) @@ -86,10 +92,10 @@ function wrapUnifiedCommand (command, operation, name) { } return instrument(operation, command, this, arguments, server, ns, ops, { name }) } - return shimmer.wrap(command, wrapped) + return wrapped } -function wrapConnectionCommand (command, operation, name) { +function wrapConnectionCommand (command, operation, name, instrumentFn = instrument) { const wrapped = function (ns, ops) { if (!startCh.hasSubscribers) { return command.apply(this, arguments) @@ -101,9 +107,9 @@ function wrapConnectionCommand (command, operation, name) { const topology = { s: { options } } ns = `${ns.db}.${ns.collection}` - return instrument(operation, command, this, arguments, topology, ns, ops, { name }) + return instrumentFn(operation, command, this, arguments, topology, ns, ops, { name }) } - return shimmer.wrap(command, wrapped) + return wrapped } function wrapQuery (query, operation, name) { @@ -117,7 +123,7 @@ function wrapQuery (query, operation, name) { return instrument(operation, query, this, arguments, pool, ns, ops) } - return shimmer.wrap(query, wrapped) + return wrapped } function wrapCursor (cursor, operation, name) { @@ -129,7 +135,7 @@ function wrapCursor (cursor, operation, name) { const ns = this.ns return instrument(operation, cursor, this, arguments, pool, ns, {}, { name }) } - return shimmer.wrap(cursor, wrapped) + return wrapped } function wrapCommand (command, operation, name) { @@ -139,7 +145,7 @@ function wrapCommand (command, operation, name) { } return instrument(operation, command, this, arguments, this, ns, ops, { name }) } - return shimmer.wrap(command, wrapped) + return wrapped } function instrument (operation, command, ctx, args, server, ns, ops, options = {}) { @@ -158,7 +164,7 @@ function instrument (operation, command, ctx, args, server, ns, ops, options = { return asyncResource.runInAsyncScope(() => { startCh.publish({ ns, ops, options: serverInfo, name }) - args[index] = asyncResource.bind(function (err, res) { + args[index] = shimmer.wrapFunction(callback, callback => asyncResource.bind(function (err, res) { if (err) { errorCh.publish(err) } @@ -168,7 +174,7 @@ function instrument (operation, command, ctx, args, server, ns, ops, options = { if (callback) { return callback.apply(this, arguments) } - }) + })) try { return command.apply(ctx, args) @@ -179,3 +185,26 @@ function instrument (operation, command, ctx, args, server, ns, ops, options = { } }) } + +function instrumentPromise (operation, command, ctx, args, server, ns, ops, options = {}) { + const name = options.name || (ops && Object.keys(ops)[0]) + + const serverInfo = server && server.s && server.s.options + const asyncResource = new AsyncResource('bound-anonymous-fn') + + return asyncResource.runInAsyncScope(() => { + startCh.publish({ ns, ops, options: serverInfo, name }) + + const promise = command.apply(ctx, args) + + return promise.then(function (res) { + finishCh.publish() + return res + }, function (err) { + errorCh.publish(err) + finishCh.publish() + + return Promise.reject(err) + }) + }) +} diff --git a/packages/datadog-instrumentations/src/mongoose.js b/packages/datadog-instrumentations/src/mongoose.js index 4b13eaccdb3..8116e38380e 100644 --- a/packages/datadog-instrumentations/src/mongoose.js +++ b/packages/datadog-instrumentations/src/mongoose.js @@ -21,7 +21,8 @@ addHook({ name: 'mongoose', versions: ['>=4.6.4 <5', '5', '6', '>=7'] }, mongoose => { - if (mongoose.Promise !== global.Promise) { + // As of Mongoose 7, custom promise libraries are no longer supported and mongoose.Promise may be undefined + if (mongoose.Promise && mongoose.Promise !== global.Promise) { shimmer.wrap(mongoose.Promise.prototype, 'then', wrapThen) } @@ -79,21 +80,26 @@ addHook({ }) let callbackWrapped = false - const lastArgumentIndex = arguments.length - 1 - if (typeof arguments[lastArgumentIndex] === 'function') { - // is a callback, wrap it to execute finish() - shimmer.wrap(arguments, lastArgumentIndex, originalCb => { - return function () { - finish() + const wrapCallbackIfExist = (args) => { + const lastArgumentIndex = args.length - 1 - return originalCb.apply(this, arguments) - } - }) + if (typeof args[lastArgumentIndex] === 'function') { + // is a callback, wrap it to execute finish() + shimmer.wrap(args, lastArgumentIndex, originalCb => { + return function () { + finish() + + return originalCb.apply(this, arguments) + } + }) - callbackWrapped = true + callbackWrapped = true + } } + wrapCallbackIfExist(arguments) + return asyncResource.runInAsyncScope(() => { startCh.publish({ filters, @@ -106,30 +112,37 @@ addHook({ if (!callbackWrapped) { shimmer.wrap(res, 'exec', originalExec => { return function wrappedExec () { + if (!callbackWrapped) { + wrapCallbackIfExist(arguments) + } + const execResult = originalExec.apply(this, arguments) + if (callbackWrapped || typeof execResult?.then !== 'function') { + return execResult + } + // wrap them method, wrap resolve and reject methods shimmer.wrap(execResult, 'then', originalThen => { return function wrappedThen () { const resolve = arguments[0] const reject = arguments[1] - // not using shimmer here because resolve/reject could be empty - arguments[0] = function wrappedResolve () { + arguments[0] = shimmer.wrapFunction(resolve, resolve => function wrappedResolve () { finish() if (resolve) { return resolve.apply(this, arguments) } - } + }) - arguments[1] = function wrappedReject () { + arguments[1] = shimmer.wrapFunction(reject, reject => function wrappedReject () { finish() if (reject) { return reject.apply(this, arguments) } - } + }) return originalThen.apply(this, arguments) } @@ -155,7 +168,7 @@ addHook({ versions: ['6', '>=7'], file: 'lib/helpers/query/sanitizeFilter.js' }, sanitizeFilter => { - return shimmer.wrap(sanitizeFilter, function wrappedSanitizeFilter () { + return shimmer.wrapFunction(sanitizeFilter, sanitizeFilter => function wrappedSanitizeFilter () { const sanitizedObject = sanitizeFilter.apply(this, arguments) if (sanitizeFilterFinishCh.hasSubscribers) { diff --git a/packages/datadog-instrumentations/src/mquery.js b/packages/datadog-instrumentations/src/mquery.js new file mode 100644 index 00000000000..c300736f3fe --- /dev/null +++ b/packages/datadog-instrumentations/src/mquery.js @@ -0,0 +1,65 @@ +'use strict' + +const dc = require('dc-polyfill') +const { + channel, + addHook +} = require('./helpers/instrument') +const shimmer = require('../../datadog-shimmer') + +const prepareCh = channel('datadog:mquery:filter:prepare') +const tracingCh = dc.tracingChannel('datadog:mquery:filter') + +const methods = [ + 'find', + 'findOne', + 'findOneAndRemove', + 'findOneAndDelete', + 'count', + 'distinct', + 'where' +] + +const methodsOptionalArgs = ['findOneAndUpdate'] + +function getFilters (args, methodName) { + const [arg0, arg1] = args + + const filters = arg0 !== null && typeof arg0 === 'object' ? [arg0] : [] + + if (arg1 !== null && typeof arg1 === 'object' && methodsOptionalArgs.includes(methodName)) { + filters.push(arg1) + } + + return filters +} + +addHook({ + name: 'mquery', + versions: ['>=5.0.0'] +}, Query => { + [...methods, ...methodsOptionalArgs].forEach(methodName => { + if (!(methodName in Query.prototype)) return + + shimmer.wrap(Query.prototype, methodName, method => { + return function () { + if (prepareCh.hasSubscribers) { + const filters = getFilters(arguments, methodName) + if (filters?.length) { + prepareCh.publish({ filters }) + } + } + + return method.apply(this, arguments) + } + }) + }) + + shimmer.wrap(Query.prototype, 'exec', originalExec => { + return function wrappedExec () { + return tracingCh.tracePromise(originalExec, {}, this, arguments) + } + }) + + return Query +}) diff --git a/packages/datadog-instrumentations/src/mysql.js b/packages/datadog-instrumentations/src/mysql.js index ca7a7d0d341..a7b2c4fb8b9 100644 --- a/packages/datadog-instrumentations/src/mysql.js +++ b/packages/datadog-instrumentations/src/mysql.js @@ -37,14 +37,14 @@ addHook({ name: 'mysql', file: 'lib/Connection.js', versions: ['>=2'] }, Connect if (res._callback) { const cb = callbackResource.bind(res._callback) - res._callback = asyncResource.bind(function (error, result) { + res._callback = shimmer.wrapFunction(cb, cb => asyncResource.bind(function (error, result) { if (error) { errorCh.publish(error) } finishCh.publish(result) return cb.apply(this, arguments) - }) + })) } else { const cb = asyncResource.bind(function () { finishCh.publish(undefined) @@ -92,7 +92,7 @@ addHook({ name: 'mysql', file: 'lib/Pool.js', versions: ['>=2'] }, Pool => { const cb = arguments[arguments.length - 1] if (typeof cb === 'function') { - arguments[arguments.length - 1] = shimmer.wrap(cb, function () { + arguments[arguments.length - 1] = shimmer.wrapFunction(cb, cb => function () { finish() return cb.apply(this, arguments) }) diff --git a/packages/datadog-instrumentations/src/mysql2.js b/packages/datadog-instrumentations/src/mysql2.js index 809077bcb7c..096eec0e80e 100644 --- a/packages/datadog-instrumentations/src/mysql2.js +++ b/packages/datadog-instrumentations/src/mysql2.js @@ -6,11 +6,14 @@ const { AsyncResource } = require('./helpers/instrument') const shimmer = require('../../datadog-shimmer') +const semver = require('semver') -addHook({ name: 'mysql2', file: 'lib/connection.js', versions: ['>=1'] }, Connection => { +addHook({ name: 'mysql2', file: 'lib/connection.js', versions: ['>=1'] }, (Connection, version) => { const startCh = channel('apm:mysql2:query:start') const finishCh = channel('apm:mysql2:query:finish') const errorCh = channel('apm:mysql2:query:error') + const startOuterQueryCh = channel('datadog:mysql2:outerquery:start') + const shouldEmitEndAfterQueryAbort = semver.intersects(version, '>=1.3.3') shimmer.wrap(Connection.prototype, 'addCommand', addCommand => function (cmd) { if (!startCh.hasSubscribers) return addCommand.apply(this, arguments) @@ -28,22 +31,92 @@ addHook({ name: 'mysql2', file: 'lib/connection.js', versions: ['>=1'] }, Connec return asyncResource.bind(addCommand, this).apply(this, arguments) }) + shimmer.wrap(Connection.prototype, 'query', query => function (sql, values, cb) { + if (!startOuterQueryCh.hasSubscribers) return query.apply(this, arguments) + + if (typeof sql === 'object') sql = sql?.sql + + if (!sql) return query.apply(this, arguments) + + const abortController = new AbortController() + startOuterQueryCh.publish({ sql, abortController }) + + if (abortController.signal.aborted) { + const addCommand = this.addCommand + this.addCommand = function (cmd) { return cmd } + + let queryCommand + try { + queryCommand = query.apply(this, arguments) + } finally { + this.addCommand = addCommand + } + + cb = queryCommand.onResult + + process.nextTick(() => { + if (cb) { + cb(abortController.signal.reason) + } else { + queryCommand.emit('error', abortController.signal.reason) + } + + if (shouldEmitEndAfterQueryAbort) { + queryCommand.emit('end') + } + }) + + return queryCommand + } + + return query.apply(this, arguments) + }) + + shimmer.wrap(Connection.prototype, 'execute', execute => function (sql, values, cb) { + if (!startOuterQueryCh.hasSubscribers) return execute.apply(this, arguments) + + if (typeof sql === 'object') sql = sql?.sql + + if (!sql) return execute.apply(this, arguments) + + const abortController = new AbortController() + startOuterQueryCh.publish({ sql, abortController }) + + if (abortController.signal.aborted) { + const addCommand = this.addCommand + this.addCommand = function (cmd) { return cmd } + + let result + try { + result = execute.apply(this, arguments) + } finally { + this.addCommand = addCommand + } + + result?.onResult(abortController.signal.reason) + + return result + } + + return execute.apply(this, arguments) + }) + return Connection function bindExecute (cmd, execute, asyncResource) { - return asyncResource.bind(function executeWithTrace (packet, connection) { + return shimmer.wrapFunction(execute, execute => asyncResource.bind(function executeWithTrace (packet, connection) { if (this.onResult) { this.onResult = asyncResource.bind(this.onResult) } return execute.apply(this, arguments) - }, cmd) + }, cmd)) } function wrapExecute (cmd, execute, asyncResource, config) { const callbackResource = new AsyncResource('bound-anonymous-fn') - return asyncResource.bind(function executeWithTrace (packet, connection) { + return shimmer.wrapFunction(execute, execute => asyncResource.bind(function executeWithTrace (packet, connection) { const sql = cmd.statement ? cmd.statement.query : cmd.sql const payload = { sql, conf: config } startCh.publish(payload) @@ -57,13 +130,13 @@ addHook({ name: 'mysql2', file: 'lib/connection.js', versions: ['>=1'] }, Connec if (this.onResult) { const onResult = callbackResource.bind(this.onResult) - this.onResult = asyncResource.bind(function (error) { + this.onResult = shimmer.wrapFunction(onResult, onResult => asyncResource.bind(function (error) { if (error) { errorCh.publish(error) } finishCh.publish(undefined) onResult.apply(this, arguments) - }, 'bound-anonymous-fn', this) + }, 'bound-anonymous-fn', this)) } else { this.on('error', asyncResource.bind(error => errorCh.publish(error))) this.on('end', asyncResource.bind(() => finishCh.publish(undefined))) @@ -76,6 +149,152 @@ addHook({ name: 'mysql2', file: 'lib/connection.js', versions: ['>=1'] }, Connec } catch (err) { errorCh.publish(err) } - }, cmd) + }, cmd)) } }) + +addHook({ name: 'mysql2', file: 'lib/pool.js', versions: ['>=1'] }, (Pool, version) => { + const startOuterQueryCh = channel('datadog:mysql2:outerquery:start') + const shouldEmitEndAfterQueryAbort = semver.intersects(version, '>=1.3.3') + + shimmer.wrap(Pool.prototype, 'query', query => function (sql, values, cb) { + if (!startOuterQueryCh.hasSubscribers) return query.apply(this, arguments) + + if (typeof sql === 'object') sql = sql?.sql + + if (!sql) return query.apply(this, arguments) + + const abortController = new AbortController() + startOuterQueryCh.publish({ sql, abortController }) + + if (abortController.signal.aborted) { + const getConnection = this.getConnection + this.getConnection = function () {} + + let queryCommand + try { + queryCommand = query.apply(this, arguments) + } finally { + this.getConnection = getConnection + } + + process.nextTick(() => { + if (queryCommand.onResult) { + queryCommand.onResult(abortController.signal.reason) + } else { + queryCommand.emit('error', abortController.signal.reason) + } + + if (shouldEmitEndAfterQueryAbort) { + queryCommand.emit('end') + } + }) + + return queryCommand + } + + return query.apply(this, arguments) + }) + + shimmer.wrap(Pool.prototype, 'execute', execute => function (sql, values, cb) { + if (!startOuterQueryCh.hasSubscribers) return execute.apply(this, arguments) + + if (typeof sql === 'object') sql = sql?.sql + + if (!sql) return execute.apply(this, arguments) + + const abortController = new AbortController() + startOuterQueryCh.publish({ sql, abortController }) + + if (abortController.signal.aborted) { + if (typeof values === 'function') { + cb = values + } + + process.nextTick(() => { + cb(abortController.signal.reason) + }) + return + } + + return execute.apply(this, arguments) + }) + + return Pool +}) + +// PoolNamespace.prototype.query does not exist in mysql2<2.3.0 +addHook({ name: 'mysql2', file: 'lib/pool_cluster.js', versions: ['>=2.3.0'] }, PoolCluster => { + const startOuterQueryCh = channel('datadog:mysql2:outerquery:start') + const wrappedPoolNamespaces = new WeakSet() + + shimmer.wrap(PoolCluster.prototype, 'of', of => function () { + const poolNamespace = of.apply(this, arguments) + + if (startOuterQueryCh.hasSubscribers && !wrappedPoolNamespaces.has(poolNamespace)) { + shimmer.wrap(poolNamespace, 'query', query => function (sql, values, cb) { + if (typeof sql === 'object') sql = sql?.sql + + if (!sql) return query.apply(this, arguments) + + const abortController = new AbortController() + startOuterQueryCh.publish({ sql, abortController }) + + if (abortController.signal.aborted) { + const getConnection = this.getConnection + this.getConnection = function () {} + + let queryCommand + try { + queryCommand = query.apply(this, arguments) + } finally { + this.getConnection = getConnection + } + + process.nextTick(() => { + if (queryCommand.onResult) { + queryCommand.onResult(abortController.signal.reason) + } else { + queryCommand.emit('error', abortController.signal.reason) + } + + queryCommand.emit('end') + }) + + return queryCommand + } + + return query.apply(this, arguments) + }) + + shimmer.wrap(poolNamespace, 'execute', execute => function (sql, values, cb) { + if (typeof sql === 'object') sql = sql?.sql + + if (!sql) return execute.apply(this, arguments) + + const abortController = new AbortController() + startOuterQueryCh.publish({ sql, abortController }) + + if (abortController.signal.aborted) { + if (typeof values === 'function') { + cb = values + } + + process.nextTick(() => { + cb(abortController.signal.reason) + }) + + return + } + + return execute.apply(this, arguments) + }) + + wrappedPoolNamespaces.add(poolNamespace) + } + + return poolNamespace + }) + + return PoolCluster +}) diff --git a/packages/datadog-instrumentations/src/net.js b/packages/datadog-instrumentations/src/net.js index a5de6f511ba..b9862a97681 100644 --- a/packages/datadog-instrumentations/src/net.js +++ b/packages/datadog-instrumentations/src/net.js @@ -15,10 +15,18 @@ const startTCPCh = channel('apm:net:tcp:start') const finishTCPCh = channel('apm:net:tcp:finish') const errorTCPCh = channel('apm:net:tcp:error') -const connectionCh = channel(`apm:net:tcp:connection`) +const connectionCh = channel('apm:net:tcp:connection') -addHook({ name: 'net' }, net => { - require('dns') +const names = ['net', 'node:net'] + +addHook({ name: names }, (net, version, name) => { + // explicitly require dns so that net gets an instrumented instance + // so that we don't miss the dns calls + if (name === 'net') { + require('dns') + } else { + require('node:dns') + } shimmer.wrap(net.Socket.prototype, 'connect', connect => function () { if (!startICPCh.hasSubscribers || !startTCPCh.hasSubscribers) { @@ -50,7 +58,7 @@ addHook({ name: 'net' }, net => { } const emit = this.emit - this.emit = function (eventName) { + this.emit = shimmer.wrapFunction(emit, emit => function (eventName) { switch (eventName) { case 'ready': case 'connect': @@ -60,7 +68,7 @@ addHook({ name: 'net' }, net => { default: return emit.apply(this, arguments) } - } + }) try { return connect.apply(this, arguments) diff --git a/packages/datadog-instrumentations/src/next.js b/packages/datadog-instrumentations/src/next.js index b4406ba60b0..57f90f71ee4 100644 --- a/packages/datadog-instrumentations/src/next.js +++ b/packages/datadog-instrumentations/src/next.js @@ -46,7 +46,7 @@ function wrapHandleApiRequest (handleApiRequest) { function wrapHandleApiRequestWithMatch (handleApiRequest) { return function (req, res, query, match) { return instrument(req, res, () => { - const page = (typeof match === 'object' && typeof match.definition === 'object') + const page = (match !== null && typeof match === 'object' && typeof match.definition === 'object') ? match.definition.pathname : undefined @@ -65,7 +65,7 @@ function wrapRenderToHTML (renderToHTML) { function wrapRenderErrorToHTML (renderErrorToHTML) { return function (err, req, res, pathname, query) { - return instrument(req, res, () => renderErrorToHTML.apply(this, arguments)) + return instrument(req, res, err, () => renderErrorToHTML.apply(this, arguments)) } } @@ -76,8 +76,8 @@ function wrapRenderToResponse (renderToResponse) { } function wrapRenderErrorToResponse (renderErrorToResponse) { - return function (ctx) { - return instrument(ctx.req, ctx.res, () => renderErrorToResponse.apply(this, arguments)) + return function (ctx, err) { + return instrument(ctx.req, ctx.res, err, () => renderErrorToResponse.apply(this, arguments)) } } @@ -111,13 +111,23 @@ function getPageFromPath (page, dynamicRoutes = []) { return getPagePath(page) } -function instrument (req, res, handler) { +function instrument (req, res, error, handler) { + if (typeof error === 'function') { + handler = error + error = null + } + req = req.originalRequest || req res = res.originalResponse || res // TODO support middleware properly in the future? const isMiddleware = req.headers[MIDDLEWARE_HEADER] - if (isMiddleware || requests.has(req)) return handler() + if (isMiddleware || requests.has(req)) { + if (error) { + errorChannel.publish({ error }) + } + return handler() + } requests.add(req) @@ -178,7 +188,7 @@ function finish (ctx, result, err) { // however, it is not provided as a class function or exported property addHook({ name: 'next', - versions: ['>=13.3.0'], + versions: ['>=13.3.0 <14.2.7'], file: 'dist/server/web/spec-extension/adapters/next-request.js' }, NextRequestAdapter => { shimmer.wrap(NextRequestAdapter.NextRequestAdapter, 'fromNodeNextRequest', fromNodeNextRequest => { @@ -193,7 +203,7 @@ addHook({ addHook({ name: 'next', - versions: ['>=11.1'], + versions: ['>=11.1 <14.2.7'], file: 'dist/server/serve-static.js' }, serveStatic => shimmer.wrap(serveStatic, 'serveStatic', wrapServeStatic)) @@ -203,7 +213,7 @@ addHook({ file: 'dist/next-server/server/serve-static.js' }, serveStatic => shimmer.wrap(serveStatic, 'serveStatic', wrapServeStatic)) -addHook({ name: 'next', versions: ['>=11.1'], file: 'dist/server/next-server.js' }, nextServer => { +addHook({ name: 'next', versions: ['>=11.1 <14.2.7'], file: 'dist/server/next-server.js' }, nextServer => { const Server = nextServer.default shimmer.wrap(Server.prototype, 'handleRequest', wrapHandleRequest) @@ -220,7 +230,7 @@ addHook({ name: 'next', versions: ['>=11.1'], file: 'dist/server/next-server.js' }) // `handleApiRequest` changes parameters/implementation at 13.2.0 -addHook({ name: 'next', versions: ['>=13.2'], file: 'dist/server/next-server.js' }, nextServer => { +addHook({ name: 'next', versions: ['>=13.2 <14.2.7'], file: 'dist/server/next-server.js' }, nextServer => { const Server = nextServer.default shimmer.wrap(Server.prototype, 'handleApiRequest', wrapHandleApiRequestWithMatch) return nextServer @@ -254,7 +264,7 @@ addHook({ addHook({ name: 'next', - versions: ['>=13'], + versions: ['>=13 <14.2.7'], file: 'dist/server/web/spec-extension/request.js' }, request => { const nextUrlDescriptor = Object.getOwnPropertyDescriptor(request.NextRequest.prototype, 'nextUrl') @@ -280,9 +290,23 @@ addHook({ shimmer.massWrap(request.NextRequest.prototype, ['text', 'json'], function (originalMethod) { return async function wrappedJson () { const body = await originalMethod.apply(this, arguments) - bodyParsedChannel.publish({ - body - }) + + bodyParsedChannel.publish({ body }) + + return body + } + }) + + shimmer.wrap(request.NextRequest.prototype, 'formData', function (originalFormData) { + return async function wrappedFormData () { + const body = await originalFormData.apply(this, arguments) + + let normalizedBody = body + if (typeof body.entries === 'function') { + normalizedBody = Object.fromEntries(body.entries()) + } + bodyParsedChannel.publish({ body: normalizedBody }) + return body } }) diff --git a/packages/datadog-instrumentations/src/nyc.js b/packages/datadog-instrumentations/src/nyc.js new file mode 100644 index 00000000000..34210a78f06 --- /dev/null +++ b/packages/datadog-instrumentations/src/nyc.js @@ -0,0 +1,23 @@ +const { addHook, channel } = require('./helpers/instrument') +const shimmer = require('../../datadog-shimmer') + +const codeCoverageWrapCh = channel('ci:nyc:wrap') + +addHook({ + name: 'nyc', + versions: ['>=17'] +}, (nycPackage) => { + shimmer.wrap(nycPackage.prototype, 'wrap', wrap => async function () { + // Only relevant if the config `all` is set to true + try { + if (JSON.parse(process.env.NYC_CONFIG).all) { + codeCoverageWrapCh.publish(this) + } + } catch (e) { + // ignore errors + } + + return wrap.apply(this, arguments) + }) + return nycPackage +}) diff --git a/packages/datadog-instrumentations/src/openai.js b/packages/datadog-instrumentations/src/openai.js index a3311668132..940b5919d24 100644 --- a/packages/datadog-instrumentations/src/openai.js +++ b/packages/datadog-instrumentations/src/openai.js @@ -1,14 +1,104 @@ 'use strict' -const { - channel, - addHook -} = require('./helpers/instrument') +const { addHook } = require('./helpers/instrument') const shimmer = require('../../datadog-shimmer') -const startCh = channel('apm:openai:request:start') -const finishCh = channel('apm:openai:request:finish') -const errorCh = channel('apm:openai:request:error') +const tracingChannel = require('dc-polyfill').tracingChannel +const ch = tracingChannel('apm:openai:request') + +const V4_PACKAGE_SHIMS = [ + { + file: 'resources/chat/completions.js', + targetClass: 'Completions', + baseResource: 'chat.completions', + methods: ['create'], + streamedResponse: true + }, + { + file: 'resources/completions.js', + targetClass: 'Completions', + baseResource: 'completions', + methods: ['create'], + streamedResponse: true + }, + { + file: 'resources/embeddings.js', + targetClass: 'Embeddings', + baseResource: 'embeddings', + methods: ['create'] + }, + { + file: 'resources/files.js', + targetClass: 'Files', + baseResource: 'files', + methods: ['create', 'del', 'list', 'retrieve'] + }, + { + file: 'resources/files.js', + targetClass: 'Files', + baseResource: 'files', + methods: ['retrieveContent'], + versions: ['>=4.0.0 <4.17.1'] + }, + { + file: 'resources/files.js', + targetClass: 'Files', + baseResource: 'files', + methods: ['content'], // replaced `retrieveContent` in v4.17.1 + versions: ['>=4.17.1'] + }, + { + file: 'resources/images.js', + targetClass: 'Images', + baseResource: 'images', + methods: ['createVariation', 'edit', 'generate'] + }, + { + file: 'resources/fine-tuning/jobs/jobs.js', + targetClass: 'Jobs', + baseResource: 'fine_tuning.jobs', + methods: ['cancel', 'create', 'list', 'listEvents', 'retrieve'], + versions: ['>=4.34.0'] // file location changed in 4.34.0 + }, + { + file: 'resources/fine-tuning/jobs.js', + targetClass: 'Jobs', + baseResource: 'fine_tuning.jobs', + methods: ['cancel', 'create', 'list', 'listEvents', 'retrieve'], + versions: ['>=4.1.0 <4.34.0'] + }, + { + file: 'resources/fine-tunes.js', // deprecated after 4.1.0 + targetClass: 'FineTunes', + baseResource: 'fine-tune', + methods: ['cancel', 'create', 'list', 'listEvents', 'retrieve'], + versions: ['>=4.0.0 <4.1.0'] + }, + { + file: 'resources/models.js', + targetClass: 'Models', + baseResource: 'models', + methods: ['del', 'list', 'retrieve'] + }, + { + file: 'resources/moderations.js', + targetClass: 'Moderations', + baseResource: 'moderations', + methods: ['create'] + }, + { + file: 'resources/audio/transcriptions.js', + targetClass: 'Transcriptions', + baseResource: 'audio.transcriptions', + methods: ['create'] + }, + { + file: 'resources/audio/translations.js', + targetClass: 'Translations', + baseResource: 'audio.translations', + methods: ['create'] + } +] addHook({ name: 'openai', file: 'dist/api.js', versions: ['>=3.0.0 <4'] }, exports => { const methodNames = Object.getOwnPropertyNames(exports.OpenAIApi.prototype) @@ -16,35 +106,256 @@ addHook({ name: 'openai', file: 'dist/api.js', versions: ['>=3.0.0 <4'] }, expor for (const methodName of methodNames) { shimmer.wrap(exports.OpenAIApi.prototype, methodName, fn => function () { - if (!startCh.hasSubscribers) { + if (!ch.start.hasSubscribers) { return fn.apply(this, arguments) } - startCh.publish({ + const ctx = { methodName, args: arguments, basePath: this.basePath, apiKey: this.configuration.apiKey + } + + return ch.tracePromise(fn, ctx, this, ...arguments) + }) + } + + return exports +}) + +function addStreamedChunk (content, chunk) { + content.usage = chunk.usage // add usage if it was specified to be returned + for (const choice of chunk.choices) { + const choiceIdx = choice.index + const oldChoice = content.choices.find(choice => choice?.index === choiceIdx) + if (!oldChoice) { + // we don't know which choices arrive in which order + content.choices[choiceIdx] = choice + } else { + if (!oldChoice.finish_reason) { + oldChoice.finish_reason = choice.finish_reason + } + + // delta exists on chat completions + const delta = choice.delta + + if (delta) { + const content = delta.content + if (content) { + if (oldChoice.delta.content) { // we don't want to append to undefined + oldChoice.delta.content += content + } else { + oldChoice.delta.content = content + } + } + } else { + const text = choice.text + if (text) { + if (oldChoice.text) { + oldChoice.text += text + } else { + oldChoice.text = text + } + } + } + + // tools only exist on chat completions + const tools = delta && choice.delta.tool_calls + + if (tools) { + oldChoice.delta.tool_calls = tools.map((newTool, toolIdx) => { + const oldTool = oldChoice.delta.tool_calls?.[toolIdx] + + if (oldTool) { + oldTool.function.arguments += newTool.function.arguments + } else { + return newTool + } + + return oldTool + }) + } + } + } +} + +function convertBufferstoObjects (chunks = []) { + return Buffer + .concat(chunks) // combine the buffers + .toString() // stringify + .split(/(?=data:)/) // split on "data:" + .map(chunk => chunk.split('\n').join('')) // remove newlines + .map(chunk => chunk.substring(6)) // remove 'data: ' from the front + .slice(0, -1) // remove the last [DONE] message + .map(JSON.parse) // parse all of the returned objects +} + +/** + * For streamed responses, we need to accumulate all of the content in + * the chunks, and let the combined content be the final response. + * This way, spans look the same as when not streamed. + */ +function wrapStreamIterator (response, options, n, ctx) { + let processChunksAsBuffers = false + let chunks = [] + return function (itr) { + return function () { + const iterator = itr.apply(this, arguments) + shimmer.wrap(iterator, 'next', next => function () { + return next.apply(this, arguments) + .then(res => { + const { done, value: chunk } = res + + if (chunk) { + chunks.push(chunk) + if (chunk instanceof Buffer) { + // this operation should be safe + // if one chunk is a buffer (versus a plain object), the rest should be as well + processChunksAsBuffers = true + } + } + + if (done) { + let body = {} + chunks = chunks.filter(chunk => chunk != null) // filter null or undefined values + + if (chunks) { + if (processChunksAsBuffers) { + chunks = convertBufferstoObjects(chunks) + } + + if (chunks.length) { + // define the initial body having all the content outside of choices from the first chunk + // this will include import data like created, id, model, etc. + body = { ...chunks[0], choices: Array.from({ length: n }) } + // start from the first chunk, and add its choices into the body + for (let i = 0; i < chunks.length; i++) { + addStreamedChunk(body, chunks[i]) + } + } + } + + finish(ctx, { + headers: response.headers, + data: body, + request: { + path: response.url, + method: options.method + } + }) + } + + return res + }) + .catch(err => { + finish(ctx, undefined, err) + + throw err + }) }) + return iterator + } + } +} + +for (const shim of V4_PACKAGE_SHIMS) { + const { file, targetClass, baseResource, methods, versions, streamedResponse } = shim + addHook({ name: 'openai', file, versions: versions || ['>=4'] }, exports => { + const targetPrototype = exports[targetClass].prototype + + for (const methodName of methods) { + shimmer.wrap(targetPrototype, methodName, methodFn => function () { + if (!ch.start.hasSubscribers) { + return methodFn.apply(this, arguments) + } + + // The OpenAI library lets you set `stream: true` on the options arg to any method + // However, we only want to handle streamed responses in specific cases + // chat.completions and completions + const stream = streamedResponse && getOption(arguments, 'stream', false) - return fn.apply(this, arguments) - .then((response) => { - finishCh.publish({ - headers: response.headers, - body: response.data, - path: response.request.path, - method: response.request.method + // we need to compute how many prompts we are sending in streamed cases for completions + // not applicable for chat completiond + let n + if (stream) { + n = getOption(arguments, 'n', 1) + const prompt = getOption(arguments, 'prompt') + if (Array.isArray(prompt) && typeof prompt[0] !== 'number') { + n *= prompt.length + } + } + + const client = this._client || this.client + + const ctx = { + methodName: `${baseResource}.${methodName}`, + args: arguments, + basePath: client.baseURL, + apiKey: client.apiKey + } + + return ch.start.runStores(ctx, () => { + const apiProm = methodFn.apply(this, arguments) + + // wrapping `parse` avoids problematic wrapping of `then` when trying to call + // `withResponse` in userland code after. This way, we can return the whole `APIPromise` + shimmer.wrap(apiProm, 'parse', origApiPromParse => function () { + return origApiPromParse.apply(this, arguments) + // the original response is wrapped in a promise, so we need to unwrap it + .then(body => Promise.all([this.responsePromise, body])) + .then(([{ response, options }, body]) => { + if (stream) { + if (body.iterator) { + shimmer.wrap(body, 'iterator', wrapStreamIterator(response, options, n, ctx)) + } else { + shimmer.wrap( + body.response.body, Symbol.asyncIterator, wrapStreamIterator(response, options, n, ctx) + ) + } + } else { + finish(ctx, { + headers: response.headers, + data: body, + request: { + path: response.url, + method: options.method + } + }) + } + + return body + }) + .catch(error => { + finish(ctx, undefined, error) + + throw error + }) + .finally(() => { + // maybe we don't want to unwrap here in case the promise is re-used? + // other hand: we want to avoid resource leakage + shimmer.unwrap(apiProm, 'parse') + }) }) - return response + return apiProm }) - .catch((err) => { - errorCh.publish({ err }) + }) + } + return exports + }) +} - throw err - }) - }) +function finish (ctx, response, error) { + if (error) { + ctx.error = error + ch.error.publish(ctx) } - return exports -}) + ctx.result = response + ch.asyncEnd.publish(ctx) +} + +function getOption (args, option, defaultValue) { + return args[args.length - 1]?.[option] || defaultValue +} diff --git a/packages/datadog-instrumentations/src/oracledb.js b/packages/datadog-instrumentations/src/oracledb.js index 15f34b1dc0e..61db967fdfe 100644 --- a/packages/datadog-instrumentations/src/oracledb.js +++ b/packages/datadog-instrumentations/src/oracledb.js @@ -21,7 +21,7 @@ function finish (err) { finishChannel.publish(undefined) } -addHook({ name: 'oracledb', versions: ['5'] }, oracledb => { +addHook({ name: 'oracledb', versions: ['>=5'] }, oracledb => { shimmer.wrap(oracledb.Connection.prototype, 'execute', execute => { return function wrappedExecute (dbQuery, ...args) { if (!startChannel.hasSubscribers) { @@ -31,10 +31,10 @@ addHook({ name: 'oracledb', versions: ['5'] }, oracledb => { if (arguments.length && typeof arguments[arguments.length - 1] === 'function') { const cb = arguments[arguments.length - 1] const outerAr = new AsyncResource('apm:oracledb:outer-scope') - arguments[arguments.length - 1] = function wrappedCb (err, result) { + arguments[arguments.length - 1] = shimmer.wrapFunction(cb, cb => function wrappedCb (err, result) { finish(err) return outerAr.runInAsyncScope(() => cb.apply(this, arguments)) - } + }) } return new AsyncResource('apm:oracledb:inner-scope').runInAsyncScope(() => { @@ -67,12 +67,12 @@ addHook({ name: 'oracledb', versions: ['5'] }, oracledb => { shimmer.wrap(oracledb, 'getConnection', getConnection => { return function wrappedGetConnection (connAttrs, callback) { if (callback) { - arguments[1] = (err, connection) => { + arguments[1] = shimmer.wrapFunction(callback, callback => (err, connection) => { if (connection) { connectionAttributes.set(connection, connAttrs) } callback(err, connection) - } + }) getConnection.apply(this, arguments) } else { @@ -86,12 +86,12 @@ addHook({ name: 'oracledb', versions: ['5'] }, oracledb => { shimmer.wrap(oracledb, 'createPool', createPool => { return function wrappedCreatePool (poolAttrs, callback) { if (callback) { - arguments[1] = (err, pool) => { + arguments[1] = shimmer.wrapFunction(callback, callback => (err, pool) => { if (pool) { poolAttributes.set(pool, poolAttrs) } callback(err, pool) - } + }) createPool.apply(this, arguments) } else { @@ -109,12 +109,12 @@ addHook({ name: 'oracledb', versions: ['5'] }, oracledb => { callback = arguments[arguments.length - 1] } if (callback) { - arguments[arguments.length - 1] = (err, connection) => { + arguments[arguments.length - 1] = shimmer.wrapFunction(callback, callback => (err, connection) => { if (connection) { connectionAttributes.set(connection, poolAttributes.get(this)) } callback(err, connection) - } + }) getConnection.apply(this, arguments) } else { return getConnection.apply(this, arguments).then((connection) => { diff --git a/packages/datadog-instrumentations/src/otel-sdk-trace.js b/packages/datadog-instrumentations/src/otel-sdk-trace.js index 319a2fda2f3..4e0d5310a31 100644 --- a/packages/datadog-instrumentations/src/otel-sdk-trace.js +++ b/packages/datadog-instrumentations/src/otel-sdk-trace.js @@ -4,7 +4,12 @@ const { addHook } = require('./helpers/instrument') const shimmer = require('../../datadog-shimmer') const tracer = require('../../dd-trace') -if (process.env.DD_TRACE_OTEL_ENABLED) { +const otelSdkEnabled = process.env.DD_TRACE_OTEL_ENABLED || +process.env.OTEL_SDK_DISABLED + ? !process.env.OTEL_SDK_DISABLED + : undefined + +if (otelSdkEnabled) { addHook({ name: '@opentelemetry/sdk-trace-node', file: 'build/src/NodeTracerProvider.js', diff --git a/packages/datadog-instrumentations/src/passport-http.js b/packages/datadog-instrumentations/src/passport-http.js index 3ffc369a395..0969d2d3fc9 100644 --- a/packages/datadog-instrumentations/src/passport-http.js +++ b/packages/datadog-instrumentations/src/passport-http.js @@ -9,7 +9,7 @@ addHook({ file: 'lib/passport-http/strategies/basic.js', versions: ['>=0.3.0'] }, BasicStrategy => { - return shimmer.wrap(BasicStrategy, function () { + return shimmer.wrapFunction(BasicStrategy, BasicStrategy => function () { const type = 'http' if (typeof arguments[0] === 'function') { diff --git a/packages/datadog-instrumentations/src/passport-local.js b/packages/datadog-instrumentations/src/passport-local.js index d0c48c56ccb..dab74eb470e 100644 --- a/packages/datadog-instrumentations/src/passport-local.js +++ b/packages/datadog-instrumentations/src/passport-local.js @@ -9,7 +9,7 @@ addHook({ file: 'lib/strategy.js', versions: ['>=1.0.0'] }, Strategy => { - return shimmer.wrap(Strategy, function () { + return shimmer.wrapFunction(Strategy, Strategy => function () { const type = 'local' if (typeof arguments[0] === 'function') { diff --git a/packages/datadog-instrumentations/src/passport-utils.js b/packages/datadog-instrumentations/src/passport-utils.js index 5af55ca94c0..7969ab486b4 100644 --- a/packages/datadog-instrumentations/src/passport-utils.js +++ b/packages/datadog-instrumentations/src/passport-utils.js @@ -10,7 +10,8 @@ function wrapVerifiedAndPublish (username, password, verified, type) { return verified } - return shimmer.wrap(verified, function (err, user, info) { + // eslint-disable-next-line n/handle-callback-err + return shimmer.wrapFunction(verified, verified => function (err, user, info) { const credentials = { type, username } passportVerifyChannel.publish({ credentials, user }) return verified.apply(this, arguments) diff --git a/packages/datadog-instrumentations/src/pg.js b/packages/datadog-instrumentations/src/pg.js index 69f69c28486..55642d82e96 100644 --- a/packages/datadog-instrumentations/src/pg.js +++ b/packages/datadog-instrumentations/src/pg.js @@ -35,7 +35,7 @@ function wrapQuery (query) { const asyncResource = new AsyncResource('bound-anonymous-fn') const processId = this.processID - const pgQuery = arguments[0] && typeof arguments[0] === 'object' + const pgQuery = arguments[0] !== null && typeof arguments[0] === 'object' ? arguments[0] : { text: arguments[0] } @@ -53,14 +53,15 @@ function wrapQuery (query) { } return asyncResource.runInAsyncScope(() => { + const abortController = new AbortController() + startCh.publish({ params: this.connectionParameters, query: pgQuery, - processId + processId, + abortController }) - arguments[0] = pgQuery - const finish = asyncResource.bind(function (error) { if (error) { errorCh.publish(error) @@ -68,6 +69,43 @@ function wrapQuery (query) { finishCh.publish() }) + if (abortController.signal.aborted) { + const error = abortController.signal.reason || new Error('Aborted') + + // eslint-disable-next-line max-len + // Based on: https://github.com/brianc/node-postgres/blob/54eb0fa216aaccd727765641e7d1cf5da2bc483d/packages/pg/lib/client.js#L510 + const reusingQuery = typeof pgQuery.submit === 'function' + const callback = arguments[arguments.length - 1] + + finish(error) + + if (reusingQuery) { + if (!pgQuery.callback && typeof callback === 'function') { + pgQuery.callback = callback + } + + if (pgQuery.callback) { + pgQuery.callback(error) + } else { + process.nextTick(() => { + pgQuery.emit('error', error) + }) + } + + return pgQuery + } + + if (typeof callback === 'function') { + callback(error) + + return + } + + return Promise.reject(error) + } + + arguments[0] = pgQuery + const retval = query.apply(this, arguments) const queryQueue = this.queryQueue || this._queryQueue const activeQuery = this.activeQuery || this._activeQuery @@ -109,11 +147,14 @@ function wrapPoolQuery (query) { const asyncResource = new AsyncResource('bound-anonymous-fn') - const pgQuery = arguments[0] && typeof arguments[0] === 'object' ? arguments[0] : { text: arguments[0] } + const pgQuery = arguments[0] !== null && typeof arguments[0] === 'object' ? arguments[0] : { text: arguments[0] } return asyncResource.runInAsyncScope(() => { + const abortController = new AbortController() + startPoolQueryCh.publish({ - query: pgQuery + query: pgQuery, + abortController }) const finish = asyncResource.bind(function () { @@ -121,8 +162,22 @@ function wrapPoolQuery (query) { }) const cb = arguments[arguments.length - 1] + + if (abortController.signal.aborted) { + const error = abortController.signal.reason || new Error('Aborted') + finish() + + if (typeof cb === 'function') { + cb(error) + + return + } else { + return Promise.reject(error) + } + } + if (typeof cb === 'function') { - arguments[arguments.length - 1] = shimmer.wrap(cb, function () { + arguments[arguments.length - 1] = shimmer.wrapFunction(cb, cb => function () { finish() return cb.apply(this, arguments) }) diff --git a/packages/datadog-instrumentations/src/pino.js b/packages/datadog-instrumentations/src/pino.js index 09a5653f176..503986519ae 100644 --- a/packages/datadog-instrumentations/src/pino.js +++ b/packages/datadog-instrumentations/src/pino.js @@ -76,19 +76,19 @@ function wrapPrettyFactory (prettyFactory) { addHook({ name: 'pino', versions: ['2 - 3', '4', '>=5 <5.14.0'] }, pino => { const asJsonSym = (pino.symbols && pino.symbols.asJsonSym) || 'asJson' - return shimmer.wrap(pino, wrapPino(asJsonSym, wrapAsJson, pino)) + return shimmer.wrapFunction(pino, pino => wrapPino(asJsonSym, wrapAsJson, pino)) }) addHook({ name: 'pino', versions: ['>=5.14.0 <6.8.0'] }, pino => { const mixinSym = pino.symbols.mixinSym - return shimmer.wrap(pino, wrapPino(mixinSym, wrapMixin, pino)) + return shimmer.wrapFunction(pino, pino => wrapPino(mixinSym, wrapMixin, pino)) }) addHook({ name: 'pino', versions: ['>=6.8.0'] }, pino => { const mixinSym = pino.symbols.mixinSym - const wrapped = shimmer.wrap(pino, wrapPino(mixinSym, wrapMixin, pino)) + const wrapped = shimmer.wrapFunction(pino, pino => wrapPino(mixinSym, wrapMixin, pino)) wrapped.pino = wrapped wrapped.default = wrapped @@ -101,5 +101,5 @@ addHook({ name: 'pino-pretty', file: 'lib/utils.js', versions: ['>=3'] }, utils }) addHook({ name: 'pino-pretty', versions: ['1 - 2'] }, prettyFactory => { - return shimmer.wrap(prettyFactory, wrapPrettyFactory(prettyFactory)) + return shimmer.wrapFunction(prettyFactory, wrapPrettyFactory) }) diff --git a/packages/datadog-instrumentations/src/playwright.js b/packages/datadog-instrumentations/src/playwright.js index a209e228ffb..e8332d65c8d 100644 --- a/packages/datadog-instrumentations/src/playwright.js +++ b/packages/datadog-instrumentations/src/playwright.js @@ -1,6 +1,9 @@ +const semver = require('semver') + const { addHook, channel, AsyncResource } = require('./helpers/instrument') const shimmer = require('../../datadog-shimmer') -const { parseAnnotations } = require('../../dd-trace/src/plugins/util/test') +const { parseAnnotations, getTestSuitePath } = require('../../dd-trace/src/plugins/util/test') +const log = require('../../dd-trace/src/log') const testStartCh = channel('ci:playwright:test:start') const testFinishCh = channel('ci:playwright:test:finish') @@ -8,12 +11,19 @@ const testFinishCh = channel('ci:playwright:test:finish') const testSessionStartCh = channel('ci:playwright:session:start') const testSessionFinishCh = channel('ci:playwright:session:finish') +const libraryConfigurationCh = channel('ci:playwright:library-configuration') +const knownTestsCh = channel('ci:playwright:known-tests') + const testSuiteStartCh = channel('ci:playwright:test-suite:start') const testSuiteFinishCh = channel('ci:playwright:test-suite:finish') const testToAr = new WeakMap() const testSuiteToAr = new Map() const testSuiteToTestStatuses = new Map() +const testSuiteToErrors = new Map() +const testSessionAsyncResource = new AsyncResource('bound-anonymous-fn') + +let applyRepeatEachIndex = null let startedSuites = [] @@ -25,6 +35,46 @@ const STATUS_TO_TEST_STATUS = { } let remainingTestsByFile = {} +let isEarlyFlakeDetectionEnabled = false +let earlyFlakeDetectionNumRetries = 0 +let isFlakyTestRetriesEnabled = false +let flakyTestRetriesCount = 0 +let knownTests = {} +let rootDir = '' +const MINIMUM_SUPPORTED_VERSION_EFD = '1.38.0' + +function isNewTest (test) { + const testSuite = getTestSuitePath(test._requireFile, rootDir) + const testsForSuite = knownTests?.playwright?.[testSuite] || [] + + return !testsForSuite.includes(test.title) +} + +function getSuiteType (test, type) { + let suite = test.parent + while (suite && suite._type !== type) { + suite = suite.parent + } + return suite +} + +// Copy of Suite#_deepClone but with a function to filter tests +function deepCloneSuite (suite, filterTest) { + const copy = suite._clone() + for (const entry of suite._entries) { + if (entry.constructor.name === 'Suite') { + copy._addSuite(deepCloneSuite(entry, filterTest)) + } else { + if (filterTest(entry)) { + const copiedTest = entry._clone() + copiedTest._ddIsNew = true + copiedTest._ddIsEfdRetry = true + copy._addTest(copiedTest) + } + } + } + return copy +} function getTestsBySuiteFromTestGroups (testGroups) { return testGroups.reduce((acc, { requireFile, tests }) => { @@ -73,14 +123,116 @@ function getRootDir (playwrightRunner) { if (playwrightRunner._configDir) { return playwrightRunner._configDir } - if (playwrightRunner._config && playwrightRunner._config.config) { - return playwrightRunner._config.config.rootDir + if (playwrightRunner._config) { + return playwrightRunner._config.config?.rootDir || process.cwd() } return process.cwd() } -function testBeginHandler (test) { - const { _requireFile: testSuiteAbsolutePath, title: testName, _type, location: { line: testSourceLine } } = test +function getProjectsFromRunner (runner) { + const config = getPlaywrightConfig(runner) + return config.projects?.map((project) => { + if (project.project) { + return project.project + } + return project + }) +} + +function getProjectsFromDispatcher (dispatcher) { + const newConfig = dispatcher._config?.config?.projects + if (newConfig) { + return newConfig + } + // old + return dispatcher._loader?.fullConfig()?.projects +} + +function getBrowserNameFromProjects (projects, test) { + if (!projects || !test) { + return null + } + const { _projectIndex, _projectId: testProjectId } = test + + if (_projectIndex !== undefined) { + return projects[_projectIndex]?.name + } + + return projects.find(({ __projectId, _id, name }) => { + if (__projectId !== undefined) { + return __projectId === testProjectId + } + if (_id !== undefined) { + return _id === testProjectId + } + return name === testProjectId + })?.name +} + +function formatTestHookError (error, hookType, isTimeout) { + let hookError = error + if (error) { + hookError.message = `Error in ${hookType} hook: ${error.message}` + } + if (!hookError && isTimeout) { + hookError = new Error(`${hookType} hook timed out`) + } + return hookError +} + +function addErrorToTestSuite (testSuiteAbsolutePath, error) { + if (testSuiteToErrors.has(testSuiteAbsolutePath)) { + testSuiteToErrors.get(testSuiteAbsolutePath).push(error) + } else { + testSuiteToErrors.set(testSuiteAbsolutePath, [error]) + } +} + +function getTestSuiteError (testSuiteAbsolutePath) { + const errors = testSuiteToErrors.get(testSuiteAbsolutePath) + if (!errors) { + return null + } + if (errors.length === 1) { + return errors[0] + } + return new Error(`${errors.length} errors in this test suite:\n${errors.map(e => e.message).join('\n------\n')}`) +} + +function getTestByTestId (dispatcher, testId) { + if (dispatcher._testById) { + return dispatcher._testById.get(testId)?.test + } + const allTests = dispatcher._allTests || dispatcher._ddAllTests + if (allTests) { + return allTests.find(({ id }) => id === testId) + } +} + +function getChannelPromise (channelToPublishTo) { + return new Promise(resolve => { + testSessionAsyncResource.runInAsyncScope(() => { + channelToPublishTo.publish({ onDone: resolve }) + }) + }) +} +// eslint-disable-next-line +// Inspired by https://github.com/microsoft/playwright/blob/2b77ed4d7aafa85a600caa0b0d101b72c8437eeb/packages/playwright/src/reporters/base.ts#L293 +// We can't use test.outcome() directly because it's set on follow up handlers: +// our `testEndHandler` is called before the outcome is set. +function testWillRetry (test, testStatus) { + return testStatus === 'fail' && test.results.length <= test.retries +} + +function testBeginHandler (test, browserName) { + const { + _requireFile: testSuiteAbsolutePath, + title: testName, + _type, + location: { + line: testSourceLine + } + } = test if (_type === 'beforeAll' || _type === 'afterAll') { return @@ -100,11 +252,11 @@ function testBeginHandler (test) { const testAsyncResource = new AsyncResource('bound-anonymous-fn') testToAr.set(test, testAsyncResource) testAsyncResource.runInAsyncScope(() => { - testStartCh.publish({ testName, testSuiteAbsolutePath, testSourceLine }) + testStartCh.publish({ testName, testSuiteAbsolutePath, testSourceLine, browserName }) }) } -function testEndHandler (test, annotations, testStatus, error) { +function testEndHandler (test, annotations, testStatus, error, isTimeout) { let annotationTags if (annotations.length) { annotationTags = parseAnnotations(annotations) @@ -112,24 +264,44 @@ function testEndHandler (test, annotations, testStatus, error) { const { _requireFile: testSuiteAbsolutePath, results, _type } = test if (_type === 'beforeAll' || _type === 'afterAll') { + const hookError = formatTestHookError(error, _type, isTimeout) + + if (hookError) { + addErrorToTestSuite(testSuiteAbsolutePath, hookError) + } return } const testResult = results[results.length - 1] const testAsyncResource = testToAr.get(test) testAsyncResource.runInAsyncScope(() => { - testFinishCh.publish({ testStatus, steps: testResult.steps, error, extraTags: annotationTags }) + testFinishCh.publish({ + testStatus, + steps: testResult?.steps || [], + isRetry: testResult?.retry > 0, + error, + extraTags: annotationTags, + isNew: test._ddIsNew, + isEfdRetry: test._ddIsEfdRetry + }) }) - if (!testSuiteToTestStatuses.has(testSuiteAbsolutePath)) { - testSuiteToTestStatuses.set(testSuiteAbsolutePath, [testStatus]) - } else { + if (testSuiteToTestStatuses.has(testSuiteAbsolutePath)) { testSuiteToTestStatuses.get(testSuiteAbsolutePath).push(testStatus) + } else { + testSuiteToTestStatuses.set(testSuiteAbsolutePath, [testStatus]) + } + + if (error) { + addErrorToTestSuite(testSuiteAbsolutePath, error) } - remainingTestsByFile[testSuiteAbsolutePath] = remainingTestsByFile[testSuiteAbsolutePath] - .filter(currentTest => currentTest !== test) + if (!testWillRetry(test, testStatus)) { + remainingTestsByFile[testSuiteAbsolutePath] = remainingTestsByFile[testSuiteAbsolutePath] + .filter(currentTest => currentTest !== test) + } + // Last test, we finish the suite if (!remainingTestsByFile[testSuiteAbsolutePath].length) { const testStatuses = testSuiteToTestStatuses.get(testSuiteAbsolutePath) @@ -140,9 +312,10 @@ function testEndHandler (test, annotations, testStatus, error) { testSuiteStatus = 'skip' } + const suiteError = getTestSuiteError(testSuiteAbsolutePath) const testSuiteAsyncResource = testSuiteToAr.get(testSuiteAbsolutePath) testSuiteAsyncResource.runInAsyncScope(() => { - testSuiteFinishCh.publish(testSuiteStatus) + testSuiteFinishCh.publish({ status: testSuiteStatus, error: suiteError }) }) } } @@ -155,7 +328,12 @@ function dispatcherRunWrapper (run) { } function dispatcherRunWrapperNew (run) { - return function () { + return function (testGroups) { + if (!this._allTests) { + // Removed in https://github.com/microsoft/playwright/commit/1e52c37b254a441cccf332520f60225a5acc14c7 + // Not available from >=1.44.0 + this._ddAllTests = testGroups.map(g => g.tests).flat() + } remainingTestsByFile = getTestsBySuiteFromTestGroups(arguments[0]) return run.apply(this, arguments) } @@ -166,18 +344,20 @@ function dispatcherHook (dispatcherExport) { shimmer.wrap(dispatcherExport.Dispatcher.prototype, '_createWorker', createWorker => function () { const dispatcher = this const worker = createWorker.apply(this, arguments) - worker.process.on('message', ({ method, params }) => { if (method === 'testBegin') { const { test } = dispatcher._testById.get(params.testId) - testBeginHandler(test) + const projects = getProjectsFromDispatcher(dispatcher) + const browser = getBrowserNameFromProjects(projects, test) + testBeginHandler(test, browser) } else if (method === 'testEnd') { const { test } = dispatcher._testById.get(params.testId) const { results } = test const testResult = results[results.length - 1] - testEndHandler(test, params.annotations, STATUS_TO_TEST_STATUS[testResult.status], testResult.error) + const isTimeout = testResult.status === 'timedOut' + testEndHandler(test, params.annotations, STATUS_TO_TEST_STATUS[testResult.status], testResult.error, isTimeout) } }) @@ -186,15 +366,6 @@ function dispatcherHook (dispatcherExport) { return dispatcherExport } -function getTestByTestId (dispatcher, testId) { - if (dispatcher._testById) { - return dispatcher._testById.get(testId)?.test - } - if (dispatcher._allTests) { - return dispatcher._allTests.find(({ id }) => id === testId) - } -} - function dispatcherHookNew (dispatcherExport, runWrapper) { shimmer.wrap(dispatcherExport.Dispatcher.prototype, 'run', runWrapper) shimmer.wrap(dispatcherExport.Dispatcher.prototype, '_createWorker', createWorker => function () { @@ -203,12 +374,15 @@ function dispatcherHookNew (dispatcherExport, runWrapper) { worker.on('testBegin', ({ testId }) => { const test = getTestByTestId(dispatcher, testId) - testBeginHandler(test) + const projects = getProjectsFromDispatcher(dispatcher) + const browser = getBrowserNameFromProjects(projects, test) + testBeginHandler(test, browser) }) worker.on('testEnd', ({ testId, status, errors, annotations }) => { const test = getTestByTestId(dispatcher, testId) - testEndHandler(test, annotations, STATUS_TO_TEST_STATUS[status], errors && errors[0]) + const isTimeout = status === 'timedOut' + testEndHandler(test, annotations, STATUS_TO_TEST_STATUS[status], errors && errors[0], isTimeout) }) return worker @@ -218,8 +392,9 @@ function dispatcherHookNew (dispatcherExport, runWrapper) { function runnerHook (runnerExport, playwrightVersion) { shimmer.wrap(runnerExport.Runner.prototype, 'runAllTests', runAllTests => async function () { - const testSessionAsyncResource = new AsyncResource('bound-anonymous-fn') - const rootDir = getRootDir(this) + let onDone + + rootDir = getRootDir(this) const processArgv = process.argv.slice(2).join(' ') const command = `playwright ${processArgv}` @@ -227,6 +402,43 @@ function runnerHook (runnerExport, playwrightVersion) { testSessionStartCh.publish({ command, frameworkVersion: playwrightVersion, rootDir }) }) + try { + const { err, libraryConfig } = await getChannelPromise(libraryConfigurationCh) + if (!err) { + isEarlyFlakeDetectionEnabled = libraryConfig.isEarlyFlakeDetectionEnabled + earlyFlakeDetectionNumRetries = libraryConfig.earlyFlakeDetectionNumRetries + isFlakyTestRetriesEnabled = libraryConfig.isFlakyTestRetriesEnabled + flakyTestRetriesCount = libraryConfig.flakyTestRetriesCount + } + } catch (e) { + isEarlyFlakeDetectionEnabled = false + log.error(e) + } + + if (isEarlyFlakeDetectionEnabled && semver.gte(playwrightVersion, MINIMUM_SUPPORTED_VERSION_EFD)) { + try { + const { err, knownTests: receivedKnownTests } = await getChannelPromise(knownTestsCh) + if (!err) { + knownTests = receivedKnownTests + } else { + isEarlyFlakeDetectionEnabled = false + } + } catch (err) { + isEarlyFlakeDetectionEnabled = false + log.error(err) + } + } + + const projects = getProjectsFromRunner(this) + + if (isFlakyTestRetriesEnabled && flakyTestRetriesCount > 0) { + projects.forEach(project => { + if (project.retries === 0) { // Only if it hasn't been set by the user + project.retries = flakyTestRetriesCount + } + }) + } + const runAllTestsReturn = await runAllTests.apply(this, arguments) Object.values(remainingTestsByFile).forEach(tests => { @@ -234,19 +446,23 @@ function runnerHook (runnerExport, playwrightVersion) { // there were tests that did not go through `testBegin` or `testEnd`, // because they were skipped tests.forEach(test => { - testBeginHandler(test) + const browser = getBrowserNameFromProjects(projects, test) + testBeginHandler(test, browser) testEndHandler(test, [], 'skip') }) }) const sessionStatus = runAllTestsReturn.status || runAllTestsReturn - let onDone const flushWait = new Promise(resolve => { onDone = resolve }) testSessionAsyncResource.runInAsyncScope(() => { - testSessionFinishCh.publish({ status: STATUS_TO_TEST_STATUS[sessionStatus], onDone }) + testSessionFinishCh.publish({ + status: STATUS_TO_TEST_STATUS[sessionStatus], + isEarlyFlakeDetectionEnabled, + onDone + }) }) await flushWait @@ -295,8 +511,59 @@ addHook({ file: 'lib/runner/runner.js', versions: ['>=1.38.0'] }, runnerHook) + addHook({ name: 'playwright', file: 'lib/runner/dispatcher.js', versions: ['>=1.38.0'] }, (dispatcher) => dispatcherHookNew(dispatcher, dispatcherRunWrapperNew)) + +// Hook used for early flake detection. EFD only works from >=1.38.0 +addHook({ + name: 'playwright', + file: 'lib/common/suiteUtils.js', + versions: [`>=${MINIMUM_SUPPORTED_VERSION_EFD}`] +}, suiteUtilsPackage => { + // We grab `applyRepeatEachIndex` to use it later + // `applyRepeatEachIndex` needs to be applied to a cloned suite + applyRepeatEachIndex = suiteUtilsPackage.applyRepeatEachIndex + return suiteUtilsPackage +}) + +// Hook used for early flake detection. EFD only works from >=1.38.0 +addHook({ + name: 'playwright', + file: 'lib/runner/loadUtils.js', + versions: [`>=${MINIMUM_SUPPORTED_VERSION_EFD}`] +}, (loadUtilsPackage) => { + const oldCreateRootSuite = loadUtilsPackage.createRootSuite + + async function newCreateRootSuite () { + const rootSuite = await oldCreateRootSuite.apply(this, arguments) + if (!isEarlyFlakeDetectionEnabled) { + return rootSuite + } + const newTests = rootSuite + .allTests() + .filter(isNewTest) + + newTests.forEach(newTest => { + newTest._ddIsNew = true + if (newTest.expectedStatus !== 'skipped') { + const fileSuite = getSuiteType(newTest, 'file') + const projectSuite = getSuiteType(newTest, 'project') + for (let repeatEachIndex = 0; repeatEachIndex < earlyFlakeDetectionNumRetries; repeatEachIndex++) { + const copyFileSuite = deepCloneSuite(fileSuite, isNewTest) + applyRepeatEachIndex(projectSuite._fullProject, copyFileSuite, repeatEachIndex + 1) + projectSuite._addSuite(copyFileSuite) + } + } + }) + + return rootSuite + } + + loadUtilsPackage.createRootSuite = newCreateRootSuite + + return loadUtilsPackage +}) diff --git a/packages/datadog-instrumentations/src/process.js b/packages/datadog-instrumentations/src/process.js new file mode 100644 index 00000000000..429b0d8f574 --- /dev/null +++ b/packages/datadog-instrumentations/src/process.js @@ -0,0 +1,29 @@ +'use strict' + +const shimmer = require('../../datadog-shimmer') +const { channel } = require('dc-polyfill') + +const startSetUncaughtExceptionCaptureCallback = channel('datadog:process:setUncaughtExceptionCaptureCallback:start') + +if (process.setUncaughtExceptionCaptureCallback) { + let currentCallback + + shimmer.wrap(process, 'setUncaughtExceptionCaptureCallback', + function wrapSetUncaughtExceptionCaptureCallback (originalSetUncaughtExceptionCaptureCallback) { + return function setUncaughtExceptionCaptureCallback (newCallback) { + if (startSetUncaughtExceptionCaptureCallback.hasSubscribers) { + const abortController = new AbortController() + startSetUncaughtExceptionCaptureCallback.publish({ newCallback, currentCallback, abortController }) + if (abortController.signal.aborted) { + return + } + } + + const result = originalSetUncaughtExceptionCaptureCallback.apply(this, arguments) + + currentCallback = newCallback + + return result + } + }) +} diff --git a/packages/datadog-instrumentations/src/protobufjs.js b/packages/datadog-instrumentations/src/protobufjs.js new file mode 100644 index 00000000000..79cbb4ee3a1 --- /dev/null +++ b/packages/datadog-instrumentations/src/protobufjs.js @@ -0,0 +1,127 @@ +const shimmer = require('../../datadog-shimmer') +const { addHook } = require('./helpers/instrument') + +const dc = require('dc-polyfill') +const serializeChannel = dc.channel('apm:protobufjs:serialize-start') +const deserializeChannel = dc.channel('apm:protobufjs:deserialize-end') + +function wrapSerialization (messageClass) { + if (messageClass?.encode) { + shimmer.wrap(messageClass, 'encode', original => function () { + if (!serializeChannel.hasSubscribers) { + return original.apply(this, arguments) + } + serializeChannel.publish({ messageClass: this }) + return original.apply(this, arguments) + }) + } +} + +function wrapDeserialization (messageClass) { + if (messageClass?.decode) { + shimmer.wrap(messageClass, 'decode', original => function () { + if (!deserializeChannel.hasSubscribers) { + return original.apply(this, arguments) + } + const result = original.apply(this, arguments) + deserializeChannel.publish({ messageClass: result }) + return result + }) + } +} + +function wrapSetup (messageClass) { + if (messageClass?.setup) { + shimmer.wrap(messageClass, 'setup', original => function () { + const result = original.apply(this, arguments) + + wrapSerialization(messageClass) + wrapDeserialization(messageClass) + + return result + }) + } +} + +function wrapProtobufClasses (root) { + if (!root) { + return + } + + if (root.decode) { + wrapSetup(root) + } + + if (root.nestedArray) { + for (const subRoot of root.nestedArray) { + wrapProtobufClasses(subRoot) + } + } +} + +function wrapReflection (protobuf) { + const reflectionMethods = [ + { + target: protobuf.Root, + name: 'fromJSON' + }, + { + target: protobuf.Type.prototype, + name: 'fromObject' + } + ] + + reflectionMethods.forEach(method => { + shimmer.wrap(method.target, method.name, original => function () { + const result = original.apply(this, arguments) + if (result.nested) { + for (const type in result.nested) { + wrapSetup(result.nested[type]) + } + } + if (result.$type) { + wrapSetup(result.$type) + } + return result + }) + }) +} + +function isPromise (obj) { + return !!obj && (typeof obj === 'object' || typeof obj === 'function') && typeof obj.then === 'function' +} + +addHook({ + name: 'protobufjs', + versions: ['>=6.8.0'] +}, protobuf => { + shimmer.wrap(protobuf.Root.prototype, 'load', original => function () { + const result = original.apply(this, arguments) + if (isPromise(result)) { + return result.then(root => { + wrapProtobufClasses(root) + return root + }) + } else { + // If result is not a promise, directly wrap the protobuf classes + wrapProtobufClasses(result) + return result + } + }) + + shimmer.wrap(protobuf.Root.prototype, 'loadSync', original => function () { + const root = original.apply(this, arguments) + wrapProtobufClasses(root) + return root + }) + + shimmer.wrap(protobuf, 'Type', Original => function () { + const typeInstance = new Original(...arguments) + wrapSetup(typeInstance) + return typeInstance + }) + + wrapReflection(protobuf) + + return protobuf +}) diff --git a/packages/datadog-instrumentations/src/redis.js b/packages/datadog-instrumentations/src/redis.js index 8d2365e86a2..8da93ae08ab 100644 --- a/packages/datadog-instrumentations/src/redis.js +++ b/packages/datadog-instrumentations/src/redis.js @@ -156,12 +156,12 @@ function start (client, command, args, url = {}) { } function wrapCallback (finishCh, errorCh, callback) { - return function (err) { + return shimmer.wrapFunction(callback, callback => function (err) { finish(finishCh, errorCh, err) if (callback) { return callback.apply(this, arguments) } - } + }) } function finish (finishCh, errorCh, error) { diff --git a/packages/datadog-instrumentations/src/restify.js b/packages/datadog-instrumentations/src/restify.js index 2644f916b3f..b21689fa734 100644 --- a/packages/datadog-instrumentations/src/restify.js +++ b/packages/datadog-instrumentations/src/restify.js @@ -40,7 +40,7 @@ function wrapMiddleware (middleware) { function wrapFn (fn) { if (Array.isArray(fn)) return wrapMiddleware(fn) - return function (req, res, next) { + return shimmer.wrapFunction(fn, fn => function (req, res, next) { if (typeof next === 'function') { arguments[2] = wrapNext(req, next) } @@ -51,11 +51,11 @@ function wrapFn (fn) { try { const result = fn.apply(this, arguments) - if (result && typeof result === 'object' && typeof result.then === 'function') { + if (result !== null && typeof result === 'object' && typeof result.then === 'function') { return result.then(function () { nextChannel.publish({ req }) finishChannel.publish({ req }) - return arguments + return arguments[0] }).catch(function (error) { errorChannel.publish({ req, error }) nextChannel.publish({ req }) @@ -72,16 +72,16 @@ function wrapFn (fn) { } finally { exitChannel.publish({ req }) } - } + }) } function wrapNext (req, next) { - return function () { + return shimmer.wrapFunction(next, next => function () { nextChannel.publish({ req }) finishChannel.publish({ req }) next.apply(this, arguments) - } + }) } addHook({ name: 'restify', versions: ['>=3'], file: 'lib/server.js' }, Server => { diff --git a/packages/datadog-instrumentations/src/rhea.js b/packages/datadog-instrumentations/src/rhea.js index 955b72b67b2..205e8730460 100644 --- a/packages/datadog-instrumentations/src/rhea.js +++ b/packages/datadog-instrumentations/src/rhea.js @@ -22,7 +22,7 @@ const dispatchReceiveCh = channel('apm:rhea:receive:dispatch') const errorReceiveCh = channel('apm:rhea:receive:error') const finishReceiveCh = channel('apm:rhea:receive:finish') -const contexts = new WeakMap() +const contexts = new WeakMap() // key: delivery Fn, val: context addHook({ name: 'rhea', versions: ['>=1'] }, rhea => { shimmer.wrap(rhea.message, 'encode', encode => function (msg) { @@ -45,14 +45,17 @@ addHook({ name: 'rhea', versions: ['>=1'], file: 'lib/link.js' }, obj => { const { host, port } = getHostAndPort(this.connection) const targetAddress = this.options && this.options.target && - this.options.target.address ? this.options.target.address : undefined + this.options.target.address + ? this.options.target.address + : undefined const asyncResource = new AsyncResource('bound-anonymous-fn') return asyncResource.runInAsyncScope(() => { startSendCh.publish({ targetAddress, host, port, msg }) const delivery = send.apply(this, arguments) const context = { - asyncResource + asyncResource, + connection: this.connection } contexts.set(delivery, context) @@ -80,7 +83,8 @@ addHook({ name: 'rhea', versions: ['>=1'], file: 'lib/link.js' }, obj => { if (msgObj.delivery) { const context = { - asyncResource + asyncResource, + connection: this.connection } contexts.set(msgObj.delivery, context) msgObj.delivery.update = wrapDeliveryUpdate(msgObj.delivery, msgObj.delivery.update) @@ -114,7 +118,7 @@ addHook({ name: 'rhea', versions: ['>=1'], file: 'lib/connection.js' }, Connecti asyncResource.runInAsyncScope(() => { errorReceiveCh.publish(error) - beforeFinish(delivery, null) + exports.beforeFinish(delivery, null) finishReceiveCh.publish() }) }) @@ -149,11 +153,11 @@ function wrapDeliveryUpdate (obj, update) { const asyncResource = context.asyncResource if (obj && asyncResource) { const cb = asyncResource.bind(update) - return AsyncResource.bind(function wrappedUpdate (settled, stateData) { + return shimmer.wrapFunction(cb, cb => AsyncResource.bind(function wrappedUpdate (settled, stateData) { const state = getStateFromData(stateData) dispatchReceiveCh.publish({ state }) return cb.apply(this, arguments) - }) + })) } return function wrappedUpdate (settled, stateData) { return update.apply(this, arguments) @@ -174,7 +178,7 @@ function patchCircularBuffer (proto, Session) { } if (CircularBuffer && !patched.has(CircularBuffer.prototype)) { shimmer.wrap(CircularBuffer.prototype, 'pop_if', popIf => function (fn) { - arguments[0] = AsyncResource.bind(function (entry) { + arguments[0] = shimmer.wrapFunction(fn, fn => AsyncResource.bind(function (entry) { const context = contexts.get(entry) const asyncResource = context && context.asyncResource @@ -185,15 +189,16 @@ function patchCircularBuffer (proto, Session) { if (shouldPop) { const remoteState = entry.remote_state const state = remoteState && remoteState.constructor - ? entry.remote_state.constructor.composite_type : undefined + ? entry.remote_state.constructor.composite_type + : undefined asyncResource.runInAsyncScope(() => { - beforeFinish(entry, state) + exports.beforeFinish(entry, state) finishSendCh.publish() }) } return shouldPop - }) + })) return popIf.apply(this, arguments) }) patched.add(CircularBuffer.prototype) @@ -217,13 +222,13 @@ function addToInFlightDeliveries (connection, delivery) { } function beforeFinish (delivery, state) { - const obj = contexts.get(delivery) - if (obj) { + const context = contexts.get(delivery) + if (context) { if (state) { dispatchReceiveCh.publish({ state }) } - if (obj.connection && obj.connection[inFlightDeliveries]) { - obj.connection[inFlightDeliveries].delete(delivery) + if (context.connection && context.connection[inFlightDeliveries]) { + context.connection[inFlightDeliveries].delete(delivery) } } } @@ -238,3 +243,7 @@ function getStateFromData (stateData) { } } } + +module.exports.inFlightDeliveries = inFlightDeliveries +module.exports.beforeFinish = beforeFinish +module.exports.contexts = contexts diff --git a/packages/datadog-instrumentations/src/router.js b/packages/datadog-instrumentations/src/router.js index 9ac38caf6c6..cdd08f9f539 100644 --- a/packages/datadog-instrumentations/src/router.js +++ b/packages/datadog-instrumentations/src/router.js @@ -1,6 +1,6 @@ 'use strict' -const METHODS = require('methods').concat('all') +const METHODS = require('http').METHODS.map(v => v.toLowerCase()).concat('all') const pathToRegExp = require('path-to-regexp') const shimmer = require('../../datadog-shimmer') const { addHook, channel } = require('./helpers/instrument') @@ -18,7 +18,7 @@ function createWrapRouterMethod (name) { function wrapLayerHandle (layer, original) { original._name = original._name || layer.name - const handle = shimmer.wrap(original, function () { + const handle = shimmer.wrapFunction(original, original => function () { if (!enterChannel.hasSubscribers) return original.apply(this, arguments) const matchers = layerMatchers.get(layer) @@ -89,7 +89,7 @@ function createWrapRouterMethod (name) { } function wrapNext (req, next) { - return function (error) { + return shimmer.wrapFunction(next, next => function (error) { if (error && error !== 'route' && error !== 'router') { errorChannel.publish({ req, error }) } @@ -98,7 +98,7 @@ function createWrapRouterMethod (name) { finishChannel.publish({ req }) next.apply(this, arguments) - } + }) } function extractMatchers (fn) { @@ -151,7 +151,7 @@ function createWrapRouterMethod (name) { } function wrapMethod (original) { - return function methodWithTrace (fn) { + return shimmer.wrapFunction(original, original => function methodWithTrace (fn) { const offset = this.stack ? [].concat(this.stack).length : 0 const router = original.apply(this, arguments) @@ -162,7 +162,7 @@ function createWrapRouterMethod (name) { wrapStack(this.stack, offset, extractMatchers(fn)) return router - } + }) } return wrapMethod diff --git a/packages/datadog-instrumentations/src/selenium.js b/packages/datadog-instrumentations/src/selenium.js new file mode 100644 index 00000000000..141aa967e40 --- /dev/null +++ b/packages/datadog-instrumentations/src/selenium.js @@ -0,0 +1,76 @@ +const { addHook, channel } = require('./helpers/instrument') +const shimmer = require('../../datadog-shimmer') + +const ciSeleniumDriverGetStartCh = channel('ci:selenium:driver:get') + +const RUM_STOP_SESSION_SCRIPT = ` +if (window.DD_RUM && window.DD_RUM.stopSession) { + window.DD_RUM.stopSession(); + return true; +} else { + return false; +} +` +const IS_RUM_ACTIVE_SCRIPT = 'return !!window.DD_RUM' + +const DD_CIVISIBILITY_RUM_FLUSH_WAIT_MILLIS = 500 +const DD_CIVISIBILITY_TEST_EXECUTION_ID_COOKIE_NAME = 'datadog-ci-visibility-test-execution-id' + +// TODO: can we increase the supported version range? +addHook({ + name: 'selenium-webdriver', + versions: ['>=4.11.0'] +}, (seleniumPackage, seleniumVersion) => { + // TODO: do not turn this into async. Use promises + shimmer.wrap(seleniumPackage.WebDriver.prototype, 'get', get => async function () { + if (!ciSeleniumDriverGetStartCh.hasSubscribers) { + return get.apply(this, arguments) + } + let traceId + const setTraceId = (inputTraceId) => { + traceId = inputTraceId + } + const getResult = await get.apply(this, arguments) + + const isRumActive = await this.executeScript(IS_RUM_ACTIVE_SCRIPT) + const capabilities = await this.getCapabilities() + + ciSeleniumDriverGetStartCh.publish({ + setTraceId, + seleniumVersion, + browserName: capabilities.getBrowserName(), + browserVersion: capabilities.getBrowserVersion(), + isRumActive + }) + + if (traceId && isRumActive) { + await this.manage().addCookie({ + name: DD_CIVISIBILITY_TEST_EXECUTION_ID_COOKIE_NAME, + value: traceId + }) + } + + return getResult + }) + + shimmer.wrap(seleniumPackage.WebDriver.prototype, 'quit', quit => async function () { + if (!ciSeleniumDriverGetStartCh.hasSubscribers) { + return quit.apply(this, arguments) + } + const isRumActive = await this.executeScript(RUM_STOP_SESSION_SCRIPT) + + if (isRumActive) { + // We'll have time for RUM to flush the events (there's no callback to know when it's done) + await new Promise(resolve => { + setTimeout(() => { + resolve() + }, DD_CIVISIBILITY_RUM_FLUSH_WAIT_MILLIS) + }) + await this.manage().deleteCookie(DD_CIVISIBILITY_TEST_EXECUTION_ID_COOKIE_NAME) + } + + return quit.apply(this, arguments) + }) + + return seleniumPackage +}) diff --git a/packages/datadog-instrumentations/src/sharedb.js b/packages/datadog-instrumentations/src/sharedb.js index fb5ad60bd4c..f0851874de4 100644 --- a/packages/datadog-instrumentations/src/sharedb.js +++ b/packages/datadog-instrumentations/src/sharedb.js @@ -48,14 +48,14 @@ addHook({ name: 'sharedb', versions: ['>=1'], file: 'lib/agent.js' }, Agent => { callback = callbackResource.bind(callback) - arguments[1] = asyncResource.bind(function (error, res) { + arguments[1] = shimmer.wrapFunction(callback, callback => asyncResource.bind(function (error, res) { if (error) { errorCh.publish(error) } finishCh.publish({ request, res }) return callback.apply(this, arguments) - }) + })) try { return origHandleMessageFn.apply(this, arguments) diff --git a/packages/datadog-instrumentations/src/tedious.js b/packages/datadog-instrumentations/src/tedious.js index f1094a6d30f..3a80b03ad75 100644 --- a/packages/datadog-instrumentations/src/tedious.js +++ b/packages/datadog-instrumentations/src/tedious.js @@ -7,7 +7,7 @@ const { } = require('./helpers/instrument') const shimmer = require('../../datadog-shimmer') -addHook({ name: 'tedious', versions: [ '>=1.0.0' ] }, tedious => { +addHook({ name: 'tedious', versions: ['>=1.0.0'] }, tedious => { const startCh = channel('apm:tedious:request:start') const finishCh = channel('apm:tedious:request:finish') const errorCh = channel('apm:tedious:request:error') diff --git a/packages/datadog-instrumentations/src/undici.js b/packages/datadog-instrumentations/src/undici.js new file mode 100644 index 00000000000..cd3207ea9c3 --- /dev/null +++ b/packages/datadog-instrumentations/src/undici.js @@ -0,0 +1,18 @@ +'use strict' + +const { + addHook +} = require('./helpers/instrument') +const shimmer = require('../../datadog-shimmer') + +const tracingChannel = require('dc-polyfill').tracingChannel +const ch = tracingChannel('apm:undici:fetch') + +const { createWrapFetch } = require('./helpers/fetch') + +addHook({ + name: 'undici', + versions: ['^4.4.1', '5', '>=6.0.0'] +}, undici => { + return shimmer.wrap(undici, 'fetch', createWrapFetch(undici.Request, ch)) +}) diff --git a/packages/datadog-instrumentations/src/utils/src/extract-package-and-module-path.js b/packages/datadog-instrumentations/src/utils/src/extract-package-and-module-path.js new file mode 100644 index 00000000000..7a48565e379 --- /dev/null +++ b/packages/datadog-instrumentations/src/utils/src/extract-package-and-module-path.js @@ -0,0 +1,36 @@ +'use strict' + +const NM = 'node_modules/' + +/** + * For a given full path to a module, + * return the package name it belongs to and the local path to the module + * input: '/foo/node_modules/@co/stuff/foo/bar/baz.js' + * output: { pkg: '@co/stuff', path: 'foo/bar/baz.js', pkgJson: '/foo/node_modules/@co/stuff/package.json' } + */ +module.exports = function extractPackageAndModulePath (fullPath) { + const nm = fullPath.lastIndexOf(NM) + if (nm < 0) { + return { pkg: null, path: null } + } + + const subPath = fullPath.substring(nm + NM.length) + const firstSlash = subPath.indexOf('/') + + const firstPath = fullPath.substring(fullPath[0], nm + NM.length) + + if (subPath[0] === '@') { + const secondSlash = subPath.substring(firstSlash + 1).indexOf('/') + return { + pkg: subPath.substring(0, firstSlash + 1 + secondSlash), + path: subPath.substring(firstSlash + 1 + secondSlash + 1), + pkgJson: firstPath + subPath.substring(0, firstSlash + 1 + secondSlash) + '/package.json' + } + } + + return { + pkg: subPath.substring(0, firstSlash), + path: subPath.substring(firstSlash + 1), + pkgJson: firstPath + subPath.substring(0, firstSlash) + '/package.json' + } +} diff --git a/packages/datadog-instrumentations/src/vitest.js b/packages/datadog-instrumentations/src/vitest.js new file mode 100644 index 00000000000..f0117e0e8c0 --- /dev/null +++ b/packages/datadog-instrumentations/src/vitest.js @@ -0,0 +1,565 @@ +const { addHook, channel, AsyncResource } = require('./helpers/instrument') +const shimmer = require('../../datadog-shimmer') +const log = require('../../dd-trace/src/log') + +// test hooks +const testStartCh = channel('ci:vitest:test:start') +const testFinishTimeCh = channel('ci:vitest:test:finish-time') +const testPassCh = channel('ci:vitest:test:pass') +const testErrorCh = channel('ci:vitest:test:error') +const testSkipCh = channel('ci:vitest:test:skip') +const isNewTestCh = channel('ci:vitest:test:is-new') + +// test suite hooks +const testSuiteStartCh = channel('ci:vitest:test-suite:start') +const testSuiteFinishCh = channel('ci:vitest:test-suite:finish') +const testSuiteErrorCh = channel('ci:vitest:test-suite:error') + +// test session hooks +const testSessionStartCh = channel('ci:vitest:session:start') +const testSessionFinishCh = channel('ci:vitest:session:finish') +const libraryConfigurationCh = channel('ci:vitest:library-configuration') +const knownTestsCh = channel('ci:vitest:known-tests') +const isEarlyFlakeDetectionFaultyCh = channel('ci:vitest:is-early-flake-detection-faulty') + +const taskToAsync = new WeakMap() +const taskToStatuses = new WeakMap() +const newTasks = new WeakSet() +const switchedStatuses = new WeakSet() +const sessionAsyncResource = new AsyncResource('bound-anonymous-fn') + +function isReporterPackage (vitestPackage) { + return vitestPackage.B?.name === 'BaseSequencer' +} + +// from 2.0.0 +function isReporterPackageNew (vitestPackage) { + return vitestPackage.e?.name === 'BaseSequencer' +} + +function isReporterPackageNewest (vitestPackage) { + return vitestPackage.h?.name === 'BaseSequencer' +} + +function getChannelPromise (channelToPublishTo) { + return new Promise(resolve => { + sessionAsyncResource.runInAsyncScope(() => { + channelToPublishTo.publish({ onDone: resolve }) + }) + }) +} + +function getSessionStatus (state) { + if (state.getCountOfFailedTests() > 0) { + return 'fail' + } + if (state.pathsSet.size === 0) { + return 'skip' + } + return 'pass' +} + +// eslint-disable-next-line +// From https://github.com/vitest-dev/vitest/blob/51c04e2f44d91322b334f8ccbcdb368facc3f8ec/packages/runner/src/run.ts#L243-L250 +function getVitestTestStatus (test, retryCount) { + if (test.result.state !== 'fail') { + if (!test.repeats) { + return 'pass' + } else if (test.repeats && (test.retry ?? 0) === retryCount) { + return 'pass' + } + } + return 'fail' +} + +function getTypeTasks (fileTasks, type = 'test') { + const typeTasks = [] + + function getTasks (tasks) { + for (const task of tasks) { + if (task.type === type) { + typeTasks.push(task) + } else if (task.tasks) { + getTasks(task.tasks) + } + } + } + + getTasks(fileTasks) + + return typeTasks +} + +function getTestName (task) { + let testName = task.name + let currentTask = task.suite + + while (currentTask) { + if (currentTask.name) { + testName = `${currentTask.name} ${testName}` + } + currentTask = currentTask.suite + } + + return testName +} + +function getSortWrapper (sort) { + return async function () { + if (!testSessionFinishCh.hasSubscribers) { + return sort.apply(this, arguments) + } + // There isn't any other async function that we seem to be able to hook into + // So we will use the sort from BaseSequencer. This means that a custom sequencer + // will not work. This will be a known limitation. + let isFlakyTestRetriesEnabled = false + let flakyTestRetriesCount = 0 + let isEarlyFlakeDetectionEnabled = false + let earlyFlakeDetectionNumRetries = 0 + let isEarlyFlakeDetectionFaulty = false + let knownTests = {} + + try { + const { err, libraryConfig } = await getChannelPromise(libraryConfigurationCh) + if (!err) { + isFlakyTestRetriesEnabled = libraryConfig.isFlakyTestRetriesEnabled + flakyTestRetriesCount = libraryConfig.flakyTestRetriesCount + isEarlyFlakeDetectionEnabled = libraryConfig.isEarlyFlakeDetectionEnabled + earlyFlakeDetectionNumRetries = libraryConfig.earlyFlakeDetectionNumRetries + } + } catch (e) { + isFlakyTestRetriesEnabled = false + isEarlyFlakeDetectionEnabled = false + } + + if (isFlakyTestRetriesEnabled && !this.ctx.config.retry && flakyTestRetriesCount > 0) { + this.ctx.config.retry = flakyTestRetriesCount + } + + if (isEarlyFlakeDetectionEnabled) { + const knownTestsResponse = await getChannelPromise(knownTestsCh) + if (!knownTestsResponse.err) { + knownTests = knownTestsResponse.knownTests + const testFilepaths = await this.ctx.getTestFilepaths() + + isEarlyFlakeDetectionFaultyCh.publish({ + knownTests: knownTests.vitest || {}, + testFilepaths, + onDone: (isFaulty) => { + isEarlyFlakeDetectionFaulty = isFaulty + } + }) + if (isEarlyFlakeDetectionFaulty) { + isEarlyFlakeDetectionEnabled = false + log.warn('Early flake detection is disabled because the number of new tests is too high.') + } else { + // TODO: use this to pass session and module IDs to the worker, instead of polluting process.env + // Note: setting this.ctx.config.provide directly does not work because it's cached + try { + const workspaceProject = this.ctx.getCoreWorkspaceProject() + workspaceProject._provided._ddKnownTests = knownTests.vitest + workspaceProject._provided._ddIsEarlyFlakeDetectionEnabled = isEarlyFlakeDetectionEnabled + workspaceProject._provided._ddEarlyFlakeDetectionNumRetries = earlyFlakeDetectionNumRetries + } catch (e) { + log.warn('Could not send known tests to workers so Early Flake Detection will not work.') + } + } + } else { + isEarlyFlakeDetectionEnabled = false + } + } + + let testCodeCoverageLinesTotal + + if (this.ctx.coverageProvider?.generateCoverage) { + shimmer.wrap(this.ctx.coverageProvider, 'generateCoverage', generateCoverage => async function () { + const totalCodeCoverage = await generateCoverage.apply(this, arguments) + + try { + testCodeCoverageLinesTotal = totalCodeCoverage.getCoverageSummary().lines.pct + } catch (e) { + // ignore errors + } + return totalCodeCoverage + }) + } + + shimmer.wrap(this.ctx, 'exit', exit => async function () { + let onFinish + + const flushPromise = new Promise(resolve => { + onFinish = resolve + }) + const failedSuites = this.state.getFailedFilepaths() + let error + if (failedSuites.length) { + error = new Error(`Test suites failed: ${failedSuites.length}.`) + } + + sessionAsyncResource.runInAsyncScope(() => { + testSessionFinishCh.publish({ + status: getSessionStatus(this.state), + testCodeCoverageLinesTotal, + error, + isEarlyFlakeDetectionEnabled, + isEarlyFlakeDetectionFaulty, + onFinish + }) + }) + + await flushPromise + + return exit.apply(this, arguments) + }) + + return sort.apply(this, arguments) + } +} + +function getCreateCliWrapper (vitestPackage, frameworkVersion) { + shimmer.wrap(vitestPackage, 'c', oldCreateCli => function () { + if (!testSessionStartCh.hasSubscribers) { + return oldCreateCli.apply(this, arguments) + } + sessionAsyncResource.runInAsyncScope(() => { + const processArgv = process.argv.slice(2).join(' ') + testSessionStartCh.publish({ command: `vitest ${processArgv}`, frameworkVersion }) + }) + return oldCreateCli.apply(this, arguments) + }) + + return vitestPackage +} + +addHook({ + name: 'vitest', + versions: ['>=1.6.0'], + file: 'dist/runners.js' +}, (vitestPackage) => { + const { VitestTestRunner } = vitestPackage + + // `onBeforeRunTask` is run before any repetition or attempt is run + shimmer.wrap(VitestTestRunner.prototype, 'onBeforeRunTask', onBeforeRunTask => async function (task) { + const testName = getTestName(task) + try { + const { + _ddKnownTests: knownTests, + _ddIsEarlyFlakeDetectionEnabled: isEarlyFlakeDetectionEnabled, + _ddEarlyFlakeDetectionNumRetries: numRepeats + } = globalThis.__vitest_worker__.providedContext + + if (isEarlyFlakeDetectionEnabled) { + isNewTestCh.publish({ + knownTests, + testSuiteAbsolutePath: task.file.filepath, + testName, + onDone: (isNew) => { + if (isNew) { + task.repeats = numRepeats + newTasks.add(task) + taskToStatuses.set(task, []) + } + } + }) + } + } catch (e) { + log.error('Vitest workers could not parse known tests, so Early Flake Detection will not work.') + } + + return onBeforeRunTask.apply(this, arguments) + }) + + // `onAfterRunTask` is run after all repetitions or attempts are run + shimmer.wrap(VitestTestRunner.prototype, 'onAfterRunTask', onAfterRunTask => async function (task) { + const { + _ddIsEarlyFlakeDetectionEnabled: isEarlyFlakeDetectionEnabled + } = globalThis.__vitest_worker__.providedContext + + if (isEarlyFlakeDetectionEnabled && taskToStatuses.has(task)) { + const statuses = taskToStatuses.get(task) + // If the test has passed at least once, we consider it passed + if (statuses.includes('pass')) { + if (task.result.state === 'fail') { + switchedStatuses.add(task) + } + task.result.state = 'pass' + } + } + + return onAfterRunTask.apply(this, arguments) + }) + + // test start (only tests that are not marked as skip or todo) + // `onBeforeTryTask` is run for every repetition and attempt of the test + shimmer.wrap(VitestTestRunner.prototype, 'onBeforeTryTask', onBeforeTryTask => async function (task, retryInfo) { + if (!testStartCh.hasSubscribers) { + return onBeforeTryTask.apply(this, arguments) + } + const testName = getTestName(task) + let isNew = false + let isEarlyFlakeDetectionEnabled = false + + try { + const { + _ddIsEarlyFlakeDetectionEnabled + } = globalThis.__vitest_worker__.providedContext + + isEarlyFlakeDetectionEnabled = _ddIsEarlyFlakeDetectionEnabled + + if (isEarlyFlakeDetectionEnabled) { + isNew = newTasks.has(task) + } + } catch (e) { + log.error('Vitest workers could not parse known tests, so Early Flake Detection will not work.') + } + const { retry: numAttempt, repeats: numRepetition } = retryInfo + + // We finish the previous test here because we know it has failed already + if (numAttempt > 0) { + const asyncResource = taskToAsync.get(task) + const testError = task.result?.errors?.[0] + if (asyncResource) { + asyncResource.runInAsyncScope(() => { + testErrorCh.publish({ error: testError }) + }) + } + } + + const lastExecutionStatus = task.result.state + + // These clauses handle task.repeats, whether EFD is enabled or not + // The only thing that EFD does is to forcefully pass the test if it has passed at least once + if (numRepetition > 0 && numRepetition < task.repeats) { // it may or may have not failed + // Here we finish the earlier iteration, + // as long as it's not the _last_ iteration (which will be finished normally) + + // TODO: check test duration (not to repeat if it's too slow) + const asyncResource = taskToAsync.get(task) + if (asyncResource) { + if (lastExecutionStatus === 'fail') { + const testError = task.result?.errors?.[0] + asyncResource.runInAsyncScope(() => { + testErrorCh.publish({ error: testError }) + }) + } else { + asyncResource.runInAsyncScope(() => { + testPassCh.publish({ task }) + }) + } + if (isEarlyFlakeDetectionEnabled) { + const statuses = taskToStatuses.get(task) + statuses.push(lastExecutionStatus) + // If we don't "reset" the result.state to "pass", once a repetition fails, + // vitest will always consider the test as failed, so we can't read the actual status + task.result.state = 'pass' + } + } + } else if (numRepetition === task.repeats) { + const asyncResource = taskToAsync.get(task) + if (lastExecutionStatus === 'fail') { + const testError = task.result?.errors?.[0] + asyncResource.runInAsyncScope(() => { + testErrorCh.publish({ error: testError }) + }) + } else { + asyncResource.runInAsyncScope(() => { + testPassCh.publish({ task }) + }) + } + } + + const asyncResource = new AsyncResource('bound-anonymous-fn') + taskToAsync.set(task, asyncResource) + + asyncResource.runInAsyncScope(() => { + testStartCh.publish({ + testName, + testSuiteAbsolutePath: task.file.filepath, + isRetry: numAttempt > 0 || numRepetition > 0, + isNew + }) + }) + return onBeforeTryTask.apply(this, arguments) + }) + + // test finish (only passed tests) + shimmer.wrap(VitestTestRunner.prototype, 'onAfterTryTask', onAfterTryTask => + async function (task, { retry: retryCount }) { + if (!testFinishTimeCh.hasSubscribers) { + return onAfterTryTask.apply(this, arguments) + } + const result = await onAfterTryTask.apply(this, arguments) + + const status = getVitestTestStatus(task, retryCount) + const asyncResource = taskToAsync.get(task) + + if (asyncResource) { + // We don't finish here because the test might fail in a later hook (afterEach) + asyncResource.runInAsyncScope(() => { + testFinishTimeCh.publish({ status, task }) + }) + } + + return result + }) + + return vitestPackage +}) + +// There are multiple index* files across different versions of vitest, +// so we check for the existence of BaseSequencer to determine if we are in the right file +addHook({ + name: 'vitest', + versions: ['>=1.6.0 <2.0.0'], + filePattern: 'dist/vendor/index.*' +}, (vitestPackage) => { + if (isReporterPackage(vitestPackage)) { + shimmer.wrap(vitestPackage.B.prototype, 'sort', getSortWrapper) + } + + return vitestPackage +}) + +addHook({ + name: 'vitest', + versions: ['>=2.0.0 <2.0.5'], + filePattern: 'dist/vendor/index.*' +}, (vitestPackage) => { + if (isReporterPackageNew(vitestPackage)) { + shimmer.wrap(vitestPackage.e.prototype, 'sort', getSortWrapper) + } + + return vitestPackage +}) + +addHook({ + name: 'vitest', + versions: ['>=2.1.0'], + filePattern: 'dist/chunks/RandomSequencer.*' +}, (randomSequencerPackage) => { + shimmer.wrap(randomSequencerPackage.B.prototype, 'sort', getSortWrapper) + return randomSequencerPackage +}) + +addHook({ + name: 'vitest', + versions: ['>=2.0.5 <2.1.0'], + filePattern: 'dist/chunks/index.*' +}, (vitestPackage) => { + if (isReporterPackageNewest(vitestPackage)) { + shimmer.wrap(vitestPackage.h.prototype, 'sort', getSortWrapper) + } + + return vitestPackage +}) + +// Can't specify file because compiled vitest includes hashes in their files +addHook({ + name: 'vitest', + versions: ['>=1.6.0 <2.0.5'], + filePattern: 'dist/vendor/cac.*' +}, getCreateCliWrapper) + +addHook({ + name: 'vitest', + versions: ['>=2.0.5'], + filePattern: 'dist/chunks/cac.*' +}, getCreateCliWrapper) + +// test suite start and finish +// only relevant for workers +addHook({ + name: '@vitest/runner', + versions: ['>=1.6.0'], + file: 'dist/index.js' +}, (vitestPackage, frameworkVersion) => { + shimmer.wrap(vitestPackage, 'startTests', startTests => async function (testPath) { + let testSuiteError = null + if (!testSuiteStartCh.hasSubscribers) { + return startTests.apply(this, arguments) + } + + const testSuiteAsyncResource = new AsyncResource('bound-anonymous-fn') + testSuiteAsyncResource.runInAsyncScope(() => { + testSuiteStartCh.publish({ testSuiteAbsolutePath: testPath[0], frameworkVersion }) + }) + const startTestsResponse = await startTests.apply(this, arguments) + + let onFinish = null + const onFinishPromise = new Promise(resolve => { + onFinish = resolve + }) + + const testTasks = getTypeTasks(startTestsResponse[0].tasks) + + // Only one test task per test, even if there are retries + testTasks.forEach(task => { + const testAsyncResource = taskToAsync.get(task) + const { result } = task + // We have to trick vitest into thinking that the test has passed + // but we want to report it as failed if it did fail + const isSwitchedStatus = switchedStatuses.has(task) + + if (result) { + const { state, duration, errors } = result + if (state === 'skip') { // programmatic skip + testSkipCh.publish({ testName: getTestName(task), testSuiteAbsolutePath: task.file.filepath }) + } else if (state === 'pass' && !isSwitchedStatus) { + if (testAsyncResource) { + testAsyncResource.runInAsyncScope(() => { + testPassCh.publish({ task }) + }) + } + } else if (state === 'fail' || isSwitchedStatus) { + let testError + + if (errors?.length) { + testError = errors[0] + } + + if (testAsyncResource) { + const isRetry = task.result?.retryCount > 0 + // `duration` is the duration of all the retries, so it can't be used if there are retries + testAsyncResource.runInAsyncScope(() => { + testErrorCh.publish({ duration: !isRetry ? duration : undefined, error: testError }) + }) + } + if (errors?.length) { + testSuiteError = testError // we store the error to bubble it up to the suite + } + } + } else { // test.skip or test.todo + testSkipCh.publish({ testName: getTestName(task), testSuiteAbsolutePath: task.file.filepath }) + } + }) + + const testSuiteResult = startTestsResponse[0].result + + if (testSuiteResult.errors?.length) { // Errors from root level hooks + testSuiteError = testSuiteResult.errors[0] + } else if (testSuiteResult.state === 'fail') { // Errors from `describe` level hooks + const suiteTasks = getTypeTasks(startTestsResponse[0].tasks, 'suite') + const failedSuites = suiteTasks.filter(task => task.result?.state === 'fail') + if (failedSuites.length && failedSuites[0].result?.errors?.length) { + testSuiteError = failedSuites[0].result.errors[0] + } + } + + if (testSuiteError) { + testSuiteAsyncResource.runInAsyncScope(() => { + testSuiteErrorCh.publish({ error: testSuiteError }) + }) + } + + testSuiteAsyncResource.runInAsyncScope(() => { + testSuiteFinishCh.publish({ status: testSuiteResult.state, onFinish }) + }) + + // TODO: fix too frequent flushes + await onFinishPromise + + return startTestsResponse + }) + + return vitestPackage +}) diff --git a/packages/datadog-instrumentations/src/winston.js b/packages/datadog-instrumentations/src/winston.js index 8a7bc44f3ad..9b9c4e811aa 100644 --- a/packages/datadog-instrumentations/src/winston.js +++ b/packages/datadog-instrumentations/src/winston.js @@ -8,6 +8,18 @@ const shimmer = require('../../datadog-shimmer') const patched = new WeakSet() +// Test Visibility log submission channels +const configureCh = channel('ci:log-submission:winston:configure') +const addTransport = channel('ci:log-submission:winston:add-transport') + +addHook({ name: 'winston', file: 'lib/winston/transports/index.js', versions: ['>=3'] }, transportsPackage => { + if (configureCh.hasSubscribers) { + configureCh.publish(transportsPackage.Http) + } + + return transportsPackage +}) + addHook({ name: 'winston', file: 'lib/winston/logger.js', versions: ['>=3'] }, Logger => { const logCh = channel('apm:winston:log') shimmer.wrap(Logger.prototype, 'write', write => { @@ -20,6 +32,16 @@ addHook({ name: 'winston', file: 'lib/winston/logger.js', versions: ['>=3'] }, L return write.apply(this, arguments) } }) + + shimmer.wrap(Logger.prototype, 'configure', configure => function () { + const configureResponse = configure.apply(this, arguments) + // After the original `configure`, because it resets transports + if (addTransport.hasSubscribers) { + addTransport.publish(this) + } + return configureResponse + }) + return Logger }) @@ -42,13 +64,12 @@ function wrapMethod (method, logCh) { if (patched.has(transport) || typeof transport.log !== 'function') continue - const log = transport.log - transport.log = function wrappedLog (level, msg, meta, callback) { + shimmer.wrap(transport, 'log', log => function wrappedLog (level, msg, meta, callback) { const payload = { message: meta || {} } logCh.publish(payload) arguments[2] = payload.message log.apply(this, arguments) - } + }) patched.add(transport) } } diff --git a/packages/datadog-instrumentations/test/body-parser.spec.js b/packages/datadog-instrumentations/test/body-parser.spec.js index d502bc00ea6..482ba5e772d 100644 --- a/packages/datadog-instrumentations/test/body-parser.spec.js +++ b/packages/datadog-instrumentations/test/body-parser.spec.js @@ -1,9 +1,9 @@ 'use strict' -const getPort = require('get-port') const dc = require('dc-polyfill') const axios = require('axios') const agent = require('../../dd-trace/test/plugins/agent') +const { storage } = require('../../datadog-core') withVersions('body-parser', 'body-parser', version => { describe('body parser instrumentation', () => { @@ -11,8 +11,9 @@ withVersions('body-parser', 'body-parser', version => { let port, server, middlewareProcessBodyStub before(() => { - return agent.load(['express', 'body-parser'], { client: false }) + return agent.load(['http', 'express', 'body-parser'], { client: false }) }) + before((done) => { const express = require('../../../versions/express').get() const bodyParser = require(`../../../versions/body-parser@${version}`).get() @@ -22,13 +23,12 @@ withVersions('body-parser', 'body-parser', version => { middlewareProcessBodyStub() res.end('DONE') }) - getPort().then(newPort => { - port = newPort - server = app.listen(port, () => { - done() - }) + server = app.listen(0, () => { + port = server.address().port + done() }) }) + beforeEach(async () => { middlewareProcessBodyStub = sinon.stub() }) @@ -71,5 +71,27 @@ withVersions('body-parser', 'body-parser', version => { bodyParserReadCh.unsubscribe(blockRequest) }) + + it('should not lose the http async context', async () => { + let store + let payload + + function handler (data) { + store = storage.getStore() + payload = data + } + bodyParserReadCh.subscribe(handler) + + const res = await axios.post(`http://localhost:${port}/`, { key: 'value' }) + + expect(store).to.have.property('req', payload.req) + expect(store).to.have.property('res', payload.res) + expect(store).to.have.property('span') + + expect(middlewareProcessBodyStub).to.be.calledOnce + expect(res.data).to.be.equal('DONE') + + bodyParserReadCh.unsubscribe(handler) + }) }) }) diff --git a/packages/datadog-instrumentations/test/check_require_cache.spec.js b/packages/datadog-instrumentations/test/check_require_cache.spec.js new file mode 100644 index 00000000000..168eac97d78 --- /dev/null +++ b/packages/datadog-instrumentations/test/check_require_cache.spec.js @@ -0,0 +1,34 @@ +'use strict' + +const { exec } = require('node:child_process') + +describe('check_require_cache', () => { + const opts = { + cwd: __dirname, + env: { + DD_TRACE_DEBUG: 'true' + } + } + + it('should be no warnings when tracer is loaded first', (done) => { + exec(`${process.execPath} ./check_require_cache/good-order.js`, opts, (error, stdout, stderr) => { + expect(error).to.be.null + expect(stdout).to.be.empty + expect(stderr).to.be.empty + done() + }) + }) + + // stderr is empty on Windows + if (process.platform !== 'windows') { + it('should find warnings when tracer loaded late', (done) => { + exec(`${process.execPath} ./check_require_cache/bad-order.js`, opts, (error, stdout, stderr) => { + expect(error).to.be.null + expect(stdout).to.be.empty + expect(stderr).to.not.be.empty + expect(stderr).to.include("Package 'express' was loaded") + done() + }) + }) + } +}) diff --git a/packages/datadog-instrumentations/test/check_require_cache/bad-order.js b/packages/datadog-instrumentations/test/check_require_cache/bad-order.js new file mode 100755 index 00000000000..a5fab991153 --- /dev/null +++ b/packages/datadog-instrumentations/test/check_require_cache/bad-order.js @@ -0,0 +1,7 @@ +#!/usr/bin/env node + +require('express') // package required before tracer +const tracer = require('../../../../') +tracer.init() + +process.exit() diff --git a/packages/datadog-instrumentations/test/check_require_cache/good-order.js b/packages/datadog-instrumentations/test/check_require_cache/good-order.js new file mode 100755 index 00000000000..72bd2c666b9 --- /dev/null +++ b/packages/datadog-instrumentations/test/check_require_cache/good-order.js @@ -0,0 +1,7 @@ +#!/usr/bin/env node + +const tracer = require('../../../../') +require('express') // package required after tracer +tracer.init() + +process.exit() diff --git a/packages/datadog-instrumentations/test/child_process.spec.js b/packages/datadog-instrumentations/test/child_process.spec.js new file mode 100644 index 00000000000..ffd002e8a6b --- /dev/null +++ b/packages/datadog-instrumentations/test/child_process.spec.js @@ -0,0 +1,399 @@ +'use strict' + +const { promisify } = require('util') +const agent = require('../../dd-trace/test/plugins/agent') +const dc = require('dc-polyfill') +const { NODE_MAJOR } = require('../../../version') + +describe('child process', () => { + const modules = ['child_process', 'node:child_process'] + const execAsyncMethods = ['execFile', 'spawn'] + const execAsyncShellMethods = ['exec'] + const execSyncMethods = ['execFileSync'] + const execSyncShellMethods = ['execSync'] + + const childProcessChannel = dc.tracingChannel('datadog:child_process:execution') + + modules.forEach((childProcessModuleName) => { + describe(childProcessModuleName, () => { + let start, finish, error, childProcess, asyncFinish + + before(() => { + return agent.load(childProcessModuleName) + }) + + after(() => { + return agent.close({ ritmReset: false }) + }) + + beforeEach(() => { + start = sinon.stub() + finish = sinon.stub() + error = sinon.stub() + asyncFinish = sinon.stub() + + childProcessChannel.subscribe({ + start, + end: finish, + asyncEnd: asyncFinish, + error + }) + + childProcess = require(childProcessModuleName) + }) + + afterEach(() => { + childProcessChannel.unsubscribe({ + start, + end: finish, + asyncEnd: asyncFinish, + error + }) + }) + + describe('async methods', (done) => { + describe('command not interpreted by a shell by default', () => { + execAsyncMethods.forEach(methodName => { + describe(`method ${methodName}`, () => { + it('should execute success callbacks', (done) => { + const childEmitter = childProcess[methodName]('ls') + + childEmitter.once('close', () => { + expect(start).to.have.been.calledOnceWith({ command: 'ls', shell: false }) + expect(asyncFinish).to.have.been.calledOnceWith({ command: 'ls', shell: false, result: 0 }) + expect(error).not.to.have.been.called + done() + }) + }) + + it('should execute error callback', (done) => { + const childEmitter = childProcess[methodName]('invalid_command_test') + + childEmitter.once('close', () => { + expect(start).to.have.been.calledOnceWith({ command: 'invalid_command_test', shell: false }) + expect(asyncFinish).to.have.been.calledOnceWith({ + command: 'invalid_command_test', + shell: false, + result: -2 + }) + expect(error).to.have.been.calledOnce + done() + }) + }) + + it('should execute error callback with `exit 1` command', (done) => { + const childEmitter = childProcess[methodName]('node -e "process.exit(1)"', { shell: true }) + + childEmitter.once('close', () => { + expect(start).to.have.been.calledOnceWith({ command: 'node -e "process.exit(1)"', shell: true }) + expect(asyncFinish).to.have.been.calledOnceWith({ + command: 'node -e "process.exit(1)"', + shell: true, + result: 1 + }) + expect(error).to.have.been.calledOnce + done() + }) + }) + }) + + if (methodName !== 'spawn') { + describe(`method ${methodName} with promisify`, () => { + it('should execute success callbacks', async () => { + await promisify(childProcess[methodName])('echo') + expect(start.firstCall.firstArg).to.include({ + command: 'echo', + shell: false + }) + + expect(asyncFinish).to.have.been.calledOnceWith({ + command: 'echo', + shell: false, + result: { + stdout: '\n', + stderr: '' + } + }) + expect(error).not.to.have.been.called + }) + + it('should execute error callback', async () => { + try { + await promisify(childProcess[methodName])('invalid_command_test') + } catch (e) { + expect(start).to.have.been.calledOnce + expect(start.firstCall.firstArg).to.include({ command: 'invalid_command_test', shell: false }) + + const errStub = new Error('spawn invalid_command_test ENOENT') + errStub.code = 'ENOENT' + errStub.errno = -2 + + expect(asyncFinish).to.have.been.calledOnce + expect(asyncFinish.firstCall.firstArg).to.include({ command: 'invalid_command_test', shell: false }) + expect(asyncFinish.firstCall.firstArg).to.deep.include({ + command: 'invalid_command_test', + shell: false, + error: errStub + }) + + expect(error).to.have.been.calledOnce + } + }) + + it('should execute error callback with `exit 1` command', async () => { + const errStub = new Error('Command failed: node -e "process.exit(1)"\n') + errStub.code = 1 + errStub.cmd = 'node -e "process.exit(1)"' + + try { + await promisify(childProcess[methodName])('node -e "process.exit(1)"', { shell: true }) + } catch (e) { + expect(start).to.have.been.calledOnce + expect(start.firstCall.firstArg).to.include({ command: 'node -e "process.exit(1)"', shell: true }) + + expect(asyncFinish).to.have.been.calledOnce + expect(asyncFinish.firstCall.firstArg).to.include({ + command: 'node -e "process.exit(1)"', + shell: true + }) + expect(asyncFinish.firstCall.firstArg).to.deep.include({ + command: 'node -e "process.exit(1)"', + shell: true, + error: errStub + }) + + expect(error).to.have.been.calledOnce + } + }) + }) + } + }) + }) + + describe('command interpreted by a shell by default', () => { + execAsyncShellMethods.forEach(methodName => { + describe(`method ${methodName}`, () => { + it('should execute success callbacks', (done) => { + const res = childProcess[methodName]('ls') + + res.once('close', () => { + expect(start).to.have.been.calledOnceWith({ command: 'ls', shell: true }) + expect(asyncFinish).to.have.been.calledOnceWith({ command: 'ls', shell: true, result: 0 }) + expect(error).not.to.have.been.called + done() + }) + }) + + it('should execute error callback with `exit 1` command', (done) => { + const res = childProcess[methodName]('node -e "process.exit(1)"') + + res.once('close', () => { + expect(start).to.have.been.calledOnceWith({ command: 'node -e "process.exit(1)"', shell: true }) + expect(asyncFinish).to.have.been.calledOnceWith({ + command: 'node -e "process.exit(1)"', + shell: true, + result: 1 + }) + expect(error).to.have.been.called + done() + }) + }) + + it('should execute error callback', (done) => { + const res = childProcess[methodName]('invalid_command_test') + + res.once('close', () => { + expect(start).to.have.been.calledOnceWith({ command: 'invalid_command_test', shell: true }) + expect(error).to.have.been.calledOnce + expect(asyncFinish).to.have.been.calledOnceWith({ + command: 'invalid_command_test', + shell: true, + result: 127 + }) + done() + }) + }) + }) + + describe(`method ${methodName} with promisify`, () => { + it('should execute success callbacks', async () => { + await promisify(childProcess[methodName])('echo') + expect(start).to.have.been.calledOnceWith({ + command: 'echo', + shell: true + }) + expect(asyncFinish).to.have.been.calledOnceWith({ + command: 'echo', + shell: true, + result: 0 + }) + expect(error).not.to.have.been.called + }) + + it('should execute error callback', async () => { + try { + await promisify(childProcess[methodName])('invalid_command_test') + return Promise.reject(new Error('Command expected to fail')) + } catch (e) { + expect(start).to.have.been.calledOnceWith({ command: 'invalid_command_test', shell: true }) + expect(asyncFinish).to.have.been.calledOnce + expect(error).to.have.been.calledOnce + } + }) + + it('should execute error callback with `exit 1` command', async () => { + try { + await promisify(childProcess[methodName])('node -e "process.exit(1)"') + return Promise.reject(new Error('Command expected to fail')) + } catch (e) { + expect(start).to.have.been.calledOnceWith({ command: 'node -e "process.exit(1)"', shell: true }) + expect(asyncFinish).to.have.been.calledOnceWith({ + command: 'node -e "process.exit(1)"', + shell: true, + result: 1 + }) + expect(error).to.have.been.calledOnce + } + }) + }) + }) + }) + }) + + describe('sync methods', () => { + describe('command not interpreted by a shell', () => { + execSyncMethods.forEach(methodName => { + describe(`method ${methodName}`, () => { + it('should execute success callbacks', () => { + const result = childProcess[methodName]('ls') + + expect(start).to.have.been.calledOnceWith({ + command: 'ls', + shell: false, + result + }, + 'tracing:datadog:child_process:execution:start') + + expect(finish).to.have.been.calledOnceWith({ + command: 'ls', + shell: false, + result + }, + 'tracing:datadog:child_process:execution:end') + + expect(error).not.to.have.been.called + }) + + it('should execute error callback', () => { + let childError + try { + childProcess[methodName]('invalid_command_test') + } catch (error) { + childError = error + } finally { + expect(start).to.have.been.calledOnceWith({ + command: 'invalid_command_test', + shell: false, + error: childError + }) + expect(finish).to.have.been.calledOnce + expect(error).to.have.been.calledOnce + } + }) + + it('should execute error callback with `exit 1` command', () => { + let childError + try { + childProcess[methodName]('node -e "process.exit(1)"') + } catch (error) { + childError = error + } finally { + expect(start).to.have.been.calledOnceWith({ + command: 'node -e "process.exit(1)"', + shell: false, + error: childError + }) + expect(finish).to.have.been.calledOnce + } + }) + if (methodName !== 'execFileSync' || NODE_MAJOR > 16) { + // when a process return an invalid code, in node <=16, in execFileSync with shell:true + // an exception is not thrown + it('should execute error callback with `exit 1` command with shell: true', () => { + let childError + try { + childProcess[methodName]('node -e "process.exit(1)"', { shell: true }) + } catch (error) { + childError = error + } finally { + expect(start).to.have.been.calledOnceWith({ + command: 'node -e "process.exit(1)"', + shell: true, + error: childError + }) + expect(finish).to.have.been.calledOnce + } + }) + } + }) + }) + }) + + describe('command interpreted by a shell by default', () => { + execSyncShellMethods.forEach(methodName => { + describe(`method ${methodName}`, () => { + it('should execute success callbacks', () => { + const result = childProcess[methodName]('ls') + + expect(start).to.have.been.calledOnceWith({ + command: 'ls', + shell: true, + result + }) + expect(finish).to.have.been.calledOnceWith({ + command: 'ls', + shell: true, + result + }) + expect(error).not.to.have.been.called + }) + + it('should execute error callback', () => { + let childError + try { + childProcess[methodName]('invalid_command_test') + } catch (error) { + childError = error + } finally { + expect(start).to.have.been.calledOnceWith({ + command: 'invalid_command_test', + shell: true, + error: childError + }) + expect(finish).to.have.been.calledOnce + expect(error).to.have.been.calledOnce + } + }) + + it('should execute error callback with `exit 1` command', () => { + let childError + try { + childProcess[methodName]('node -e "process.exit(1)"') + } catch (error) { + childError = error + } finally { + expect(start).to.have.been.calledOnceWith({ + command: 'node -e "process.exit(1)"', + shell: true, + error: childError + }) + expect(finish).to.have.been.calledOnce + } + }) + }) + }) + }) + }) + }) + }) +}) diff --git a/packages/datadog-instrumentations/test/cookie-parser.spec.js b/packages/datadog-instrumentations/test/cookie-parser.spec.js index 4137ddbef63..799d434dd05 100644 --- a/packages/datadog-instrumentations/test/cookie-parser.spec.js +++ b/packages/datadog-instrumentations/test/cookie-parser.spec.js @@ -1,7 +1,6 @@ 'use strict' const { assert } = require('chai') -const getPort = require('get-port') const dc = require('dc-polyfill') const axios = require('axios') const agent = require('../../dd-trace/test/plugins/agent') @@ -14,6 +13,7 @@ withVersions('cookie-parser', 'cookie-parser', version => { before(() => { return agent.load(['express', 'cookie-parser'], { client: false }) }) + before((done) => { const express = require('../../../versions/express').get() const cookieParser = require(`../../../versions/cookie-parser@${version}`).get() @@ -23,13 +23,12 @@ withVersions('cookie-parser', 'cookie-parser', version => { middlewareProcessCookieStub() res.end('DONE') }) - getPort().then(newPort => { - port = newPort - server = app.listen(port, () => { - done() - }) + server = app.listen(0, () => { + port = server.address().port + done() }) }) + beforeEach(async () => { middlewareProcessCookieStub = sinon.stub() }) diff --git a/packages/datadog-instrumentations/test/express-mongo-sanitize.spec.js b/packages/datadog-instrumentations/test/express-mongo-sanitize.spec.js index d9a314d5abc..3fcf981e528 100644 --- a/packages/datadog-instrumentations/test/express-mongo-sanitize.spec.js +++ b/packages/datadog-instrumentations/test/express-mongo-sanitize.spec.js @@ -1,7 +1,6 @@ 'use strict' const agent = require('../../dd-trace/test/plugins/agent') -const getPort = require('get-port') const { channel } = require('dc-polyfill') const axios = require('axios') describe('express-mongo-sanitize', () => { @@ -25,11 +24,9 @@ describe('express-mongo-sanitize', () => { res.end() }) - getPort().then(newPort => { - port = newPort - server = app.listen(port, () => { - done() - }) + server = app.listen(0, () => { + port = server.address().port + done() }) }) @@ -58,7 +55,7 @@ describe('express-mongo-sanitize', () => { await axios.get(`http://localhost:${port}/?param[$eq]=paramvalue`) expect(requestBody).to.be.calledOnce - expect(requestBody.firstCall.args[0].query.param['$eq']).to.be.undefined + expect(requestBody.firstCall.args[0].query.param.$eq).to.be.undefined }) }) @@ -89,7 +86,7 @@ describe('express-mongo-sanitize', () => { await axios.get(`http://localhost:${port}/?param[$eq]=paramvalue`) expect(requestBody).to.be.calledOnce - expect(requestBody.firstCall.args[0].query.param['$eq']).to.be.undefined + expect(requestBody.firstCall.args[0].query.param.$eq).to.be.undefined }) it('subscription is called with expected parameters without sanitization request', async () => { @@ -111,7 +108,7 @@ describe('express-mongo-sanitize', () => { expect(subscription).to.be.calledOnce expect(subscription.firstCall.args[0].sanitizedProperties) .to.be.deep.equal(['body', 'params', 'headers', 'query']) - expect(subscription.firstCall.args[0].req.query.param['$eq']).to.be.undefined + expect(subscription.firstCall.args[0].req.query.param.$eq).to.be.undefined }) }) }) @@ -150,7 +147,7 @@ describe('express-mongo-sanitize', () => { const objectToSanitize = { unsafeKey: { - '$ne': 'test' + $ne: 'test' }, safeKey: 'safeValue' } @@ -158,7 +155,7 @@ describe('express-mongo-sanitize', () => { const sanitizedObject = expressMongoSanitize.sanitize(objectToSanitize) expect(sanitizedObject.safeKey).to.be.equal(objectToSanitize.safeKey) - expect(sanitizedObject.unsafeKey['$ne']).to.be.undefined + expect(sanitizedObject.unsafeKey.$ne).to.be.undefined }) }) @@ -193,7 +190,7 @@ describe('express-mongo-sanitize', () => { const objectToSanitize = { unsafeKey: { - '$ne': 'test' + $ne: 'test' }, safeKey: 'safeValue' } @@ -201,7 +198,7 @@ describe('express-mongo-sanitize', () => { const sanitizedObject = expressMongoSanitize.sanitize(objectToSanitize) expect(sanitizedObject.safeKey).to.be.equal(objectToSanitize.safeKey) - expect(sanitizedObject.unsafeKey['$ne']).to.be.undefined + expect(sanitizedObject.unsafeKey.$ne).to.be.undefined expect(subscription).to.be.calledOnceWith({ sanitizedObject }) }) }) diff --git a/packages/datadog-instrumentations/test/express.spec.js b/packages/datadog-instrumentations/test/express.spec.js index 88f75164be6..d21b9be3e0a 100644 --- a/packages/datadog-instrumentations/test/express.spec.js +++ b/packages/datadog-instrumentations/test/express.spec.js @@ -1,7 +1,6 @@ 'use strict' const agent = require('../../dd-trace/test/plugins/agent') -const getPort = require('get-port') const axios = require('axios') const dc = require('dc-polyfill') @@ -13,6 +12,7 @@ withVersions('express', 'express', version => { before(() => { return agent.load(['express', 'body-parser'], { client: false }) }) + before((done) => { const express = require('../../../versions/express').get() const app = express() @@ -20,13 +20,12 @@ withVersions('express', 'express', version => { requestBody() res.end('DONE') }) - getPort().then(newPort => { - port = newPort - server = app.listen(port, () => { - done() - }) + server = app.listen(0, () => { + port = server.address().port + done() }) }) + beforeEach(async () => { requestBody = sinon.stub() }) diff --git a/packages/datadog-instrumentations/test/fs.spec.js b/packages/datadog-instrumentations/test/fs.spec.js index d9810f5c493..01217febb1c 100644 --- a/packages/datadog-instrumentations/test/fs.spec.js +++ b/packages/datadog-instrumentations/test/fs.spec.js @@ -12,6 +12,7 @@ describe('fs instrumentation', () => { }) }) } + it('require fs should work', () => { return agent.load('fs', undefined, { flushInterval: 1 }).then(() => { const fs = require('fs') diff --git a/packages/datadog-instrumentations/test/generic-pool.spec.js b/packages/datadog-instrumentations/test/generic-pool.spec.js index f07d8d6c477..eee62a991ea 100644 --- a/packages/datadog-instrumentations/test/generic-pool.spec.js +++ b/packages/datadog-instrumentations/test/generic-pool.spec.js @@ -28,6 +28,7 @@ describe('Instrumentation', () => { const store = 'store' storage.run(store, () => { + // eslint-disable-next-line n/handle-callback-err pool.acquire((err, resource) => { pool.release(resource) expect(storage.getStore()).to.equal(store) diff --git a/packages/datadog-instrumentations/test/http.spec.js b/packages/datadog-instrumentations/test/http.spec.js new file mode 100644 index 00000000000..ec4e989876f --- /dev/null +++ b/packages/datadog-instrumentations/test/http.spec.js @@ -0,0 +1,184 @@ +'use strict' + +const { assert } = require('chai') +const dc = require('dc-polyfill') + +const agent = require('../../dd-trace/test/plugins/agent') +describe('client', () => { + let url, http, startChannelCb, endChannelCb, asyncStartChannelCb, errorChannelCb + + const startChannel = dc.channel('apm:http:client:request:start') + const endChannel = dc.channel('apm:http:client:request:end') + const asyncStartChannel = dc.channel('apm:http:client:request:asyncStart') + const errorChannel = dc.channel('apm:http:client:request:error') + + before(async () => { + await agent.load('http') + }) + + after(() => { + return agent.close() + }) + + beforeEach(() => { + startChannelCb = sinon.stub() + endChannelCb = sinon.stub() + asyncStartChannelCb = sinon.stub() + errorChannelCb = sinon.stub() + + startChannel.subscribe(startChannelCb) + endChannel.subscribe(endChannelCb) + asyncStartChannel.subscribe(asyncStartChannelCb) + errorChannel.subscribe(errorChannelCb) + }) + + afterEach(() => { + startChannel.unsubscribe(startChannelCb) + endChannel.unsubscribe(endChannelCb) + asyncStartChannel.unsubscribe(asyncStartChannelCb) + errorChannel.unsubscribe(errorChannelCb) + }) + + /* + * Necessary because the tracer makes extra requests to the agent + * and the same stub could be called multiple times + */ + function getContextFromStubByUrl (url, stub) { + for (const args of stub.args) { + const arg = args[0] + if (arg.args?.originalUrl === url) { + return arg + } + } + return null + } + + ['http', 'https'].forEach((httpSchema) => { + describe(`using ${httpSchema}`, () => { + describe('abort controller', () => { + function abortCallback (ctx) { + if (ctx.args.originalUrl === url) { + ctx.abortController.abort() + } + } + + before(() => { + http = require(httpSchema) + url = `${httpSchema}://www.datadoghq.com` + }) + + it('abortController is sent on startChannel', (done) => { + http.get(url, (res) => { + res.on('data', () => {}) + res.on('end', () => { + done() + }) + }) + + sinon.assert.called(startChannelCb) + const ctx = getContextFromStubByUrl(url, startChannelCb) + assert.isNotNull(ctx) + assert.instanceOf(ctx.abortController, AbortController) + }) + + it('Request is aborted', (done) => { + startChannelCb.callsFake(abortCallback) + + const cr = http.get(url, () => { + done(new Error('Request should be blocked')) + }) + + cr.on('error', () => { + done() + }) + }) + + it('Request is aborted with custom error', (done) => { + class CustomError extends Error { } + + startChannelCb.callsFake((ctx) => { + if (ctx.args.originalUrl === url) { + ctx.abortController.abort(new CustomError('Custom error')) + } + }) + + const cr = http.get(url, () => { + done(new Error('Request should be blocked')) + }) + + cr.on('error', (e) => { + try { + assert.instanceOf(e, CustomError) + assert.strictEqual(e.message, 'Custom error') + + done() + } catch (e) { + done(e) + } + }) + }) + + it('Error is sent on errorChannel on abort', (done) => { + startChannelCb.callsFake(abortCallback) + + const cr = http.get(url, () => { + done(new Error('Request should be blocked')) + }) + + cr.on('error', () => { + try { + sinon.assert.calledOnce(errorChannelCb) + assert.instanceOf(errorChannelCb.firstCall.args[0].error, Error) + + done() + } catch (e) { + done(e) + } + }) + }) + + it('endChannel is called on abort', (done) => { + startChannelCb.callsFake(abortCallback) + + const cr = http.get(url, () => { + done(new Error('Request should be blocked')) + }) + + cr.on('error', () => { + try { + sinon.assert.called(endChannelCb) + const ctx = getContextFromStubByUrl(url, endChannelCb) + assert.strictEqual(ctx.args.originalUrl, url) + + done() + } catch (e) { + done(e) + } + }) + }) + + it('asyncStartChannel is not called on abort', (done) => { + startChannelCb.callsFake(abortCallback) + + const cr = http.get(url, () => { + done(new Error('Request should be blocked')) + }) + + cr.on('error', () => { + try { + // Necessary because the tracer makes extra requests to the agent + if (asyncStartChannelCb.called) { + const ctx = getContextFromStubByUrl(url, asyncStartChannelCb) + assert.isNull(ctx) + } + + done() + } catch (e) { + done(e.message) + } + }) + }) + }) + }) + }) +}) diff --git a/packages/datadog-instrumentations/test/knex.spec.js b/packages/datadog-instrumentations/test/knex.spec.js index 1684983f9fc..3c9e9c6bd29 100644 --- a/packages/datadog-instrumentations/test/knex.spec.js +++ b/packages/datadog-instrumentations/test/knex.spec.js @@ -20,6 +20,7 @@ describe('Instrumentation', () => { } }) }) + afterEach(() => client.destroy()) it('should propagate context', () => diff --git a/packages/datadog-instrumentations/test/mongoose.spec.js b/packages/datadog-instrumentations/test/mongoose.spec.js index adaae709c82..a97208cd259 100644 --- a/packages/datadog-instrumentations/test/mongoose.spec.js +++ b/packages/datadog-instrumentations/test/mongoose.spec.js @@ -178,6 +178,7 @@ describe('mongoose instrumentations', () => { Test.deleteOne({ type: 'test' }, (err) => { expect(err).to.be.null + // eslint-disable-next-line n/handle-callback-err Test.count({ type: 'test' }, (err, res) => { expect(res).to.be.equal(2) // 3 -> delete 1 -> 2 @@ -258,6 +259,7 @@ describe('mongoose instrumentations', () => { expect(item).not.to.be.null expect(item.name).to.be.equal('test1') + // eslint-disable-next-line n/handle-callback-err Test.count({ type: 'test' }, (err, res) => { expect(res).to.be.equal(2) // 3 -> delete 1 -> 2 @@ -367,7 +369,7 @@ describe('mongoose instrumentations', () => { describe('findOneAndUpdate', () => { if (range !== '>=7') { it('continue working as expected with cb', (done) => { - Test.findOneAndUpdate({ name: 'test1' }, { '$set': { name: 'test1-modified' } }, (err) => { + Test.findOneAndUpdate({ name: 'test1' }, { $set: { name: 'test1-modified' } }, (err) => { expect(err).to.be.null Test.findOne({ name: 'test1-modified' }, (err, item) => { @@ -381,7 +383,7 @@ describe('mongoose instrumentations', () => { } it('continue working as expected with then', (done) => { - Test.findOneAndUpdate({ name: 'test1' }, { '$set': { name: 'test1-modified' } }).then((res) => { + Test.findOneAndUpdate({ name: 'test1' }, { $set: { name: 'test1-modified' } }).then((res) => { Test.findOne({ name: 'test1-modified' }).then((item) => { expect(item).not.to.be.null @@ -390,7 +392,7 @@ describe('mongoose instrumentations', () => { }) }) - testCallbacksCalled('findOneAndUpdate', [{ type: 'test' }, { '$set': { name: 'test1-modified' } }]) + testCallbacksCalled('findOneAndUpdate', [{ type: 'test' }, { $set: { name: 'test1-modified' } }]) }) if (semver.intersects(version, '>=5')) { @@ -398,7 +400,7 @@ describe('mongoose instrumentations', () => { if (range !== '>=7') { it('continue working as expected with cb', (done) => { Test.updateMany({ type: 'test' }, { - '$set': { + $set: { other: 'modified-other' } }, (err) => { @@ -420,9 +422,10 @@ describe('mongoose instrumentations', () => { it('continue working as expected with then', (done) => { Test.updateMany({ type: 'test' }, { - '$set': { + $set: { other: 'modified-other' } + // eslint-disable-next-line n/handle-callback-err }).then((err) => { Test.find({ type: 'test' }).then((items) => { expect(items.length).to.be.equal(3) @@ -436,7 +439,7 @@ describe('mongoose instrumentations', () => { }) }) - testCallbacksCalled('updateMany', [{ type: 'test' }, { '$set': { other: 'modified-other' } }]) + testCallbacksCalled('updateMany', [{ type: 'test' }, { $set: { other: 'modified-other' } }]) }) } @@ -445,7 +448,7 @@ describe('mongoose instrumentations', () => { if (range !== '>=7') { it('continue working as expected with cb', (done) => { Test.updateOne({ name: 'test1' }, { - '$set': { + $set: { other: 'modified-other' } }, (err) => { @@ -463,7 +466,7 @@ describe('mongoose instrumentations', () => { it('continue working as expected with then', (done) => { Test.updateOne({ name: 'test1' }, { - '$set': { + $set: { other: 'modified-other' } }).then(() => { @@ -475,7 +478,7 @@ describe('mongoose instrumentations', () => { }) }) - testCallbacksCalled('updateOne', [{ name: 'test1' }, { '$set': { other: 'modified-other' } }]) + testCallbacksCalled('updateOne', [{ name: 'test1' }, { $set: { other: 'modified-other' } }]) }) } }) @@ -483,8 +486,8 @@ describe('mongoose instrumentations', () => { if (semver.intersects(version, '>=6')) { describe('sanitizeFilter', () => { it('continues working as expected without sanitization', () => { - const source = { 'username': 'test' } - const expected = { 'username': 'test' } + const source = { username: 'test' } + const expected = { username: 'test' } const sanitizedObject = mongoose.sanitizeFilter(source) @@ -492,8 +495,8 @@ describe('mongoose instrumentations', () => { }) it('continues working as expected without sanitization', () => { - const source = { 'username': { '$ne': 'test' } } - const expected = { 'username': { '$eq': { '$ne': 'test' } } } + const source = { username: { $ne: 'test' } } + const expected = { username: { $eq: { $ne: 'test' } } } const sanitizedObject = mongoose.sanitizeFilter(source) @@ -501,7 +504,7 @@ describe('mongoose instrumentations', () => { }) it('channel is published with the result object', () => { - const source = { 'username': { '$ne': 'test' } } + const source = { username: { $ne: 'test' } } const listener = sinon.stub() sanitizeFilterFinishCh.subscribe(listener) diff --git a/packages/datadog-instrumentations/test/mysql2.spec.js b/packages/datadog-instrumentations/test/mysql2.spec.js new file mode 100644 index 00000000000..89e35f2a1f7 --- /dev/null +++ b/packages/datadog-instrumentations/test/mysql2.spec.js @@ -0,0 +1,718 @@ +'use strict' + +const { channel } = require('../src/helpers/instrument') +const agent = require('../../dd-trace/test/plugins/agent') +const { assert } = require('chai') +const semver = require('semver') + +describe('mysql2 instrumentation', () => { + withVersions('mysql2', 'mysql2', version => { + function abort ({ sql, abortController }) { + assert.isString(sql) + const error = new Error('Test') + abortController.abort(error) + + if (!abortController.signal.reason) { + abortController.signal.reason = error + } + } + + function noop () {} + + const config = { + host: '127.0.0.1', + user: 'root', + database: 'db' + } + + const sql = 'SELECT 1' + let startCh, mysql2, shouldEmitEndAfterQueryAbort + let apmQueryStartChannel, apmQueryStart, mysql2Version + + before(() => { + startCh = channel('datadog:mysql2:outerquery:start') + return agent.load(['mysql2']) + }) + + before(() => { + const mysql2Require = require(`../../../versions/mysql2@${version}`) + mysql2Version = mysql2Require.version() + // in v1.3.3 CommandQuery started to emit 'end' after 'error' event + shouldEmitEndAfterQueryAbort = semver.intersects(mysql2Version, '>=1.3.3') + mysql2 = mysql2Require.get() + apmQueryStartChannel = channel('apm:mysql2:query:start') + }) + + beforeEach(() => { + apmQueryStart = sinon.stub() + apmQueryStartChannel.subscribe(apmQueryStart) + }) + + afterEach(() => { + if (startCh?.hasSubscribers) { + startCh.unsubscribe(abort) + startCh.unsubscribe(noop) + } + apmQueryStartChannel.unsubscribe(apmQueryStart) + }) + + describe('lib/connection.js', () => { + let connection + + beforeEach(() => { + connection = mysql2.createConnection(config) + + connection.connect() + }) + + afterEach((done) => { + connection.end(() => done()) + }) + + describe('Connection.prototype.query', () => { + describe('with string as query', () => { + describe('with callback', () => { + it('should abort the query on abortController.abort()', (done) => { + startCh.subscribe(abort) + const query = connection.query(sql, (err) => { + assert.propertyVal(err, 'message', 'Test') + sinon.assert.notCalled(apmQueryStart) + + if (!shouldEmitEndAfterQueryAbort) done() + }) + + query.on('end', () => done()) + }) + + it('should work without abortController.abort()', (done) => { + startCh.subscribe(noop) + connection.query(sql, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + + it('should work without subscriptions', (done) => { + connection.query(sql, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + }) + + describe('without callback', () => { + it('should abort the query on abortController.abort()', (done) => { + startCh.subscribe(abort) + + const query = connection.query(sql) + + query.on('error', (err) => { + assert.propertyVal(err, 'message', 'Test') + sinon.assert.notCalled(apmQueryStart) + if (!shouldEmitEndAfterQueryAbort) done() + }) + + query.on('end', () => done()) + }) + + it('should work without abortController.abort()', (done) => { + startCh.subscribe(noop) + + const query = connection.query(sql) + + query.on('error', (err) => done(err)) + query.on('end', () => { + sinon.assert.called(apmQueryStart) + + done() + }) + }) + + it('should work without subscriptions', (done) => { + const query = connection.query(sql) + + query.on('error', (err) => done(err)) + query.on('end', () => { + sinon.assert.called(apmQueryStart) + + done() + }) + }) + }) + }) + + describe('with object as query', () => { + describe('with callback', () => { + it('should abort the query on abortController.abort()', (done) => { + startCh.subscribe(abort) + const query = mysql2.Connection.createQuery(sql, (err) => { + assert.propertyVal(err, 'message', 'Test') + sinon.assert.notCalled(apmQueryStart) + + if (!shouldEmitEndAfterQueryAbort) done() + }, null, {}) + connection.query(query) + + query.on('end', () => done()) + }) + + it('should work without abortController.abort()', (done) => { + startCh.subscribe(noop) + + const query = mysql2.Connection.createQuery(sql, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }, null, {}) + + connection.query(query) + }) + + it('should work without subscriptions', (done) => { + const query = mysql2.Connection.createQuery(sql, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }, null, {}) + + connection.query(query) + }) + }) + + describe('without callback', () => { + it('should abort the query on abortController.abort()', (done) => { + startCh.subscribe(abort) + + const query = mysql2.Connection.createQuery(sql, null, null, {}) + query.on('error', (err) => { + assert.propertyVal(err, 'message', 'Test') + sinon.assert.notCalled(apmQueryStart) + + if (!shouldEmitEndAfterQueryAbort) done() + }) + + connection.query(query) + + query.on('end', () => done()) + }) + + it('should work without abortController.abort()', (done) => { + startCh.subscribe(noop) + + const query = mysql2.Connection.createQuery(sql, null, null, {}) + query.on('error', (err) => done(err)) + query.on('end', () => { + sinon.assert.called(apmQueryStart) + + done() + }) + + connection.query(query) + }) + + it('should work without subscriptions', (done) => { + const query = mysql2.Connection.createQuery(sql, null, null, {}) + query.on('error', (err) => done(err)) + query.on('end', () => { + sinon.assert.called(apmQueryStart) + + done() + }) + + connection.query(query) + }) + }) + }) + }) + + describe('Connection.prototype.execute', () => { + describe('with the query in options', () => { + it('should abort the query on abortController.abort()', (done) => { + startCh.subscribe(abort) + + const options = { sql } + const commandExecute = connection.execute(options, (err) => { + assert.propertyVal(err, 'message', 'Test') + sinon.assert.notCalled(apmQueryStart) + + done() + }) + + assert.equal(commandExecute.sql, options.sql) + }) + + it('should work without abortController.abort()', (done) => { + startCh.subscribe(noop) + + const options = { sql } + + connection.execute(options, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + + it('should work without subscriptions', (done) => { + const options = { sql } + + connection.execute(options, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + }) + + describe('with sql as string', () => { + it('should abort the query on abortController.abort()', (done) => { + startCh.subscribe(abort) + + connection.execute(sql, (err) => { + assert.propertyVal(err, 'message', 'Test') + sinon.assert.notCalled(apmQueryStart) + done() + }) + }) + + it('should work without abortController.abort()', (done) => { + startCh.subscribe(noop) + + connection.execute(sql, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + + it('should work without subscriptions', (done) => { + const options = { sql } + + connection.execute(options, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + }) + }) + }) + + describe('lib/pool.js', () => { + let pool + + before(() => { + pool = mysql2.createPool(config) + }) + + describe('Pool.prototype.query', () => { + describe('with object as query', () => { + describe('with callback', () => { + it('should abort the query on abortController.abort()', (done) => { + startCh.subscribe(abort) + const query = pool.query({ sql }, (err) => { + assert.propertyVal(err, 'message', 'Test') + sinon.assert.notCalled(apmQueryStart) + + if (!shouldEmitEndAfterQueryAbort) done() + }) + + query.on('end', () => done()) + }) + + it('should work without abortController.abort()', (done) => { + startCh.subscribe(noop) + + pool.query({ sql }, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + + it('should work without subscriptions', (done) => { + pool.query({ sql }, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + }) + + describe('without callback', () => { + it('should abort the query on abortController.abort()', (done) => { + startCh.subscribe(abort) + const query = pool.query({ sql }) + query.on('error', err => { + assert.propertyVal(err, 'message', 'Test') + sinon.assert.notCalled(apmQueryStart) + if (!shouldEmitEndAfterQueryAbort) done() + }) + + query.on('end', () => done()) + }) + + it('should work without abortController.abort()', (done) => { + startCh.subscribe(noop) + const query = pool.query({ sql }) + + query.on('error', err => done(err)) + query.on('end', () => { + sinon.assert.called(apmQueryStart) + + done() + }) + }) + + it('should work without subscriptions', (done) => { + pool.query({ sql }, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + }) + }) + + describe('with string as query', () => { + describe('with callback', () => { + it('should abort the query on abortController.abort()', (done) => { + startCh.subscribe(abort) + const query = pool.query(sql, (err) => { + assert.propertyVal(err, 'message', 'Test') + sinon.assert.notCalled(apmQueryStart) + + if (!shouldEmitEndAfterQueryAbort) done() + }) + + query.on('end', () => done()) + }) + + it('should work without abortController.abort()', (done) => { + startCh.subscribe(noop) + + pool.query(sql, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + + it('should work without subscriptions', (done) => { + pool.query(sql, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + }) + + describe('without callback', () => { + it('should abort the query on abortController.abort()', (done) => { + startCh.subscribe(abort) + const query = pool.query(sql) + query.on('error', err => { + assert.propertyVal(err, 'message', 'Test') + sinon.assert.notCalled(apmQueryStart) + if (!shouldEmitEndAfterQueryAbort) done() + }) + + query.on('end', () => done()) + }) + + it('should work without abortController.abort()', (done) => { + startCh.subscribe(noop) + const query = pool.query(sql) + + query.on('error', err => done(err)) + query.on('end', () => { + sinon.assert.called(apmQueryStart) + + done() + }) + }) + + it('should work without subscriptions', (done) => { + pool.query(sql, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + }) + }) + }) + + describe('Pool.prototype.execute', () => { + describe('with object as query', () => { + describe('with callback', () => { + it('should abort the query on abortController.abort()', (done) => { + startCh.subscribe(abort) + pool.execute({ sql }, (err) => { + assert.propertyVal(err, 'message', 'Test') + + setTimeout(() => { + sinon.assert.notCalled(apmQueryStart) + done() + }, 100) + }) + }) + + it('should work without abortController.abort()', (done) => { + startCh.subscribe(noop) + + pool.execute({ sql }, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + + it('should work without subscriptions', (done) => { + pool.execute({ sql }, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + }) + }) + + describe('with string as query', () => { + describe('with callback', () => { + it('should abort the query on abortController.abort()', (done) => { + startCh.subscribe(abort) + pool.execute(sql, (err) => { + assert.propertyVal(err, 'message', 'Test') + + setTimeout(() => { + sinon.assert.notCalled(apmQueryStart) + done() + }, 100) + }) + }) + + it('should work without abortController.abort()', (done) => { + startCh.subscribe(noop) + + pool.execute(sql, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + + it('should work without subscriptions', (done) => { + pool.execute(sql, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + }) + }) + }) + }) + + describe('lib/pool_cluster.js', () => { + let poolCluster, connection + + before(function () { + if (!semver.satisfies(mysql2Version, '>=2.3.0')) this.skip() + poolCluster = mysql2.createPoolCluster() + poolCluster.add('clusterA', config) + }) + + beforeEach((done) => { + poolCluster.getConnection('clusterA', function (err, _connection) { + if (err) { + done(err) + return + } + + connection = _connection + + done() + }) + }) + + afterEach(() => { + connection?.release() + }) + + describe('PoolNamespace.prototype.query', () => { + describe('with string as query', () => { + it('should abort the query on abortController.abort()', (done) => { + startCh.subscribe(abort) + const namespace = poolCluster.of() + namespace.query(sql, (err) => { + assert.propertyVal(err, 'message', 'Test') + + setTimeout(() => { + sinon.assert.notCalled(apmQueryStart) + done() + }, 100) + }) + }) + + it('should work without abortController.abort()', (done) => { + startCh.subscribe(noop) + + const namespace = poolCluster.of() + namespace.query(sql, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + + it('should work without subscriptions', (done) => { + const namespace = poolCluster.of() + namespace.query(sql, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + }) + + describe('with object as query', () => { + it('should abort the query on abortController.abort()', (done) => { + startCh.subscribe(abort) + const namespace = poolCluster.of() + namespace.query({ sql }, (err) => { + assert.propertyVal(err, 'message', 'Test') + + setTimeout(() => { + sinon.assert.notCalled(apmQueryStart) + done() + }, 100) + }) + }) + + it('should work without abortController.abort()', (done) => { + startCh.subscribe(noop) + + const namespace = poolCluster.of() + namespace.query({ sql }, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + + it('should work without subscriptions', (done) => { + const namespace = poolCluster.of() + namespace.query({ sql }, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + }) + }) + + describe('PoolNamespace.prototype.execute', () => { + describe('with string as query', () => { + it('should abort the query on abortController.abort()', (done) => { + startCh.subscribe(abort) + + const namespace = poolCluster.of() + namespace.execute(sql, (err) => { + assert.propertyVal(err, 'message', 'Test') + + setTimeout(() => { + sinon.assert.notCalled(apmQueryStart) + done() + }, 100) + }) + }) + + it('should work without abortController.abort()', (done) => { + startCh.subscribe(noop) + + const namespace = poolCluster.of() + namespace.execute(sql, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + + it('should work without subscriptions', (done) => { + const namespace = poolCluster.of() + namespace.execute(sql, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + }) + + describe('with object as query', () => { + it('should abort the query on abortController.abort()', (done) => { + startCh.subscribe(abort) + + const namespace = poolCluster.of() + namespace.execute({ sql }, (err) => { + assert.propertyVal(err, 'message', 'Test') + + setTimeout(() => { + sinon.assert.notCalled(apmQueryStart) + done() + }, 100) + }) + }) + + it('should work without abortController.abort()', (done) => { + startCh.subscribe(noop) + + const namespace = poolCluster.of() + namespace.execute({ sql }, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + + it('should work without subscriptions', (done) => { + const namespace = poolCluster.of() + namespace.execute({ sql }, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + }) + }) + }) + }) +}) diff --git a/packages/datadog-instrumentations/test/passport-http.spec.js b/packages/datadog-instrumentations/test/passport-http.spec.js index e9906f7de0e..2918c935e20 100644 --- a/packages/datadog-instrumentations/test/passport-http.spec.js +++ b/packages/datadog-instrumentations/test/passport-http.spec.js @@ -1,7 +1,6 @@ 'use strict' const agent = require('../../dd-trace/test/plugins/agent') -const getPort = require('get-port') const axios = require('axios') const dc = require('dc-polyfill') @@ -13,6 +12,7 @@ withVersions('passport-http', 'passport-http', version => { before(() => { return agent.load(['express', 'passport', 'passport-http'], { client: false }) }) + before((done) => { const express = require('../../../versions/express').get() const passport = require('../../../versions/passport').get() @@ -70,13 +70,12 @@ withVersions('passport-http', 'passport-http', version => { subscriberStub(arguments[0]) }) - getPort().then(newPort => { - port = newPort - server = app.listen(port, () => { - done() - }) + server = app.listen(0, () => { + port = server.address().port + done() }) }) + beforeEach(() => { subscriberStub = sinon.stub() }) @@ -90,7 +89,7 @@ withVersions('passport-http', 'passport-http', version => { const res = await axios.get(`http://localhost:${port}/`, { headers: { // test:1234 - 'Authorization': 'Basic dGVzdDoxMjM0' + Authorization: 'Basic dGVzdDoxMjM0' } }) @@ -108,7 +107,7 @@ withVersions('passport-http', 'passport-http', version => { const res = await axios.get(`http://localhost:${port}/`, { headers: { // test:1234 - 'Authorization': 'Basic dGVzdDoxMjM0' + Authorization: 'Basic dGVzdDoxMjM0' } }) @@ -126,7 +125,7 @@ withVersions('passport-http', 'passport-http', version => { const res = await axios.get(`http://localhost:${port}/`, { headers: { // test:1 - 'Authorization': 'Basic dGVzdDox' + Authorization: 'Basic dGVzdDox' } }) diff --git a/packages/datadog-instrumentations/test/passport-local.spec.js b/packages/datadog-instrumentations/test/passport-local.spec.js index c7a43c50c2e..d54f02b289f 100644 --- a/packages/datadog-instrumentations/test/passport-local.spec.js +++ b/packages/datadog-instrumentations/test/passport-local.spec.js @@ -1,7 +1,6 @@ 'use strict' const agent = require('../../dd-trace/test/plugins/agent') -const getPort = require('get-port') const axios = require('axios') const dc = require('dc-polyfill') @@ -13,9 +12,10 @@ withVersions('passport-local', 'passport-local', version => { before(() => { return agent.load(['express', 'passport', 'passport-local'], { client: false }) }) + before((done) => { const express = require('../../../versions/express').get() - const passport = require(`../../../versions/passport`).get() + const passport = require('../../../versions/passport').get() const LocalStrategy = require(`../../../versions/passport-local@${version}`).get().Strategy const app = express() @@ -71,13 +71,12 @@ withVersions('passport-local', 'passport-local', version => { subscriberStub(arguments[0]) }) - getPort().then(newPort => { - port = newPort - server = app.listen(port, () => { - done() - }) + server = app.listen(0, () => { + port = server.address().port + done() }) }) + beforeEach(() => { subscriberStub = sinon.stub() }) diff --git a/packages/datadog-instrumentations/test/pg.spec.js b/packages/datadog-instrumentations/test/pg.spec.js new file mode 100644 index 00000000000..21d1bfc0951 --- /dev/null +++ b/packages/datadog-instrumentations/test/pg.spec.js @@ -0,0 +1,244 @@ +'use strict' + +const agent = require('../../dd-trace/test/plugins/agent') +const dc = require('dc-polyfill') +const { assert } = require('chai') + +const clients = { + pg: pg => pg.Client +} + +if (process.env.PG_TEST_NATIVE === 'true') { + clients['pg.native'] = pg => pg.native.Client +} + +describe('pg instrumentation', () => { + withVersions('pg', 'pg', version => { + const queryClientStartChannel = dc.channel('apm:pg:query:start') + const queryPoolStartChannel = dc.channel('datadog:pg:pool:query:start') + + let pg + let Query + + function abortQuery ({ abortController }) { + const error = new Error('Test') + abortController.abort(error) + + if (!abortController.signal.reason) { + abortController.signal.reason = error + } + } + + before(() => { + return agent.load(['pg']) + }) + + describe('pg.Client', () => { + Object.keys(clients).forEach(implementation => { + describe(implementation, () => { + let client + + beforeEach(done => { + pg = require(`../../../versions/pg@${version}`).get() + const Client = clients[implementation](pg) + Query = Client.Query + + client = new Client({ + host: '127.0.0.1', + user: 'postgres', + password: 'postgres', + database: 'postgres', + application_name: 'test' + }) + + client.connect(err => done(err)) + }) + + afterEach(() => { + client.end() + }) + + describe('abortController', () => { + afterEach(() => { + if (queryClientStartChannel.hasSubscribers) { + queryClientStartChannel.unsubscribe(abortQuery) + } + }) + + describe('using callback', () => { + it('Should not fail if it is not aborted', (done) => { + client.query('SELECT 1', (err) => { + done(err) + }) + }) + + it('Should abort query', (done) => { + queryClientStartChannel.subscribe(abortQuery) + + client.query('SELECT 1', (err) => { + assert.propertyVal(err, 'message', 'Test') + done() + }) + }) + }) + + describe('using promise', () => { + it('Should not fail if it is not aborted', async () => { + await client.query('SELECT 1') + }) + + it('Should abort query', async () => { + queryClientStartChannel.subscribe(abortQuery) + + try { + await client.query('SELECT 1') + } catch (err) { + assert.propertyVal(err, 'message', 'Test') + + return + } + + throw new Error('Query was not aborted') + }) + }) + + describe('using query object', () => { + describe('without callback', () => { + it('Should not fail if it is not aborted', (done) => { + const query = new Query('SELECT 1') + + client.query(query) + + query.on('end', () => { + done() + }) + }) + + it('Should abort query', (done) => { + queryClientStartChannel.subscribe(abortQuery) + + const query = new Query('SELECT 1') + + client.query(query) + + query.on('error', err => { + assert.propertyVal(err, 'message', 'Test') + done() + }) + + query.on('end', () => { + done(new Error('Query was not aborted')) + }) + }) + }) + + describe('with callback in query object', () => { + it('Should not fail if it is not aborted', (done) => { + const query = new Query('SELECT 1') + query.callback = (err) => { + done(err) + } + + client.query(query) + }) + + it('Should abort query', (done) => { + queryClientStartChannel.subscribe(abortQuery) + + const query = new Query('SELECT 1') + query.callback = err => { + assert.propertyVal(err, 'message', 'Test') + done() + } + + client.query(query) + }) + }) + + describe('with callback in query parameter', () => { + it('Should not fail if it is not aborted', (done) => { + const query = new Query('SELECT 1') + + client.query(query, (err) => { + done(err) + }) + }) + + it('Should abort query', (done) => { + queryClientStartChannel.subscribe(abortQuery) + + const query = new Query('SELECT 1') + + client.query(query, err => { + assert.propertyVal(err, 'message', 'Test') + done() + }) + }) + }) + }) + }) + }) + }) + }) + + describe('pg.Pool', () => { + let pool + + beforeEach(() => { + const { Pool } = require(`../../../versions/pg@${version}`).get() + + pool = new Pool({ + host: '127.0.0.1', + user: 'postgres', + password: 'postgres', + database: 'postgres', + application_name: 'test' + }) + }) + + describe('abortController', () => { + afterEach(() => { + if (queryPoolStartChannel.hasSubscribers) { + queryPoolStartChannel.unsubscribe(abortQuery) + } + }) + + describe('using callback', () => { + it('Should not fail if it is not aborted', (done) => { + pool.query('SELECT 1', (err) => { + done(err) + }) + }) + + it('Should abort query', (done) => { + queryPoolStartChannel.subscribe(abortQuery) + + pool.query('SELECT 1', (err) => { + assert.propertyVal(err, 'message', 'Test') + done() + }) + }) + }) + + describe('using promise', () => { + it('Should not fail if it is not aborted', async () => { + await pool.query('SELECT 1') + }) + + it('Should abort query', async () => { + queryPoolStartChannel.subscribe(abortQuery) + + try { + await pool.query('SELECT 1') + } catch (err) { + assert.propertyVal(err, 'message', 'Test') + return + } + + throw new Error('Query was not aborted') + }) + }) + }) + }) + }) +}) diff --git a/packages/datadog-plugin-aerospike/test/index.spec.js b/packages/datadog-plugin-aerospike/test/index.spec.js index 11202ef9cd4..b15b118586c 100644 --- a/packages/datadog-plugin-aerospike/test/index.spec.js +++ b/packages/datadog-plugin-aerospike/test/index.spec.js @@ -1,10 +1,8 @@ 'use strict' const agent = require('../../dd-trace/test/plugins/agent') -const semver = require('semver') const { ERROR_MESSAGE, ERROR_TYPE, ERROR_STACK } = require('../../dd-trace/src/constants') const { expectedSchema, rawExpectedSchema } = require('./naming') -const { NODE_MAJOR } = require('../../../version') describe('Plugin', () => { let aerospike @@ -61,13 +59,14 @@ describe('Plugin', () => { 'test', 'aerospike.namespace' ) + it('should instrument put', done => { agent .use(traces => { const span = traces[0][0] expect(span).to.have.property('name', expectedSchema.command.opName) expect(span).to.have.property('service', expectedSchema.command.serviceName) - expect(span).to.have.property('resource', `Put`) + expect(span).to.have.property('resource', 'Put') expect(span).to.have.property('type', 'aerospike') expect(span.meta).to.have.property('span.kind', 'client') expect(span.meta).to.have.property('aerospike.key', keyString) @@ -93,7 +92,7 @@ describe('Plugin', () => { const span = traces[0][0] expect(span).to.have.property('name', expectedSchema.command.opName) expect(span).to.have.property('service', expectedSchema.command.serviceName) - expect(span).to.have.property('resource', `Connect`) + expect(span).to.have.property('resource', 'Connect') expect(span).to.have.property('type', 'aerospike') expect(span.meta).to.have.property('span.kind', 'client') expect(span.meta).to.have.property('component', 'aerospike') @@ -110,7 +109,7 @@ describe('Plugin', () => { const span = traces[0][0] expect(span).to.have.property('name', expectedSchema.command.opName) expect(span).to.have.property('service', expectedSchema.command.serviceName) - expect(span).to.have.property('resource', `Get`) + expect(span).to.have.property('resource', 'Get') expect(span).to.have.property('type', 'aerospike') expect(span.meta).to.have.property('span.kind', 'client') expect(span.meta).to.have.property('aerospike.key', keyString) @@ -134,7 +133,7 @@ describe('Plugin', () => { const span = traces[0][0] expect(span).to.have.property('name', expectedSchema.command.opName) expect(span).to.have.property('service', expectedSchema.command.serviceName) - expect(span).to.have.property('resource', `Operate`) + expect(span).to.have.property('resource', 'Operate') expect(span).to.have.property('type', 'aerospike') expect(span.meta).to.have.property('span.kind', 'client') expect(span.meta).to.have.property('aerospike.key', keyString) @@ -165,7 +164,7 @@ describe('Plugin', () => { const span = traces[0][0] expect(span).to.have.property('name', expectedSchema.command.opName) expect(span).to.have.property('service', expectedSchema.command.serviceName) - expect(span).to.have.property('resource', `IndexCreate`) + expect(span).to.have.property('resource', 'IndexCreate') expect(span).to.have.property('type', 'aerospike') expect(span.meta).to.have.property('span.kind', 'client') expect(span.meta).to.have.property('aerospike.namespace', ns) @@ -179,7 +178,7 @@ describe('Plugin', () => { aerospike.connect(config).then(client => { const index = { - ns: ns, + ns, set: 'demo', bin: 'tags', index: 'tags_idx', @@ -191,52 +190,46 @@ describe('Plugin', () => { }) }) - // skip query tests for node 16 and aerospike 4 because of an aerospike error that occurs when using query: - // AerospikeError: Sometimes our doc, or our customers' wishes, get ahead of us. - // We may have processed something that the server is not ready for (unsupported feature). - // this test works on node 14, so it is not a problem with the test but most likely a problem with the package - // version and aerospike server version mismatch which is really hard to pin down, since aerospike doesn't - // provide info on package version's compatibility with each server version - if (!(NODE_MAJOR === 16 && semver.intersects(version, '^4')) && !semver.intersects(version, '^3')) { - it('should instrument query', done => { - agent - .use(traces => { - const span = traces[0][0] - expect(span).to.have.property('name', expectedSchema.command.opName) - expect(span).to.have.property('service', expectedSchema.command.serviceName) - expect(span).to.have.property('resource', `Query`) - expect(span).to.have.property('type', 'aerospike') - expect(span.meta).to.have.property('span.kind', 'client') - expect(span.meta).to.have.property('aerospike.namespace', ns) - expect(span.meta).to.have.property('aerospike.setname', set) - expect(span.meta).to.have.property('component', 'aerospike') - }) - .then(done) - .catch(done) + it('should instrument query', done => { + agent + .use(traces => { + const span = traces[0][0] + expect(span).to.have.property('name', expectedSchema.command.opName) + expect(span).to.have.property('service', expectedSchema.command.serviceName) + expect(span).to.have.property('resource', 'Query') + expect(span).to.have.property('type', 'aerospike') + expect(span.meta).to.have.property('span.kind', 'client') + expect(span.meta).to.have.property('aerospike.namespace', ns) + expect(span.meta).to.have.property('aerospike.setname', set) + expect(span.meta).to.have.property('component', 'aerospike') + }) + .then(done) + .catch(done) - aerospike.connect(config).then(client => { - const index = { - ns: ns, - set: 'demo', - bin: 'tags', - index: 'tags_idx', - datatype: aerospike.indexDataType.STRING - } - client.createIndex(index, (error, job) => { - job.waitUntilDone((waitError) => { - const query = client.query(ns, 'demo') - const queryPolicy = { - totalTimeout: 10000 - } - query.select('id', 'tags') - query.where(aerospike.filter.contains('tags', 'green', aerospike.indexType.LIST)) - const stream = query.foreach(queryPolicy) - stream.on('end', () => { client.close(false) }) - }) + aerospike.connect(config).then(client => { + const index = { + ns, + set: 'demo', + bin: 'tags', + index: 'tags_idx', + datatype: aerospike.indexDataType.STRING + } + // eslint-disable-next-line n/handle-callback-err + client.createIndex(index, (error, job) => { + job.waitUntilDone((waitError) => { + const query = client.query(ns, 'demo') + const queryPolicy = { + totalTimeout: 10000 + } + query.select('id', 'tags') + query.where(aerospike.filter.contains('tags', 'green', aerospike.indexType.LIST)) + const stream = query.foreach(queryPolicy) + stream.on('end', () => { client.close(false) }) }) }) }) - } + }) + it('should run the callback in the parent context', done => { const obj = {} aerospike.connect(config).then(client => { diff --git a/packages/datadog-plugin-aerospike/test/naming.js b/packages/datadog-plugin-aerospike/test/naming.js index 75c360d9999..bed64a71625 100644 --- a/packages/datadog-plugin-aerospike/test/naming.js +++ b/packages/datadog-plugin-aerospike/test/naming.js @@ -14,6 +14,6 @@ const rawExpectedSchema = { } module.exports = { - rawExpectedSchema: rawExpectedSchema, + rawExpectedSchema, expectedSchema: resolveNaming(rawExpectedSchema) } diff --git a/packages/datadog-plugin-amqp10/test/index.spec.js b/packages/datadog-plugin-amqp10/test/index.spec.js index d6ac0b999ae..7a2ca7dbd9e 100644 --- a/packages/datadog-plugin-amqp10/test/index.spec.js +++ b/packages/datadog-plugin-amqp10/test/index.spec.js @@ -14,6 +14,7 @@ describe('Plugin', () => { describe('amqp10', () => { before(() => agent.load('rhea')) + after(() => agent.close({ ritmReset: false })) withVersions('amqp10', 'amqp10', version => { @@ -94,6 +95,7 @@ describe('Plugin', () => { sender.send({ key: 'value' }) }) + it('should handle errors', done => { let error diff --git a/packages/datadog-plugin-amqp10/test/integration-test/client.spec.js b/packages/datadog-plugin-amqp10/test/integration-test/client.spec.js index f4f97ee69ce..8deadf31385 100644 --- a/packages/datadog-plugin-amqp10/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-amqp10/test/integration-test/client.spec.js @@ -16,7 +16,7 @@ describe('esm', () => { before(async function () { this.timeout(20000) sandbox = await createSandbox([`'amqp10@${version}'`, 'rhea'], false, [ - `./packages/datadog-plugin-amqp10/test/integration-test/*`]) + './packages/datadog-plugin-amqp10/test/integration-test/*']) }) after(async () => { diff --git a/packages/datadog-plugin-amqp10/test/leak.js b/packages/datadog-plugin-amqp10/test/leak.js deleted file mode 100644 index 3b065a7cdb7..00000000000 --- a/packages/datadog-plugin-amqp10/test/leak.js +++ /dev/null @@ -1,53 +0,0 @@ -'use strict' - -/* eslint-disable mocha/no-return-and-callback */ - -require('../../dd-trace') - .init({ plugins: false, sampleRate: 0 }) - .use('amqp10') - -const test = require('tape') -const profile = require('../../dd-trace/test/profile') - -test('amqp10 plugin should not leak', t => { - const amqp = require('../../../versions/amqp10').get() - const client = new amqp.Client() - - return client.connect('amqp://admin:admin@localhost:5673') - .then(() => { - return Promise.all([ - client.createReceiver('amq.topic'), - client.createSender('amq.topic') - ]) - }) - .then(handlers => { - const receiver = handlers[0] - const sender = handlers[1] - const deferred = [] - - let messageIdx = 0 - let operationIdx = 0 - - for (let i = 0; i < 2000; i++) { - const promise = new Promise((resolve, reject) => { - deferred[i] = { resolve, reject } - }) - - deferred[i].promise = promise - } - - receiver.on('message', () => { - deferred[messageIdx++].resolve() - }) - - profile(t, operation, 200, 10) - .then(() => receiver.detach()) - .then(() => sender.detach()) - .then(() => client.disconnect()) - - function operation (done) { - deferred[operationIdx++].promise.then(() => done()) - sender.send({ key: 'value' }) - } - }) -}) diff --git a/packages/datadog-plugin-amqp10/test/naming.js b/packages/datadog-plugin-amqp10/test/naming.js index 6e9f332cb5e..e8bc0e42c3e 100644 --- a/packages/datadog-plugin-amqp10/test/naming.js +++ b/packages/datadog-plugin-amqp10/test/naming.js @@ -24,6 +24,6 @@ const rawExpectedSchema = { } module.exports = { - rawExpectedSchema: rawExpectedSchema, + rawExpectedSchema, expectedSchema: resolveNaming(rawExpectedSchema) } diff --git a/packages/datadog-plugin-amqplib/src/consumer.js b/packages/datadog-plugin-amqplib/src/consumer.js index 0aed1696507..92684e3f9dc 100644 --- a/packages/datadog-plugin-amqplib/src/consumer.js +++ b/packages/datadog-plugin-amqplib/src/consumer.js @@ -2,6 +2,7 @@ const { TEXT_MAP } = require('../../../ext/formats') const ConsumerPlugin = require('../../dd-trace/src/plugins/consumer') +const { getAmqpMessageSize } = require('../../dd-trace/src/datastreams/processor') const { getResourceName } = require('./util') class AmqplibConsumerPlugin extends ConsumerPlugin { @@ -13,7 +14,7 @@ class AmqplibConsumerPlugin extends ConsumerPlugin { const childOf = extract(this.tracer, message) - this.startSpan({ + const span = this.startSpan({ childOf, resource: getResourceName(method, fields), type: 'worker', @@ -26,6 +27,16 @@ class AmqplibConsumerPlugin extends ConsumerPlugin { 'amqp.destination': fields.destination } }) + + if ( + this.config.dsmEnabled && message?.properties?.headers + ) { + const payloadSize = getAmqpMessageSize({ headers: message.properties.headers, content: message.content }) + const queue = fields.queue ? fields.queue : fields.routingKey + this.tracer.decodeDataStreamsContext(message.properties.headers) + this.tracer + .setCheckpoint(['direction:in', `topic:${queue}`, 'type:rabbitmq'], span, payloadSize) + } } } diff --git a/packages/datadog-plugin-amqplib/src/producer.js b/packages/datadog-plugin-amqplib/src/producer.js index 9c3d1da8d53..5f299c80a45 100644 --- a/packages/datadog-plugin-amqplib/src/producer.js +++ b/packages/datadog-plugin-amqplib/src/producer.js @@ -3,13 +3,15 @@ const { TEXT_MAP } = require('../../../ext/formats') const { CLIENT_PORT_KEY } = require('../../dd-trace/src/constants') const ProducerPlugin = require('../../dd-trace/src/plugins/producer') +const { DsmPathwayCodec } = require('../../dd-trace/src/datastreams/pathway') +const { getAmqpMessageSize } = require('../../dd-trace/src/datastreams/processor') const { getResourceName } = require('./util') class AmqplibProducerPlugin extends ProducerPlugin { static get id () { return 'amqplib' } static get operation () { return 'command' } - start ({ channel = {}, method, fields }) { + start ({ channel = {}, method, fields, message }) { if (method !== 'basic.publish') return const stream = (channel.connection && channel.connection.stream) || {} @@ -30,6 +32,16 @@ class AmqplibProducerPlugin extends ProducerPlugin { fields.headers = fields.headers || {} this.tracer.inject(span, TEXT_MAP, fields.headers) + + if (this.config.dsmEnabled) { + const hasRoutingKey = fields.routingKey != null + const payloadSize = getAmqpMessageSize({ content: message, headers: fields.headers }) + const dataStreamsContext = this.tracer + .setCheckpoint( + ['direction:out', `exchange:${fields.exchange}`, `has_routing_key:${hasRoutingKey}`, 'type:rabbitmq'] + , span, payloadSize) + DsmPathwayCodec.encode(dataStreamsContext, fields.headers) + } } } diff --git a/packages/datadog-plugin-amqplib/test/index.spec.js b/packages/datadog-plugin-amqplib/test/index.spec.js index b6f73212a7f..d65a5c99338 100644 --- a/packages/datadog-plugin-amqplib/test/index.spec.js +++ b/packages/datadog-plugin-amqplib/test/index.spec.js @@ -13,6 +13,7 @@ describe('Plugin', () => { describe('amqplib', () => { withVersions('amqplib', 'amqplib', version => { beforeEach(() => { + process.env.DD_DATA_STREAMS_ENABLED = 'true' tracer = require('../../dd-trace') }) @@ -275,6 +276,7 @@ describe('Plugin', () => { channel.assertQueue('', {}, (err, ok) => { if (err) return channel.sendToQueue(ok.queue, Buffer.from('content')) + // eslint-disable-next-line n/handle-callback-err channel.consume(ok.queue, () => {}, {}, (err, ok) => {}) }) }, @@ -300,6 +302,116 @@ describe('Plugin', () => { .catch(done) }) }) + + describe('when data streams monitoring is enabled', function () { + this.timeout(10000) + + const expectedProducerHash = '17191234428405871432' + const expectedConsumerHash = '18277095184718602853' + + before(() => { + tracer = require('../../dd-trace') + tracer.use('amqplib') + }) + + before(async () => { + return agent.load('amqplib') + }) + + after(() => { + return agent.close({ ritmReset: false }) + }) + + it('Should emit DSM stats to the agent when sending a message', done => { + agent.expectPipelineStats(dsmStats => { + let statsPointsReceived = 0 + // we should have 1 dsm stats points + dsmStats.forEach((timeStatsBucket) => { + if (timeStatsBucket && timeStatsBucket.Stats) { + timeStatsBucket.Stats.forEach((statsBuckets) => { + statsPointsReceived += statsBuckets.Stats.length + }) + } + }) + expect(statsPointsReceived).to.be.at.least(1) + expect(agent.dsmStatsExist(agent, expectedProducerHash)).to.equal(true) + }, { timeoutMs: 10000 }).then(done, done) + + channel.assertQueue('testDSM', {}, (err, ok) => { + if (err) return done(err) + + channel.sendToQueue(ok.queue, Buffer.from('DSM pathway test')) + }) + }) + + it('Should emit DSM stats to the agent when receiving a message', done => { + agent.expectPipelineStats(dsmStats => { + let statsPointsReceived = 0 + // we should have 2 dsm stats points + dsmStats.forEach((timeStatsBucket) => { + if (timeStatsBucket && timeStatsBucket.Stats) { + timeStatsBucket.Stats.forEach((statsBuckets) => { + statsPointsReceived += statsBuckets.Stats.length + }) + } + }) + expect(statsPointsReceived).to.be.at.least(1) + expect(agent.dsmStatsExist(agent, expectedConsumerHash)).to.equal(true) + }, { timeoutMs: 10000 }).then(done, done) + + channel.assertQueue('testDSM', {}, (err, ok) => { + if (err) return done(err) + + channel.consume(ok.queue, () => {}, {}, (err, ok) => { + if (err) done(err) + }) + }) + }) + + it('Should set pathway hash tag on a span when producing', (done) => { + channel.assertQueue('testDSM', {}, (err, ok) => { + if (err) return done(err) + + channel.sendToQueue(ok.queue, Buffer.from('dsm test')) + + let produceSpanMeta = {} + agent.use(traces => { + const span = traces[0][0] + + if (span.resource.startsWith('basic.publish')) { + produceSpanMeta = span.meta + } + + expect(produceSpanMeta).to.include({ + 'pathway.hash': expectedProducerHash + }) + }, { timeoutMs: 10000 }).then(done, done) + }) + }) + + it('Should set pathway hash tag on a span when consuming', (done) => { + channel.assertQueue('testDSM', {}, (err, ok) => { + if (err) return done(err) + + channel.consume(ok.queue, () => {}, {}, (err, ok) => { + if (err) return done(err) + + let consumeSpanMeta = {} + agent.use(traces => { + const span = traces[0][0] + + if (span.resource.startsWith('basic.deliver')) { + consumeSpanMeta = span.meta + } + + expect(consumeSpanMeta).to.include({ + 'pathway.hash': expectedConsumerHash + }) + }, { timeoutMs: 10000 }).then(done, done) + }) + }) + }) + }) }) describe('with configuration', () => { diff --git a/packages/datadog-plugin-amqplib/test/integration-test/client.spec.js b/packages/datadog-plugin-amqplib/test/integration-test/client.spec.js index 1a7f0341210..f7fda5fa651 100644 --- a/packages/datadog-plugin-amqplib/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-amqplib/test/integration-test/client.spec.js @@ -17,7 +17,7 @@ describe('esm', () => { before(async function () { this.timeout(20000) sandbox = await createSandbox([`'amqplib@${version}'`], false, - [`./packages/datadog-plugin-amqplib/test/integration-test/*`]) + ['./packages/datadog-plugin-amqplib/test/integration-test/*']) }) after(async () => { diff --git a/packages/datadog-plugin-amqplib/test/leak.js b/packages/datadog-plugin-amqplib/test/leak.js deleted file mode 100644 index 933aa583b9f..00000000000 --- a/packages/datadog-plugin-amqplib/test/leak.js +++ /dev/null @@ -1,40 +0,0 @@ -'use strict' - -require('../../dd-trace') - .init({ plugins: false, sampleRate: 0 }) - .use('amqplib') - -const test = require('tape') -const profile = require('../../dd-trace/test/profile') - -test('amqplib plugin should not leak when using callbacks', t => { - require('../../../versions/amqplib').get('amqplib/callback_api') - .connect((err, conn) => { - if (err) return t.fail(err) - - conn.createChannel((err, ch) => { - if (err) return t.fail(err) - - profile(t, operation, 400).then(() => conn.close()) - - function operation (done) { - ch.assertQueue('test', {}, done) - } - }) - }) -}) - -test('amqplib plugin should not leak when using promises', t => { - require('../../../versions/amqplib').get().connect() - .then(conn => { - return conn.createChannel() - .then(ch => { - profile(t, operation, 400).then(() => conn.close()) - - function operation (done) { - ch.assertQueue('test', {}).then(done) - } - }) - }) - .catch(t.fail) -}) diff --git a/packages/datadog-plugin-amqplib/test/naming.js b/packages/datadog-plugin-amqplib/test/naming.js index ec522c5ed63..fdb90a66d8c 100644 --- a/packages/datadog-plugin-amqplib/test/naming.js +++ b/packages/datadog-plugin-amqplib/test/naming.js @@ -34,6 +34,6 @@ const rawExpectedSchema = { } module.exports = { - rawExpectedSchema: rawExpectedSchema, + rawExpectedSchema, expectedSchema: resolveNaming(rawExpectedSchema) } diff --git a/packages/datadog-plugin-apollo/src/gateway/execute.js b/packages/datadog-plugin-apollo/src/gateway/execute.js new file mode 100644 index 00000000000..80112a047a8 --- /dev/null +++ b/packages/datadog-plugin-apollo/src/gateway/execute.js @@ -0,0 +1,12 @@ +'use strict' + +const ApolloBasePlugin = require('../../../dd-trace/src/plugins/apollo') + +class ApolloGatewayExecutePlugin extends ApolloBasePlugin { + static get operation () { return 'execute' } + static get prefix () { + return 'tracing:apm:apollo:gateway:execute' + } +} + +module.exports = ApolloGatewayExecutePlugin diff --git a/packages/datadog-plugin-apollo/src/gateway/fetch.js b/packages/datadog-plugin-apollo/src/gateway/fetch.js new file mode 100644 index 00000000000..fc1a3d82837 --- /dev/null +++ b/packages/datadog-plugin-apollo/src/gateway/fetch.js @@ -0,0 +1,36 @@ +'use strict' + +const { storage } = require('../../../datadog-core') +const ApolloBasePlugin = require('../../../dd-trace/src/plugins/apollo') + +class ApolloGatewayFetchPlugin extends ApolloBasePlugin { + static get operation () { return 'fetch' } + static get prefix () { + return 'tracing:apm:apollo:gateway:fetch' + } + + bindStart (ctx) { + const store = storage.getStore() + const childOf = store ? store.span : null + + const spanData = { + childOf, + service: this.getServiceName(), + type: this.constructor.type, + meta: {} + } + + const serviceName = ctx?.attributes?.service + + if (serviceName) { spanData.meta.serviceName = serviceName } + + const span = this.startSpan(this.getOperationName(), spanData, false) + + ctx.parentStore = store + ctx.currentStore = { ...store, span } + + return ctx.currentStore + } +} + +module.exports = ApolloGatewayFetchPlugin diff --git a/packages/datadog-plugin-apollo/src/gateway/index.js b/packages/datadog-plugin-apollo/src/gateway/index.js new file mode 100644 index 00000000000..e94f19d38ca --- /dev/null +++ b/packages/datadog-plugin-apollo/src/gateway/index.js @@ -0,0 +1,36 @@ +'use strict' + +const { storage } = require('../../../datadog-core') +const CompositePlugin = require('../../../dd-trace/src/plugins/composite') +const ApolloGatewayExecutePlugin = require('./execute') +const ApolloGatewayPostProcessingPlugin = require('./postprocessing') +const ApolloGatewayRequestPlugin = require('./request') +const ApolloGatewayPlanPlugin = require('./plan') +const ApolloGatewayValidatePlugin = require('./validate') +const ApolloGatewayFetchPlugin = require('./fetch') + +class ApolloGatewayPlugin extends CompositePlugin { + static get id () { return 'gateway' } + static get plugins () { + return { + execute: ApolloGatewayExecutePlugin, + postprocessing: ApolloGatewayPostProcessingPlugin, + request: ApolloGatewayRequestPlugin, + plan: ApolloGatewayPlanPlugin, + fetch: ApolloGatewayFetchPlugin, + validate: ApolloGatewayValidatePlugin + } + } + + constructor (...args) { + super(...args) + this.addSub('apm:apollo:gateway:general:error', (ctx) => { + const store = storage.getStore() + const span = store?.span + if (!span) return + span.setTag('error', ctx.error) + }) + } +} + +module.exports = ApolloGatewayPlugin diff --git a/packages/datadog-plugin-apollo/src/gateway/plan.js b/packages/datadog-plugin-apollo/src/gateway/plan.js new file mode 100644 index 00000000000..33123184cbd --- /dev/null +++ b/packages/datadog-plugin-apollo/src/gateway/plan.js @@ -0,0 +1,12 @@ +'use strict' + +const ApolloBasePlugin = require('../../../dd-trace/src/plugins/apollo') + +class ApolloGatewayPlanPlugin extends ApolloBasePlugin { + static get operation () { return 'plan' } + static get prefix () { + return 'tracing:apm:apollo:gateway:plan' + } +} + +module.exports = ApolloGatewayPlanPlugin diff --git a/packages/datadog-plugin-apollo/src/gateway/postprocessing.js b/packages/datadog-plugin-apollo/src/gateway/postprocessing.js new file mode 100644 index 00000000000..69509d843b4 --- /dev/null +++ b/packages/datadog-plugin-apollo/src/gateway/postprocessing.js @@ -0,0 +1,12 @@ +'use strict' + +const ApolloBasePlugin = require('../../../dd-trace/src/plugins/apollo') + +class ApolloGatewayPostProcessingPlugin extends ApolloBasePlugin { + static get operation () { return 'postprocessing' } + static get prefix () { + return 'tracing:apm:apollo:gateway:postprocessing' + } +} + +module.exports = ApolloGatewayPostProcessingPlugin diff --git a/packages/datadog-plugin-apollo/src/gateway/request.js b/packages/datadog-plugin-apollo/src/gateway/request.js new file mode 100644 index 00000000000..740f487c759 --- /dev/null +++ b/packages/datadog-plugin-apollo/src/gateway/request.js @@ -0,0 +1,124 @@ +'use strict' + +const { storage } = require('../../../datadog-core') +const ApolloBasePlugin = require('../../../dd-trace/src/plugins/apollo') + +let tools + +const OPERATION_DEFINITION = 'OperationDefinition' +const FRAGMENT_DEFINITION = 'FragmentDefinition' + +class ApolloGatewayRequestPlugin extends ApolloBasePlugin { + static get operation () { return 'request' } + static get prefix () { + return 'tracing:apm:apollo:gateway:request' + } + + bindStart (ctx) { + const store = storage.getStore() + const childOf = store ? store.span : null + const spanData = { + childOf, + service: this.serviceName( + { id: `${this.constructor.id}.${this.constructor.operation}`, pluginConfig: this.config }), + type: this.constructor.type, + kind: this.constructor.kind, + meta: {} + } + + const { requestContext, gateway } = ctx + + if (requestContext?.operationName) { + spanData.meta['graphql.operation.name'] = requestContext.operationName + } + if ((this.config.source || gateway?.config?.telemetry?.includeDocument) && requestContext?.source) { + spanData.meta['graphql.source'] = requestContext.source + } + + const operationContext = + buildOperationContext(gateway.schema, requestContext.document, requestContext.request.operationName) + + if (operationContext?.operation?.operation) { + const document = requestContext?.document + const type = operationContext?.operation?.operation + const name = operationContext?.operation?.name && operationContext?.operation?.name?.value + + spanData.resource = getSignature(document, name, type, this?.config?.signature) + spanData.meta['graphql.operation.type'] = type + } + const span = this.startSpan(this.operationName({ id: `${this.constructor.id}.${this.constructor.operation}` }), + spanData, false) + + ctx.parentStore = store + ctx.currentStore = { ...store, span } + return ctx.currentStore + } + + asyncStart (ctx) { + const errors = ctx?.result?.errors + // apollo gateway catches certain errors and returns them in the result object + // we want to capture these errors as spans + if (errors instanceof Array && + errors[errors.length - 1] && errors[errors.length - 1].stack && errors[errors.length - 1].message) { + ctx.currentStore.span.setTag('error', errors[errors.length - 1]) + } + ctx.currentStore.span.finish() + return ctx.parentStore + } +} + +function buildOperationContext (schema, operationDocument, operationName) { + let operation + let operationCount = 0 + const fragments = Object.create(null) + try { + operationDocument.definitions.forEach(definition => { + switch (definition.kind) { + case OPERATION_DEFINITION: + operationCount++ + if (!operationName && operationCount > 1) { + return + } + if ( + !operationName || + (definition.name && definition.name.value === operationName) + ) { + operation = definition + } + break + case FRAGMENT_DEFINITION: + fragments[definition.name.value] = definition + break + } + }) + } catch (e) { + // safety net + } + + return { + schema, + operation, + fragments + } +} + +function getSignature (document, operationName, operationType, calculate) { + if (calculate !== false && tools !== false) { + try { + try { + tools = tools || require('../../../datadog-plugin-graphql/src/tools') + } catch (e) { + tools = false + throw e + } + + return tools.defaultEngineReportingSignature(document, operationName) + } catch (e) { + // safety net + } + } + + return [operationType, operationName].filter(val => val).join(' ') +} + +module.exports = ApolloGatewayRequestPlugin diff --git a/packages/datadog-plugin-apollo/src/gateway/validate.js b/packages/datadog-plugin-apollo/src/gateway/validate.js new file mode 100644 index 00000000000..098bdb5f1c5 --- /dev/null +++ b/packages/datadog-plugin-apollo/src/gateway/validate.js @@ -0,0 +1,25 @@ +'use strict' + +const ApolloBasePlugin = require('../../../dd-trace/src/plugins/apollo') + +class ApolloGatewayValidatePlugin extends ApolloBasePlugin { + static get operation () { return 'validate' } + static get prefix () { + return 'tracing:apm:apollo:gateway:validate' + } + + end (ctx) { + const result = ctx.result + const span = ctx.currentStore?.span + + if (!span) return + + if (result instanceof Array && + result[result.length - 1] && result[result.length - 1].stack && result[result.length - 1].message) { + span.setTag('error', result[result.length - 1]) + } + span.finish() + } +} + +module.exports = ApolloGatewayValidatePlugin diff --git a/packages/datadog-plugin-apollo/src/index.js b/packages/datadog-plugin-apollo/src/index.js new file mode 100644 index 00000000000..95f6acdf9cd --- /dev/null +++ b/packages/datadog-plugin-apollo/src/index.js @@ -0,0 +1,15 @@ +'use strict' + +const CompositePlugin = require('../../dd-trace/src/plugins/composite') +const ApolloGatewayPlugin = require('./gateway') + +class ApolloPlugin extends CompositePlugin { + static get id () { return 'apollo' } + static get plugins () { + return { + gateway: ApolloGatewayPlugin + } + } +} + +module.exports = ApolloPlugin diff --git a/packages/datadog-plugin-apollo/test/fixtures.js b/packages/datadog-plugin-apollo/test/fixtures.js new file mode 100644 index 00000000000..6e0a992f5ca --- /dev/null +++ b/packages/datadog-plugin-apollo/test/fixtures.js @@ -0,0 +1,76 @@ +const typeDefs = ` + type Query { + hello(name: String, title: String): String + human: Human + friends: [Human] + } + + type Mutation { + human: Human + } + + type Subscription { + human: Human + } + + type Human { + name: String + address: Address + pets: [Pet] + } + + type Address { + civicNumber: String + street: String + } + + type Pet { + type: String + name: String + owner: Human + colours: [Colour] + } + + type Colour { + code: String + } +` + +const resolvers = { + Query: { + hello: (_, args) => args.name, + human: () => Promise.resolve({}), + friends: () => [{ name: 'alice' }, { name: 'bob' }] + }, + Mutation: { + human: () => Promise.resolve({ name: 'human name' }) + }, + Subscription: { + human: () => Promise.resolve({ name: 'human name' }) + }, + Human: { + name: () => 'test', + address: () => ({}), + pets: () => [{}, {}, {}] + }, + Address: { + civicNumber: () => '123', + street: () => 'foo street' + }, + Pet: { + type: () => 'dog', + name: () => 'foo bar', + owner: () => ({}), + colours: () => [{}, {}] + }, + Colour: { + code: () => '#ffffff' + } +} + +const name = 'accounts' + +exports.name = name +exports.typeDefs = typeDefs +exports.url = `https://${name}.api.com.invalid` +exports.resolvers = resolvers diff --git a/packages/datadog-plugin-apollo/test/index.spec.js b/packages/datadog-plugin-apollo/test/index.spec.js new file mode 100644 index 00000000000..56a193bb606 --- /dev/null +++ b/packages/datadog-plugin-apollo/test/index.spec.js @@ -0,0 +1,506 @@ +'use strict' + +const { expect } = require('chai') +const agent = require('../../dd-trace/test/plugins/agent.js') +const { ERROR_MESSAGE, ERROR_TYPE, ERROR_STACK } = require('../../dd-trace/src/constants.js') +const { expectedSchema, rawExpectedSchema } = require('./naming.js') +const axios = require('axios') + +const accounts = require('./fixtures.js') + +const graphqlTag = require('../../../versions/graphql-tag/index.js').get() +const gql = graphqlTag.gql +accounts.typeDefs = gql(accounts.typeDefs) + +const fixtures = [accounts] + +async function execute (executor, source, variables, operationName) { + const resp = await executor({ + source, + document: gql(source), + request: { + variables + }, + operationName, + queryHash: 'hashed', + context: null, + cache: {} + }) + return resp +} + +describe('Plugin', () => { + let ApolloGateway + let LocalGraphQLDataSource + let buildSubgraphSchema + let ApolloServer + let startStandaloneServer + + function setupGateway () { + const localDataSources = Object.fromEntries( + fixtures.map((f) => [ + f.name, + new LocalGraphQLDataSource(buildSubgraphSchema(f)) + ]) + ) + + const gateway = new ApolloGateway({ + localServiceList: fixtures, + buildService (service) { + return localDataSources[service.name] + } + }) + return gateway + } + + function gateway () { + return setupGateway().load().then((res) => res) + } + + describe('@apollo/gateway', () => { + withVersions('apollo', '@apollo/gateway', version => { + before(() => { + require('../../dd-trace/index.js') + const apollo = require(`../../../versions/@apollo/gateway@${version}`).get() + const subgraph = require('../../../versions/@apollo/subgraph').get() + buildSubgraphSchema = subgraph.buildSubgraphSchema + ApolloGateway = apollo.ApolloGateway + LocalGraphQLDataSource = apollo.LocalGraphQLDataSource + }) + after(() => { + return agent.close({ ritmReset: false }) + }) + + describe('@apollo/server', () => { + let server + let port + + before(() => { + ApolloServer = require('../../../versions/@apollo/server/index.js').get().ApolloServer + startStandaloneServer = + require('../../../versions/@apollo/server@4.0.0/node_modules/@apollo/server/dist/cjs/standalone/index.js') + .startStandaloneServer + + server = new ApolloServer({ + gateway: setupGateway(), + subscriptions: false // Disable subscriptions (not supported with Apollo Gateway) + }) + + return startStandaloneServer(server, { + listen: { port: 0 } + }).then(({ url }) => { + port = new URL(url).port + }) + }) + + before(() => { + return agent.load('apollo') + }) + + after(() => { + server.stop() + }) + + it('should instrument apollo/gateway when using apollo server', done => { + const query = ` + query ExampleQuery { + human { + name + } + friends { + name + } + }` + agent + .use((traces) => { + expect(traces[0][0]).to.have.property('name', expectedSchema.server.opName) + expect(traces[0][1]).to.have.property('name', 'apollo.gateway.validate') + expect(traces[0][2]).to.have.property('name', 'apollo.gateway.plan') + expect(traces[0][3]).to.have.property('name', 'apollo.gateway.execute') + expect(traces[0][4]).to.have.property('name', 'apollo.gateway.fetch') + expect(traces[0][5]).to.have.property('name', 'apollo.gateway.postprocessing') + }) + .then(done) + .catch(done) + + axios.post(`http://localhost:${port}/`, { + query + }) + }) + }) + + describe('without configuration', () => { + before(() => { + return agent.load('apollo') + }) + + it('should instrument apollo/gateway', done => { + const operationName = 'MyQuery' + const source = `query ${operationName} { hello(name: "world") }` + const variableValues = { who: 'world' } + agent + .use((traces) => { + // the spans are in order of execution + expect(traces[0][0]).to.have.property('name', expectedSchema.server.opName) + expect(traces[0][0]).to.have.property('service', expectedSchema.server.serviceName) + expect(traces[0][0]).to.have.property('resource', 'query MyQuery{hello(name:"")}') + expect(traces[0][0]).to.have.property('type', 'web') + expect(traces[0][0]).to.have.property('error', 0) + expect(traces[0][0].meta).to.have.property('graphql.operation.name', operationName) + expect(traces[0][0].meta).to.not.have.property('graphql.source') + expect(traces[0][0].meta).to.have.property('graphql.operation.type', 'query') + expect(traces[0][0].meta).to.have.property('component', 'apollo.gateway') + + expect(traces[0][1]).to.have.property('name', 'apollo.gateway.validate') + expect(traces[0][1]).to.have.property('service', expectedSchema.server.serviceName) + expect(traces[0][1]).to.have.property('type', 'web') + expect(traces[0][1]).to.have.property('error', 0) + expect(traces[0][1].meta).to.have.property('component', 'apollo.gateway') + + expect(traces[0][2]).to.have.property('name', 'apollo.gateway.plan') + expect(traces[0][2]).to.have.property('service', expectedSchema.server.serviceName) + expect(traces[0][2]).to.have.property('type', 'web') + expect(traces[0][2]).to.have.property('error', 0) + expect(traces[0][2].meta).to.have.property('component', 'apollo.gateway') + + expect(traces[0][3]).to.have.property('name', 'apollo.gateway.execute') + expect(traces[0][3]).to.have.property('service', expectedSchema.server.serviceName) + expect(traces[0][3]).to.have.property('type', 'web') + expect(traces[0][3]).to.have.property('error', 0) + expect(traces[0][3].meta).to.have.property('component', 'apollo.gateway') + + expect(traces[0][4]).to.have.property('name', 'apollo.gateway.fetch') + expect(traces[0][4]).to.have.property('service', expectedSchema.server.serviceName) + expect(traces[0][4]).to.have.property('type', 'web') + expect(traces[0][4]).to.have.property('error', 0) + expect(traces[0][4].meta).to.have.property('serviceName', 'accounts') + expect(traces[0][4].meta).to.have.property('component', 'apollo.gateway') + + expect(traces[0][5]).to.have.property('name', 'apollo.gateway.postprocessing') + expect(traces[0][5]).to.have.property('service', expectedSchema.server.serviceName) + expect(traces[0][5]).to.have.property('type', 'web') + expect(traces[0][5]).to.have.property('error', 0) + expect(traces[0][5].meta).to.have.property('component', 'apollo.gateway') + }) + .then(done) + .catch(done) + + gateway() + .then(({ executor }) => { + return execute(executor, source, variableValues, operationName).then(() => {}) + }) + }) + + it('should instrument schema resolver', done => { + const source = '{ hello(name: "world") }' + agent + .use((traces) => { + expect(traces[0][0]).to.have.property('name', expectedSchema.server.opName) + expect(traces[0][0]).to.have.property('service', expectedSchema.server.serviceName) + expect(traces[0][0]).to.have.property('resource', '{hello(name:"")}') + expect(traces[0][0]).to.have.property('type', 'web') + expect(traces[0][0]).to.have.property('error', 0) + expect(traces[0][0].meta).to.not.have.property('graphql.source') + expect(traces[0][0].meta).to.have.property('graphql.operation.type', 'query') + expect(traces[0][0].meta).to.have.property('component', 'apollo.gateway') + }) + .then(done) + .catch(done) + + gateway() + .then(({ executor }) => { + return execute(executor, source).then(() => {}) + }) + }) + + it('should instrument nested field resolvers', done => { + const source = ` + { + human { + name + address { + civicNumber + street + } + } + } + ` + agent + .use((traces) => { + expect(traces[0][0]).to.have.property('name', expectedSchema.server.opName) + expect(traces[0][0]).to.have.property('service', expectedSchema.server.serviceName) + expect(traces[0][0]).to.have.property('resource', '{human{address{civicNumber street}name}}') + expect(traces[0][0]).to.have.property('type', 'web') + expect(traces[0][0]).to.have.property('error', 0) + expect(traces[0][0].meta).to.not.have.property('graphql.source') + expect(traces[0][0].meta).to.have.property('graphql.operation.type', 'query') + expect(traces[0][0].meta).to.have.property('component', 'apollo.gateway') + }) + .then(done) + .catch(done) + + gateway() + .then(({ executor }) => { + return execute(executor, source).then(() => {}) + }) + }) + + it('should instrument mutations', done => { + const source = 'mutation { human { name } }' + + agent + .use((traces) => { + expect(traces[0][0].meta).to.have.property('graphql.operation.type', 'mutation') + }) + .then(done) + .catch(done) + + gateway() + .then(({ executor }) => { + return execute(executor, source).then(() => {}) + }) + }) + + it('should handle a circular schema', done => { + const source = '{ human { pets { owner { name } } } }' + + gateway() + .then(({ executor }) => { + return execute(executor, source).then((result) => { + expect(result.data.human.pets[0].owner.name).to.equal('test') + }) + .then(done) + .catch(done) + }) + }) + + it('should instrument validation failure', done => { + let error + const source = `#graphql + query InvalidVariables($first: Int!, $second: Int!) { + topReviews(first: $first) { + body + } + }` + const variableValues = { who: 'world' } + agent + .use((traces) => { + expect(traces[0].length).equal(2) + expect(traces[0][0]).to.have.property('name', expectedSchema.server.opName) + expect(traces[0][0]).to.have.property('service', expectedSchema.server.serviceName) + expect(traces[0][0]).to.have.property('error', 1) + expect(traces[0][0].meta).to.have.property(ERROR_TYPE, error.name) + expect(traces[0][0].meta).to.have.property(ERROR_MESSAGE, error.message) + expect(traces[0][0].meta).to.have.property(ERROR_STACK, error.stack) + expect(traces[0][0].meta).to.have.property('component', 'apollo.gateway') + + expect(traces[0][1]).to.have.property('name', 'apollo.gateway.validate') + expect(traces[0][1]).to.have.property('service', expectedSchema.server.serviceName) + expect(traces[0][1]).to.have.property('error', 1) + expect(traces[0][1].meta).to.have.property(ERROR_TYPE, error.name) + expect(traces[0][1].meta).to.have.property(ERROR_MESSAGE, error.message) + expect(traces[0][1].meta).to.have.property(ERROR_STACK, error.stack) + expect(traces[0][1].meta).to.have.property('component', 'apollo.gateway') + }) + .then(done) + .catch(done) + + gateway() + .then(({ executor }) => { + return execute(executor, source, variableValues, 'InvalidVariables').then((result) => { + error = result.errors[1] + }) + }) + }) + + it('should instrument plan failure', done => { + let error + const operationName = 'MyQuery' + const source = `subscription ${operationName} { hello(name: "world") }` + const variableValues = { who: 'world' } + agent + .use((traces) => { + expect(traces[0].length).equal(3) + expect(traces[0][0]).to.have.property('name', expectedSchema.server.opName) + expect(traces[0][0]).to.have.property('service', expectedSchema.server.serviceName) + expect(traces[0][0]).to.have.property('error', 1) + + expect(traces[0][1]).to.have.property('name', 'apollo.gateway.validate') + expect(traces[0][1]).to.have.property('error', 0) + + expect(traces[0][2]).to.have.property('name', 'apollo.gateway.plan') + expect(traces[0][2]).to.have.property('service', expectedSchema.server.serviceName) + expect(traces[0][2]).to.have.property('error', 1) + expect(traces[0][2].meta).to.have.property(ERROR_TYPE, error.name) + expect(traces[0][2].meta).to.have.property(ERROR_MESSAGE, error.message) + expect(traces[0][2].meta).to.have.property(ERROR_STACK, error.stack) + }) + .then(done) + .catch(done) + + gateway() + .then(({ executor }) => { + return execute(executor, source, variableValues, operationName) + .then(() => {}) + .catch((e) => { + error = e + }) + }) + }) + + it('should instrument fetch failure', done => { + let error + const operationName = 'MyQuery' + const source = `query ${operationName} { hello(name: "world") }` + const variableValues = { who: 'world' } + agent + .use((traces) => { + expect(traces[0][0]).to.have.property('name', expectedSchema.server.opName) + expect(traces[0][0]).to.have.property('error', 1) + expect(traces[0][0].meta).to.have.property(ERROR_TYPE, error.name) + expect(traces[0][0].meta).to.have.property(ERROR_MESSAGE, error.message) + expect(traces[0][0].meta).to.have.property(ERROR_STACK, error.stack) + + expect(traces[0][1]).to.have.property('name', 'apollo.gateway.validate') + expect(traces[0][1]).to.have.property('error', 0) + + expect(traces[0][2]).to.have.property('name', 'apollo.gateway.plan') + expect(traces[0][2]).to.have.property('service', expectedSchema.server.serviceName) + expect(traces[0][2]).to.have.property('error', 0) + + expect(traces[0][3]).to.have.property('name', 'apollo.gateway.execute') + // In order to mimick the ApolloGateway instrumentation we also patch + // the call to the recordExceptions() method by ApolloGateway + // in version 2.3.0, there is no recordExceptions method thus we can't ever attach an error to the + // fetch span but instead the error will be propagated to the request span and be set there + if (version > '2.3.0') { + expect(traces[0][3]).to.have.property('error', 1) + expect(traces[0][3].meta).to.have.property(ERROR_TYPE, error.name) + expect(traces[0][3].meta).to.have.property(ERROR_MESSAGE, error.message) + expect(traces[0][3].meta).to.have.property(ERROR_STACK, error.stack) + } else { expect(traces[0][3]).to.have.property('error', 0) } + + expect(traces[0][4]).to.have.property('name', 'apollo.gateway.fetch') + expect(traces[0][4]).to.have.property('service', expectedSchema.server.serviceName) + expect(traces[0][4]).to.have.property('error', 1) + expect(traces[0][4].meta).to.have.property(ERROR_TYPE, error.name) + expect(traces[0][4].meta).to.have.property(ERROR_MESSAGE, error.message) + expect(traces[0][4].meta).to.have.property(ERROR_STACK, error.stack) + + expect(traces[0][5]).to.have.property('name', 'apollo.gateway.postprocessing') + expect(traces[0][5]).to.have.property('error', 0) + }) + .then(done) + .catch(done) + + const gateway = new ApolloGateway({ + localServiceList: fixtures, + fetcher: () => { + throw Error('Nooo') + } + }) + gateway.load().then(resp => { + return execute(resp.executor, source, variableValues, operationName) + .then((result) => { + const errors = result.errors + error = errors[errors.length - 1] + }) + }) + }) + + it('should run spans in the correct context', done => { + const operationName = 'MyQuery' + const source = `query ${operationName} { hello(name: "world") }` + const variableValues = { who: 'world' } + + agent + .use((traces) => { + // the spans are in order of execution + expect(traces[0][0]).to.have.property('name', expectedSchema.server.opName) + + expect(traces[0][1]).to.have.property('name', 'apollo.gateway.validate') + expect(traces[0][1].parent_id.toString()).to.equal(traces[0][0].span_id.toString()) + + expect(traces[0][2]).to.have.property('name', 'apollo.gateway.plan') + expect(traces[0][2].parent_id.toString()).to.equal(traces[0][0].span_id.toString()) + + expect(traces[0][3]).to.have.property('name', 'apollo.gateway.execute') + expect(traces[0][3].parent_id.toString()).to.equal(traces[0][0].span_id.toString()) + + expect(traces[0][4]).to.have.property('name', 'apollo.gateway.fetch') + expect(traces[0][4].parent_id.toString()).to.equal(traces[0][3].span_id.toString()) + + expect(traces[0][5]).to.have.property('name', 'apollo.gateway.postprocessing') + expect(traces[0][5].parent_id.toString()).to.equal(traces[0][3].span_id.toString()) + }) + .then(done) + .catch(done) + + gateway() + .then(({ executor }) => { + return execute(executor, source, variableValues, operationName).then(() => {}) + }) + }) + + withNamingSchema( + () => { + const operationName = 'MyQuery' + const source = `query ${operationName} { hello(name: "world") }` + const variableValues = { who: 'world' } + gateway() + .then(({ executor }) => { + return execute(executor, source, variableValues, operationName).then(() => {}) + }) + }, + rawExpectedSchema.server, + { + selectSpan: (traces) => { + return traces[0][0] + } + } + ) + + describe('with configuration', () => { + before(() => { + return agent.load('apollo', { service: 'custom', source: true, signature: false }) + }) + + it('should be configured with the correct values', done => { + const operationName = 'MyQuery' + const source = `query ${operationName} { hello(name: "world") }` + const variableValues = { who: 'world' } + agent + .use((traces) => { + expect(traces[0][0]).to.have.property('name', expectedSchema.server.opName) + expect(traces[0][0]).to.have.property('service', 'custom') + expect(traces[0][0]).to.have.property('resource', `query ${operationName}`) + expect(traces[0][0].meta).to.have.property('graphql.source', source) + + expect(traces[0][1]).to.have.property('name', 'apollo.gateway.validate') + expect(traces[0][1]).to.have.property('service', 'custom') + + expect(traces[0][2]).to.have.property('name', 'apollo.gateway.plan') + expect(traces[0][2]).to.have.property('service', 'custom') + + expect(traces[0][3]).to.have.property('name', 'apollo.gateway.execute') + expect(traces[0][3]).to.have.property('service', 'custom') + + expect(traces[0][4]).to.have.property('name', 'apollo.gateway.fetch') + expect(traces[0][4]).to.have.property('service', 'custom') + + expect(traces[0][5]).to.have.property('name', 'apollo.gateway.postprocessing') + expect(traces[0][5]).to.have.property('service', 'custom') + }) + .then(done) + .catch(done) + + gateway() + .then(({ executor }) => { + return execute(executor, source, variableValues, operationName).then(() => {}) + }) + }) + }) + }) + }) + }) +}) diff --git a/packages/datadog-plugin-apollo/test/naming.js b/packages/datadog-plugin-apollo/test/naming.js new file mode 100644 index 00000000000..bc8e2247b82 --- /dev/null +++ b/packages/datadog-plugin-apollo/test/naming.js @@ -0,0 +1,19 @@ +const { resolveNaming } = require('../../dd-trace/test/plugins/helpers') + +const rawExpectedSchema = { + server: { + v0: { + opName: 'apollo.gateway.request', + serviceName: 'test' + }, + v1: { + opName: 'apollo.gateway.request', + serviceName: 'test' + } + } +} + +module.exports = { + rawExpectedSchema, + expectedSchema: resolveNaming(rawExpectedSchema) +} diff --git a/packages/datadog-plugin-avsc/src/index.js b/packages/datadog-plugin-avsc/src/index.js new file mode 100644 index 00000000000..be0ef970e50 --- /dev/null +++ b/packages/datadog-plugin-avsc/src/index.js @@ -0,0 +1,9 @@ +const SchemaPlugin = require('../../dd-trace/src/plugins/schema') +const SchemaExtractor = require('./schema_iterator') + +class AvscPlugin extends SchemaPlugin { + static get id () { return 'avsc' } + static get schemaExtractor () { return SchemaExtractor } +} + +module.exports = AvscPlugin diff --git a/packages/datadog-plugin-avsc/src/schema_iterator.js b/packages/datadog-plugin-avsc/src/schema_iterator.js new file mode 100644 index 00000000000..c748bbf9e75 --- /dev/null +++ b/packages/datadog-plugin-avsc/src/schema_iterator.js @@ -0,0 +1,169 @@ +const AVRO = 'avro' +const { + SCHEMA_DEFINITION, + SCHEMA_ID, + SCHEMA_NAME, + SCHEMA_OPERATION, + SCHEMA_WEIGHT, + SCHEMA_TYPE +} = require('../../dd-trace/src/constants') +const log = require('../../dd-trace/src/log') +const { + SchemaBuilder +} = require('../../dd-trace/src/datastreams/schemas/schema_builder') + +class SchemaExtractor { + constructor (schema) { + this.schema = schema + } + + static getType (type) { + const typeMapping = { + string: 'string', + int: 'integer', + long: 'integer', + float: 'number', + double: 'number', + boolean: 'boolean', + bytes: 'string', + record: 'object', + enum: 'string', + array: 'array', + map: 'object', + fixed: 'string' + } + const typeName = type.typeName ?? type.name ?? type + return typeName === 'null' ? typeName : typeMapping[typeName] || 'string' + } + + static extractProperty (field, schemaName, fieldName, builder, depth) { + let array = false + let type + let format + let enumValues + let description + let ref + + const fieldType = field.type?.types ?? field.type?.typeName ?? field.type + + if (Array.isArray(fieldType)) { + // Union Type + type = 'union[' + fieldType.map(t => SchemaExtractor.getType(t.type || t)).join(',') + ']' + } else if (fieldType === 'array') { + // Array Type + array = true + const nestedType = field.type.itemsType.typeName + type = SchemaExtractor.getType(nestedType) + } else if (fieldType === 'record') { + // Nested Record Type + type = 'object' + ref = `#/components/schemas/${field.type.name}` + if (!SchemaExtractor.extractSchema(field.type, builder, depth + 1, this)) { + return false + } + } else if (fieldType === 'enum') { + enumValues = [] + let i = 0 + type = 'string' + while (field.type.symbols[i]) { + enumValues.push(field.type.symbols[i]) + i += 1 + } + } else { + // Primitive type + type = SchemaExtractor.getType(fieldType.type || fieldType) + if (fieldType === 'bytes') { + format = 'byte' + } else if (fieldType === 'int') { + format = 'int32' + } else if (fieldType === 'long') { + format = 'int64' + } else if (fieldType === 'float') { + format = 'float' + } else if (fieldType === 'double') { + format = 'double' + } + } + + return builder.addProperty(schemaName, fieldName, array, type, description, ref, format, enumValues) + } + + static extractSchema (schema, builder, depth, extractor) { + depth += 1 + const schemaName = schema.name + if (extractor) { + // if we already have a defined extractor, this is a nested schema. create a new extractor for the nested + // schema, ensure it is added to our schema builder's cache, and replace the builders iterator with our + // nested schema iterator / extractor. Once complete, add the new schema to our builder's schemas. + const nestedSchemaExtractor = new SchemaExtractor(schema) + builder.iterator = nestedSchemaExtractor + const nestedSchema = SchemaBuilder.getSchema(schemaName, nestedSchemaExtractor, builder) + for (const nestedSubSchemaName in nestedSchema.components.schemas) { + if (nestedSchema.components.schemas.hasOwnProperty(nestedSubSchemaName)) { + builder.schema.components.schemas[nestedSubSchemaName] = nestedSchema.components.schemas[nestedSubSchemaName] + } + } + return true + } else { + if (!builder.shouldExtractSchema(schemaName, depth)) { + return false + } + for (const field of schema.fields) { + if (!this.extractProperty(field, schemaName, field.name, builder, depth)) { + log.warn(`DSM: Unable to extract field with name: ${field.name} from Avro schema with name: ${schemaName}`) + } + } + } + return true + } + + static extractSchemas (descriptor, dataStreamsProcessor) { + return dataStreamsProcessor.getSchema(descriptor.name, new SchemaExtractor(descriptor)) + } + + iterateOverSchema (builder) { + this.constructor.extractSchema(this.schema, builder, 0) + } + + static attachSchemaOnSpan (args, span, operation, tracer) { + const { messageClass } = args + const descriptor = messageClass?.constructor?.type ?? messageClass + + if (!descriptor || !span) { + return + } + + if (span.context()._tags[SCHEMA_TYPE] && operation === 'serialization') { + // we have already added a schema to this span, this call is an encode of nested schema types + return + } + + span.setTag(SCHEMA_TYPE, AVRO) + span.setTag(SCHEMA_NAME, descriptor.name) + span.setTag(SCHEMA_OPERATION, operation) + + if (!tracer._dataStreamsProcessor.canSampleSchema(operation)) { + return + } + + // if the span is unsampled, do not sample the schema + if (!tracer._prioritySampler.isSampled(span)) { + return + } + + const weight = tracer._dataStreamsProcessor.trySampleSchema(operation) + if (weight === 0) { + return + } + + const schemaData = SchemaBuilder.getSchemaDefinition( + this.extractSchemas(descriptor, tracer._dataStreamsProcessor) + ) + + span.setTag(SCHEMA_DEFINITION, schemaData.definition) + span.setTag(SCHEMA_WEIGHT, weight) + span.setTag(SCHEMA_ID, schemaData.id) + } +} + +module.exports = SchemaExtractor diff --git a/packages/datadog-plugin-avsc/test/helpers.js b/packages/datadog-plugin-avsc/test/helpers.js new file mode 100644 index 00000000000..8e5be7ac433 --- /dev/null +++ b/packages/datadog-plugin-avsc/test/helpers.js @@ -0,0 +1,31 @@ +const fs = require('fs') + +async function loadMessage (avro, messageTypeName) { + if (messageTypeName === 'User') { + // Read and parse the Avro schema + const schema = JSON.parse(fs.readFileSync('packages/datadog-plugin-avsc/test/schemas/user.avsc', 'utf8')) + + // Create a file and write Avro data + const filePath = 'packages/datadog-plugin-avsc/test/schemas/users.avro' + + return { + schema, + path: filePath + } + } else if (messageTypeName === 'AdvancedUser') { + // Read and parse the Avro schema + const schema = JSON.parse(fs.readFileSync('packages/datadog-plugin-avsc/test/schemas/advanced_user.avsc', 'utf8')) + + // Create a file and write Avro data + const filePath = 'packages/datadog-plugin-avsc/test/schemas/advanced_users.avro' + + return { + schema, + path: filePath + } + } +} + +module.exports = { + loadMessage +} diff --git a/packages/datadog-plugin-avsc/test/index.spec.js b/packages/datadog-plugin-avsc/test/index.spec.js new file mode 100644 index 00000000000..b3a6db0c1f1 --- /dev/null +++ b/packages/datadog-plugin-avsc/test/index.spec.js @@ -0,0 +1,176 @@ +'use strict' + +const fs = require('fs') +const { expect } = require('chai') +const agent = require('../../dd-trace/test/plugins/agent') +const path = require('path') +const { + SCHEMA_DEFINITION, + SCHEMA_ID, + SCHEMA_NAME, + SCHEMA_OPERATION, + SCHEMA_WEIGHT, + SCHEMA_TYPE +} = require('../../dd-trace/src/constants') +const sinon = require('sinon') +const { loadMessage } = require('./helpers') +const { SchemaBuilder } = require('../../dd-trace/src/datastreams/schemas/schema_builder') + +const BASIC_USER_SCHEMA_DEF = JSON.parse( + fs.readFileSync(path.join(__dirname, 'schemas/expected_user_schema.json'), 'utf8') +) +const ADVANCED_USER_SCHEMA_DEF = JSON.parse( + fs.readFileSync(path.join(__dirname, 'schemas/expected_advanced_user_schema.json'), 'utf8') +) + +const BASIC_USER_SCHEMA_ID = '1605040621379664412' +const ADVANCED_USER_SCHEMA_ID = '919692610494986520' + +function compareJson (expected, span) { + const actual = JSON.parse(span.context()._tags[SCHEMA_DEFINITION]) + return JSON.stringify(actual) === JSON.stringify(expected) +} + +describe('Plugin', () => { + describe('avsc', function () { + this.timeout(0) + let tracer + let avro + let dateNowStub + let mockTime = 0 + + withVersions('avsc', ['avsc'], (version) => { + before(() => { + tracer = require('../../dd-trace').init() + // reset sampled schemas + if (tracer._dataStreamsProcessor?._schemaSamplers) { + tracer._dataStreamsProcessor._schemaSamplers = [] + } + }) + + describe('without configuration', () => { + before(() => { + dateNowStub = sinon.stub(Date, 'now').callsFake(() => { + const returnValue = mockTime + mockTime += 50000 // Increment by 50000 ms to ensure each DSM schema is sampled + return returnValue + }) + const cache = SchemaBuilder.getCache() + cache.clear() + return agent.load('avsc').then(() => { + avro = require(`../../../versions/avsc@${version}`).get() + }) + }) + + after(() => { + dateNowStub.restore() + return agent.close({ ritmReset: false }) + }) + + it('should serialize basic schema correctly', async () => { + const loaded = await loadMessage(avro, 'User') + const type = avro.parse(loaded.schema) + const filePath = loaded.path + + tracer.trace('user.serialize', span => { + const buf = type.toBuffer({ name: 'Alyssa', favorite_number: 256, favorite_color: null }) + fs.writeFileSync(filePath, buf) + + expect(span._name).to.equal('user.serialize') + + expect(compareJson(BASIC_USER_SCHEMA_DEF, span)).to.equal(true) + expect(span.context()._tags).to.have.property(SCHEMA_TYPE, 'avro') + expect(span.context()._tags).to.have.property(SCHEMA_NAME, 'example.avro.User') + expect(span.context()._tags).to.have.property(SCHEMA_OPERATION, 'serialization') + expect(span.context()._tags).to.have.property(SCHEMA_ID, BASIC_USER_SCHEMA_ID) + expect(span.context()._tags).to.have.property(SCHEMA_WEIGHT, 1) + }) + }) + + it('should serialize the advanced schema correctly', async () => { + const loaded = await loadMessage(avro, 'AdvancedUser') + const type = avro.parse(loaded.schema) + const filePath = loaded.path + + tracer.trace('advanced_user.serialize', span => { + const buf = type.toBuffer({ + name: 'Alyssa', + age: 30, + email: 'alyssa@example.com', + height: 5.6, + preferences: { theme: 'dark', notifications: 'enabled' }, + tags: ['vip', 'premium'], + status: 'ACTIVE', + profile_picture: Buffer.from('binarydata'), + metadata: Buffer.from('metadata12345678'), + address: { street: '123 Main St', city: 'Metropolis', zipcode: '12345' } + }) + fs.writeFileSync(filePath, buf) + + expect(span._name).to.equal('advanced_user.serialize') + + expect(compareJson(ADVANCED_USER_SCHEMA_DEF, span)).to.equal(true) + expect(span.context()._tags).to.have.property(SCHEMA_TYPE, 'avro') + expect(span.context()._tags).to.have.property(SCHEMA_NAME, 'example.avro.AdvancedUser') + expect(span.context()._tags).to.have.property(SCHEMA_OPERATION, 'serialization') + expect(span.context()._tags).to.have.property(SCHEMA_ID, ADVANCED_USER_SCHEMA_ID) + expect(span.context()._tags).to.have.property(SCHEMA_WEIGHT, 1) + }) + }) + + it('should deserialize basic schema correctly', async () => { + const loaded = await loadMessage(avro, 'User') + const type = avro.parse(loaded.schema) + const filePath = loaded.path + const buf = type.toBuffer({ name: 'Alyssa', favorite_number: 256, favorite_color: null }) + fs.writeFileSync(filePath, buf) + + tracer.trace('user.deserialize', span => { + type.fromBuffer(buf) + + expect(span._name).to.equal('user.deserialize') + + expect(compareJson(BASIC_USER_SCHEMA_DEF, span)).to.equal(true) + expect(span.context()._tags).to.have.property(SCHEMA_TYPE, 'avro') + expect(span.context()._tags).to.have.property(SCHEMA_NAME, 'example.avro.User') + expect(span.context()._tags).to.have.property(SCHEMA_OPERATION, 'deserialization') + expect(span.context()._tags).to.have.property(SCHEMA_ID, BASIC_USER_SCHEMA_ID) + expect(span.context()._tags).to.have.property(SCHEMA_WEIGHT, 1) + }) + }) + + it('should deserialize advanced schema correctly', async () => { + const loaded = await loadMessage(avro, 'AdvancedUser') + const type = avro.parse(loaded.schema) + const filePath = loaded.path + const buf = type.toBuffer({ + name: 'Alyssa', + age: 30, + email: 'alyssa@example.com', + height: 5.6, + preferences: { theme: 'dark', notifications: 'enabled' }, + tags: ['vip', 'premium'], + status: 'ACTIVE', + profile_picture: Buffer.from('binarydata'), + metadata: Buffer.from('metadata12345678'), + address: { street: '123 Main St', city: 'Metropolis', zipcode: '12345' } + }) + fs.writeFileSync(filePath, buf) + + tracer.trace('advanced_user.deserialize', span => { + type.fromBuffer(buf) + + expect(span._name).to.equal('advanced_user.deserialize') + + expect(compareJson(ADVANCED_USER_SCHEMA_DEF, span)).to.equal(true) + expect(span.context()._tags).to.have.property(SCHEMA_TYPE, 'avro') + expect(span.context()._tags).to.have.property(SCHEMA_NAME, 'example.avro.AdvancedUser') + expect(span.context()._tags).to.have.property(SCHEMA_OPERATION, 'deserialization') + expect(span.context()._tags).to.have.property(SCHEMA_ID, ADVANCED_USER_SCHEMA_ID) + expect(span.context()._tags).to.have.property(SCHEMA_WEIGHT, 1) + }) + }) + }) + }) + }) +}) diff --git a/packages/datadog-plugin-avsc/test/schemas/advanced_user.avsc b/packages/datadog-plugin-avsc/test/schemas/advanced_user.avsc new file mode 100644 index 00000000000..c25081c495e --- /dev/null +++ b/packages/datadog-plugin-avsc/test/schemas/advanced_user.avsc @@ -0,0 +1,74 @@ +{ + "namespace": "example.avro", + "type": "record", + "name": "AdvancedUser", + "fields": [ + { + "name": "email", + "type": [ + "null", + "string" + ], + "default": null + }, + { + "name": "preferences", + "type": { + "type": "map", + "values": "string" + } + }, + { + "name": "tags", + "type": { + "type": "array", + "items": "string" + } + }, + { + "name": "status", + "type": { + "type": "enum", + "name": "Status", + "symbols": [ + "ACTIVE", + "INACTIVE", + "BANNED" + ] + } + }, + { + "name": "profile_picture", + "type": "bytes" + }, + { + "name": "metadata", + "type": { + "type": "fixed", + "name": "Metadata", + "size": 16 + } + }, + { + "name": "address", + "type": { + "type": "record", + "name": "Address", + "fields": [ + { + "name": "street", + "type": "string" + }, + { + "name": "city", + "type": "string" + }, + { + "name": "zipcode", + "type": "string" + } + ] + } + } + ] +} \ No newline at end of file diff --git a/packages/datadog-plugin-avsc/test/schemas/advanced_users.avro b/packages/datadog-plugin-avsc/test/schemas/advanced_users.avro new file mode 100644 index 00000000000..1e31871c28e Binary files /dev/null and b/packages/datadog-plugin-avsc/test/schemas/advanced_users.avro differ diff --git a/packages/datadog-plugin-avsc/test/schemas/expected_advanced_user_schema.json b/packages/datadog-plugin-avsc/test/schemas/expected_advanced_user_schema.json new file mode 100644 index 00000000000..932230d2959 --- /dev/null +++ b/packages/datadog-plugin-avsc/test/schemas/expected_advanced_user_schema.json @@ -0,0 +1,57 @@ +{ + "openapi": "3.0.0", + "components": { + "schemas": { + "example.avro.AdvancedUser": { + "type": "object", + "properties": { + "email": { + "type": "union[null,string]" + }, + "preferences": { + "type": "object" + }, + "tags": { + "type": "array", + "items": { + "type": "string" + } + }, + "status": { + "type": "string", + "enum": [ + "ACTIVE", + "INACTIVE", + "BANNED" + ] + }, + "profile_picture": { + "type": "string", + "format": "byte" + }, + "metadata": { + "type": "string" + }, + "address": { + "type": "object", + "$ref": "#/components/schemas/example.avro.Address" + } + } + }, + "example.avro.Address": { + "type": "object", + "properties": { + "street": { + "type": "string" + }, + "city": { + "type": "string" + }, + "zipcode": { + "type": "string" + } + } + } + } + } +} \ No newline at end of file diff --git a/packages/datadog-plugin-avsc/test/schemas/expected_user_schema.json b/packages/datadog-plugin-avsc/test/schemas/expected_user_schema.json new file mode 100644 index 00000000000..43eec7221f0 --- /dev/null +++ b/packages/datadog-plugin-avsc/test/schemas/expected_user_schema.json @@ -0,0 +1,21 @@ +{ + "openapi": "3.0.0", + "components": { + "schemas": { + "example.avro.User": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "favorite_number": { + "type": "union[integer,null]" + }, + "favorite_color": { + "type": "union[string,null]" + } + } + } + } + } +} \ No newline at end of file diff --git a/packages/datadog-plugin-avsc/test/schemas/user.avsc b/packages/datadog-plugin-avsc/test/schemas/user.avsc new file mode 100644 index 00000000000..1e810fea0c3 --- /dev/null +++ b/packages/datadog-plugin-avsc/test/schemas/user.avsc @@ -0,0 +1,25 @@ +{ + "namespace": "example.avro", + "type": "record", + "name": "User", + "fields": [ + { + "name": "name", + "type": "string" + }, + { + "name": "favorite_number", + "type": [ + "int", + "null" + ] + }, + { + "name": "favorite_color", + "type": [ + "string", + "null" + ] + } + ] +} \ No newline at end of file diff --git a/packages/datadog-plugin-avsc/test/schemas/users.avro b/packages/datadog-plugin-avsc/test/schemas/users.avro new file mode 100644 index 00000000000..5f8bfbe9325 Binary files /dev/null and b/packages/datadog-plugin-avsc/test/schemas/users.avro differ diff --git a/packages/datadog-plugin-aws-sdk/src/base.js b/packages/datadog-plugin-aws-sdk/src/base.js index cb6cf2b6126..e815c1e00aa 100644 --- a/packages/datadog-plugin-aws-sdk/src/base.js +++ b/packages/datadog-plugin-aws-sdk/src/base.js @@ -4,9 +4,12 @@ const analyticsSampler = require('../../dd-trace/src/analytics_sampler') const ClientPlugin = require('../../dd-trace/src/plugins/client') const { storage } = require('../../datadog-core') const { isTrue } = require('../../dd-trace/src/util') +const coalesce = require('koalas') +const { tagsFromRequest, tagsFromResponse } = require('../../dd-trace/src/payload-tagging') class BaseAwsSdkPlugin extends ClientPlugin { static get id () { return 'aws' } + static get isPayloadReporter () { return false } get serviceIdentifier () { const id = this.constructor.id.toLowerCase() @@ -19,6 +22,14 @@ class BaseAwsSdkPlugin extends ClientPlugin { return id } + get cloudTaggingConfig () { + return this._tracerConfig.cloudPayloadTagging + } + + get payloadTaggingRules () { + return this.cloudTaggingConfig.rules.aws?.[this.constructor.id] + } + constructor (...args) { super(...args) @@ -37,10 +48,10 @@ class BaseAwsSdkPlugin extends ClientPlugin { 'service.name': this.serviceName(), 'aws.operation': operation, 'aws.region': awsRegion, - 'region': awsRegion, - 'aws_service': awsService, + region: awsRegion, + aws_service: awsService, 'aws.service': awsService, - 'component': 'aws-sdk' + component: 'aws-sdk' } if (this.requestTags) this.requestTags.set(request, tags) @@ -50,6 +61,12 @@ class BaseAwsSdkPlugin extends ClientPlugin { this.requestInject(span, request) + if (this.constructor.isPayloadReporter && this.cloudTaggingConfig.requestsEnabled) { + const maxDepth = this.cloudTaggingConfig.maxDepth + const requestTags = tagsFromRequest(this.payloadTaggingRules, request.params, { maxDepth }) + span.addTags(requestTags) + } + const store = storage.getStore() this.enter(span, store) @@ -64,11 +81,17 @@ class BaseAwsSdkPlugin extends ClientPlugin { span.setTag('region', region) }) - this.addSub(`apm:aws:request:complete:${this.serviceIdentifier}`, ({ response }) => { + this.addSub(`apm:aws:request:complete:${this.serviceIdentifier}`, ({ response, cbExists = false }) => { const store = storage.getStore() if (!store) return const { span } = store if (!span) return + // try to extract DSM context from response if no callback exists as extraction normally happens in CB + if (!cbExists && this.serviceIdentifier === 'sqs') { + const params = response.request.params + const operation = response.request.operation + this.responseExtractDSMContext(operation, params, response.data ?? response, span) + } this.addResponseTags(span, response) this.finish(span, response, response.error) }) @@ -109,6 +132,7 @@ class BaseAwsSdkPlugin extends ClientPlugin { const params = response.request.params const operation = response.request.operation const extraTags = this.generateTags(params, operation, response) || {} + const tags = Object.assign({ 'aws.response.request_id': response.requestId, 'resource.name': operation, @@ -116,6 +140,22 @@ class BaseAwsSdkPlugin extends ClientPlugin { }, extraTags) span.addTags(tags) + + if (this.constructor.isPayloadReporter && this.cloudTaggingConfig.responsesEnabled) { + const maxDepth = this.cloudTaggingConfig.maxDepth + const responseBody = this.extractResponseBody(response) + const responseTags = tagsFromResponse(this.payloadTaggingRules, responseBody, { maxDepth }) + span.addTags(responseTags) + } + } + + extractResponseBody (response) { + if (response.hasOwnProperty('data')) { + return response.data + } + return Object.fromEntries( + Object.entries(response).filter(([key]) => !['request', 'requestId', 'error', '$metadata'].includes(key)) + ) } generateTags () { @@ -126,8 +166,9 @@ class BaseAwsSdkPlugin extends ClientPlugin { if (err) { span.setTag('error', err) - if (err.requestId) { - span.addTags({ 'aws.response.request_id': err.requestId }) + const requestId = err.RequestId || err.requestId + if (requestId) { + span.addTags({ 'aws.response.request_id': requestId }) } } @@ -156,8 +197,22 @@ function normalizeConfig (config, serviceIdentifier) { break } + // check if AWS batch propagation or AWS_[SERVICE] batch propagation is enabled via env variable + const serviceId = serviceIdentifier.toUpperCase() + const batchPropagationEnabled = isTrue( + coalesce( + specificConfig.batchPropagationEnabled, + process.env[`DD_TRACE_AWS_SDK_${serviceId}_BATCH_PROPAGATION_ENABLED`], + config.batchPropagationEnabled, + process.env.DD_TRACE_AWS_SDK_BATCH_PROPAGATION_ENABLED, + false + ) + ) + + // Merge the specific config back into the main config return Object.assign({}, config, specificConfig, { splitByAwsService: config.splitByAwsService !== false, + batchPropagationEnabled, hooks }) } diff --git a/packages/datadog-plugin-aws-sdk/src/services/cloudwatchlogs.js b/packages/datadog-plugin-aws-sdk/src/services/cloudwatchlogs.js index f28a5a48185..8ad4c8f1b26 100644 --- a/packages/datadog-plugin-aws-sdk/src/services/cloudwatchlogs.js +++ b/packages/datadog-plugin-aws-sdk/src/services/cloudwatchlogs.js @@ -13,7 +13,7 @@ class CloudwatchLogs extends BaseAwsSdkPlugin { return Object.assign(tags, { 'resource.name': `${operation} ${params.logGroupName}`, 'aws.cloudwatch.logs.log_group_name': params.logGroupName, - 'loggroupname': params.logGroupName + loggroupname: params.logGroupName }) } } diff --git a/packages/datadog-plugin-aws-sdk/src/services/dynamodb.js b/packages/datadog-plugin-aws-sdk/src/services/dynamodb.js index 18412efc22b..4097586b2c5 100644 --- a/packages/datadog-plugin-aws-sdk/src/services/dynamodb.js +++ b/packages/datadog-plugin-aws-sdk/src/services/dynamodb.js @@ -14,14 +14,14 @@ class DynamoDb extends BaseAwsSdkPlugin { Object.assign(tags, { 'resource.name': `${operation} ${params.TableName}`, 'aws.dynamodb.table_name': params.TableName, - 'tablename': params.TableName + tablename: params.TableName }) } // batch operations have different format, collect table name for batch // https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/DynamoDB.html#batchGetItem-property` // dynamoDB batch TableName - if (params.RequestItems) { + if (params.RequestItems !== null) { if (typeof params.RequestItems === 'object') { if (Object.keys(params.RequestItems).length === 1) { const tableName = Object.keys(params.RequestItems)[0] @@ -30,7 +30,7 @@ class DynamoDb extends BaseAwsSdkPlugin { Object.assign(tags, { 'resource.name': `${operation} ${tableName}`, 'aws.dynamodb.table_name': tableName, - 'tablename': tableName + tablename: tableName }) } } diff --git a/packages/datadog-plugin-aws-sdk/src/services/eventbridge.js b/packages/datadog-plugin-aws-sdk/src/services/eventbridge.js index a38063562ed..9309411564a 100644 --- a/packages/datadog-plugin-aws-sdk/src/services/eventbridge.js +++ b/packages/datadog-plugin-aws-sdk/src/services/eventbridge.js @@ -11,7 +11,7 @@ class EventBridge extends BaseAwsSdkPlugin { return { 'resource.name': operation ? `${operation} ${params.source}` : params.source, 'aws.eventbridge.source': `${params.source}`, - 'rulename': `${rulename}` + rulename: `${rulename}` } } diff --git a/packages/datadog-plugin-aws-sdk/src/services/index.js b/packages/datadog-plugin-aws-sdk/src/services/index.js index fcfea44932e..48b6510d8d3 100644 --- a/packages/datadog-plugin-aws-sdk/src/services/index.js +++ b/packages/datadog-plugin-aws-sdk/src/services/index.js @@ -7,6 +7,9 @@ exports.kinesis = require('./kinesis') exports.lambda = require('./lambda') exports.redshift = require('./redshift') exports.s3 = require('./s3') +exports.sfn = require('./sfn') exports.sns = require('./sns') exports.sqs = require('./sqs') +exports.states = require('./states') +exports.stepfunctions = require('./stepfunctions') exports.default = require('./default') diff --git a/packages/datadog-plugin-aws-sdk/src/services/kinesis.js b/packages/datadog-plugin-aws-sdk/src/services/kinesis.js index c0e2a9d739c..60802bfc448 100644 --- a/packages/datadog-plugin-aws-sdk/src/services/kinesis.js +++ b/packages/datadog-plugin-aws-sdk/src/services/kinesis.js @@ -1,20 +1,128 @@ 'use strict' +const { + getSizeOrZero +} = require('../../../dd-trace/src/datastreams/processor') +const { DsmPathwayCodec } = require('../../../dd-trace/src/datastreams/pathway') const log = require('../../../dd-trace/src/log') const BaseAwsSdkPlugin = require('../base') +const { storage } = require('../../../datadog-core') + class Kinesis extends BaseAwsSdkPlugin { static get id () { return 'kinesis' } static get peerServicePrecursors () { return ['streamname'] } + constructor (...args) { + super(...args) + + // TODO(bengl) Find a way to create the response span tags without this WeakMap being populated + // in the base class + this.requestTags = new WeakMap() + + this.addSub('apm:aws:response:start:kinesis', obj => { + const { request, response } = obj + const store = storage.getStore() + const plugin = this + + // if we have either of these operations, we want to store the streamName param + // since it is not typically available during get/put records requests + if (request.operation === 'getShardIterator' || request.operation === 'listShards') { + this.storeStreamName(request.params, request.operation, store) + return + } + + if (request.operation === 'getRecords') { + let span + const responseExtraction = this.responseExtract(request.params, request.operation, response) + if (responseExtraction && responseExtraction.maybeChildOf) { + obj.needsFinish = true + const options = { + childOf: responseExtraction.maybeChildOf, + tags: Object.assign( + {}, + this.requestTags.get(request) || {}, + { 'span.kind': 'server' } + ) + } + span = plugin.tracer.startSpan('aws.response', options) + this.enter(span, store) + } + + // get the stream name that should have been stored previously + const { streamName } = storage.getStore() + + // extract DSM context after as we might not have a parent-child but may have a DSM context + this.responseExtractDSMContext( + request.operation, request.params, response, span || null, { streamName } + ) + } + }) + + this.addSub('apm:aws:response:finish:kinesis', err => { + const { span } = storage.getStore() + this.finish(span, null, err) + }) + } + generateTags (params, operation, response) { if (!params || !params.StreamName) return {} return { 'resource.name': `${operation} ${params.StreamName}`, 'aws.kinesis.stream_name': params.StreamName, - 'streamname': params.StreamName + streamname: params.StreamName } } + storeStreamName (params, operation, store) { + if (!operation || (operation !== 'getShardIterator' && operation !== 'listShards')) return + if (!params || !params.StreamName) return + + const streamName = params.StreamName + storage.enterWith({ ...store, streamName }) + } + + responseExtract (params, operation, response) { + if (operation !== 'getRecords') return + if (params.Limit && params.Limit !== 1) return + if (!response || !response.Records || !response.Records[0]) return + + const record = response.Records[0] + + try { + const decodedData = JSON.parse(Buffer.from(record.Data).toString()) + + return { + maybeChildOf: this.tracer.extract('text_map', decodedData._datadog), + parsedAttributes: decodedData._datadog + } + } catch (e) { + log.error(e) + } + } + + responseExtractDSMContext (operation, params, response, span, kwargs = {}) { + const { streamName } = kwargs + if (!this.config.dsmEnabled) return + if (operation !== 'getRecords') return + if (!response || !response.Records || !response.Records[0]) return + + // we only want to set the payloadSize on the span if we have one message, not repeatedly + span = response.Records.length > 1 ? null : span + + response.Records.forEach(record => { + const parsedAttributes = JSON.parse(Buffer.from(record.Data).toString()) + + if ( + parsedAttributes?._datadog && streamName + ) { + const payloadSize = getSizeOrZero(record.Data) + this.tracer.decodeDataStreamsContext(parsedAttributes._datadog) + this.tracer + .setCheckpoint(['direction:in', `topic:${streamName}`, 'type:kinesis'], span, payloadSize) + } + }) + } + // AWS-SDK will b64 kinesis payloads // or will accept an already b64 encoded payload // This method handles both @@ -32,40 +140,74 @@ class Kinesis extends BaseAwsSdkPlugin { } requestInject (span, request) { - const operation = request.operation - if (operation === 'putRecord' || operation === 'putRecords') { - if (!request.params) { + const { operation, params } = request + if (!params) return + + let stream + switch (operation) { + case 'putRecord': + stream = params.StreamArn ? params.StreamArn : (params.StreamName ? params.StreamName : '') + this.injectToMessage(span, params, stream, true) + break + case 'putRecords': + stream = params.StreamArn ? params.StreamArn : (params.StreamName ? params.StreamName : '') + for (let i = 0; i < params.Records.length; i++) { + this.injectToMessage( + span, + params.Records[i], + stream, + i === 0 || (this.config.batchPropagationEnabled) + ) + } + } + } + + injectToMessage (span, params, stream, injectTraceContext) { + if (!params) { + return + } + + let parsedData + if (injectTraceContext || this.config.dsmEnabled) { + parsedData = this._tryParse(params.Data) + if (!parsedData) { + log.error('Unable to parse payload, unable to pass trace context or set DSM checkpoint (if enabled)') return } + } + + const ddInfo = {} + // for now, we only want to inject to the first message, this may change for batches in the future + if (injectTraceContext) { this.tracer.inject(span, 'text_map', ddInfo) } - const traceData = {} - this.tracer.inject(span, 'text_map', traceData) - let injectPath - if (request.params.Records && request.params.Records.length > 0) { - injectPath = request.params.Records[0] - } else if (request.params.Data) { - injectPath = request.params - } else { - log.error('No valid payload passed, unable to pass trace context') + // set DSM hash if enabled + if (this.config.dsmEnabled) { + parsedData._datadog = ddInfo + const dataStreamsContext = this.setDSMCheckpoint(span, parsedData, stream) + DsmPathwayCodec.encode(dataStreamsContext, ddInfo) + } + + if (Object.keys(ddInfo).length !== 0) { + parsedData._datadog = ddInfo + const finalData = Buffer.from(JSON.stringify(parsedData)) + const byteSize = finalData.length + // Kinesis max payload size is 1MB + // So we must ensure adding DD context won't go over that (512b is an estimate) + if (byteSize >= 1048576) { + log.info('Payload size too large to pass context') return } - const parsedData = this._tryParse(injectPath.Data) - if (parsedData) { - parsedData._datadog = traceData - const finalData = Buffer.from(JSON.stringify(parsedData)) - const byteSize = finalData.length - // Kinesis max payload size is 1MB - // So we must ensure adding DD context won't go over that (512b is an estimate) - if (byteSize >= 1048576) { - log.info('Payload size too large to pass context') - return - } - injectPath.Data = finalData - } else { - log.error('Unable to parse payload, unable to pass trace context') - } + params.Data = finalData } } + + setDSMCheckpoint (span, parsedData, stream) { + // get payload size of request data + const payloadSize = Buffer.from(JSON.stringify(parsedData)).byteLength + const dataStreamsContext = this.tracer + .setCheckpoint(['direction:out', `topic:${stream}`, 'type:kinesis'], span, payloadSize) + return dataStreamsContext + } } module.exports = Kinesis diff --git a/packages/datadog-plugin-aws-sdk/src/services/lambda.js b/packages/datadog-plugin-aws-sdk/src/services/lambda.js index 7b6d1e323bc..f6ea874872e 100644 --- a/packages/datadog-plugin-aws-sdk/src/services/lambda.js +++ b/packages/datadog-plugin-aws-sdk/src/services/lambda.js @@ -13,7 +13,7 @@ class Lambda extends BaseAwsSdkPlugin { return Object.assign(tags, { 'resource.name': `${operation} ${params.FunctionName}`, - 'functionname': params.FunctionName, + functionname: params.FunctionName, 'aws.lambda': params.FunctionName }) } diff --git a/packages/datadog-plugin-aws-sdk/src/services/redshift.js b/packages/datadog-plugin-aws-sdk/src/services/redshift.js index 194e38a3381..2d28a400376 100644 --- a/packages/datadog-plugin-aws-sdk/src/services/redshift.js +++ b/packages/datadog-plugin-aws-sdk/src/services/redshift.js @@ -13,7 +13,7 @@ class Redshift extends BaseAwsSdkPlugin { return Object.assign(tags, { 'resource.name': `${operation} ${params.ClusterIdentifier}`, 'aws.redshift.cluster_identifier': params.ClusterIdentifier, - 'clusteridentifier': params.ClusterIdentifier + clusteridentifier: params.ClusterIdentifier }) } } diff --git a/packages/datadog-plugin-aws-sdk/src/services/s3.js b/packages/datadog-plugin-aws-sdk/src/services/s3.js index 3313e8296a4..c306c7ba0a8 100644 --- a/packages/datadog-plugin-aws-sdk/src/services/s3.js +++ b/packages/datadog-plugin-aws-sdk/src/services/s3.js @@ -14,7 +14,7 @@ class S3 extends BaseAwsSdkPlugin { return Object.assign(tags, { 'resource.name': `${operation} ${params.Bucket}`, 'aws.s3.bucket_name': params.Bucket, - 'bucketname': params.Bucket + bucketname: params.Bucket }) } } diff --git a/packages/datadog-plugin-aws-sdk/src/services/sfn.js b/packages/datadog-plugin-aws-sdk/src/services/sfn.js new file mode 100644 index 00000000000..afdc8e5b7d8 --- /dev/null +++ b/packages/datadog-plugin-aws-sdk/src/services/sfn.js @@ -0,0 +1,7 @@ +'use strict' +const Stepfunctions = require('./stepfunctions') +class Sfn extends Stepfunctions { + static get id () { return 'sfn' } +} + +module.exports = Sfn diff --git a/packages/datadog-plugin-aws-sdk/src/services/sns.js b/packages/datadog-plugin-aws-sdk/src/services/sns.js index 934b59a5d5d..4e2b16f1d18 100644 --- a/packages/datadog-plugin-aws-sdk/src/services/sns.js +++ b/packages/datadog-plugin-aws-sdk/src/services/sns.js @@ -1,26 +1,31 @@ 'use strict' +const { getHeadersSize } = require('../../../dd-trace/src/datastreams/processor') +const { DsmPathwayCodec } = require('../../../dd-trace/src/datastreams/pathway') const log = require('../../../dd-trace/src/log') const BaseAwsSdkPlugin = require('../base') class Sns extends BaseAwsSdkPlugin { static get id () { return 'sns' } static get peerServicePrecursors () { return ['topicname'] } + static get isPayloadReporter () { return true } generateTags (params, operation, response) { if (!params) return {} if (!params.TopicArn && !(response.data && response.data.TopicArn)) return {} const TopicArn = params.TopicArn || response.data.TopicArn + // Split the ARN into its parts // ex.'arn:aws:sns:us-east-1:123456789012:my-topic' const arnParts = TopicArn.split(':') // Get the topic name from the last part of the ARN const topicName = arnParts[arnParts.length - 1] + return { 'resource.name': `${operation} ${params.TopicArn || response.data.TopicArn}`, 'aws.sns.topic_arn': TopicArn, - 'topicname': topicName + topicname: topicName } // TODO: should arn be sanitized or quantized in some way here, @@ -52,17 +57,22 @@ class Sns extends BaseAwsSdkPlugin { switch (operation) { case 'publish': - this._injectMessageAttributes(span, params) + this.injectToMessage(span, params, params.TopicArn, true) break case 'publishBatch': - if (params.PublishBatchRequestEntries && params.PublishBatchRequestEntries.length > 0) { - this._injectMessageAttributes(span, params.PublishBatchRequestEntries[0]) + for (let i = 0; i < params.PublishBatchRequestEntries.length; i++) { + this.injectToMessage( + span, + params.PublishBatchRequestEntries[i], + params.TopicArn, + i === 0 || (this.config.batchPropagationEnabled) + ) } break } } - _injectMessageAttributes (span, params) { + injectToMessage (span, params, topicArn, injectTraceContext) { if (!params.MessageAttributes) { params.MessageAttributes = {} } @@ -70,11 +80,46 @@ class Sns extends BaseAwsSdkPlugin { log.info('Message attributes full, skipping trace context injection') return } + const ddInfo = {} - this.tracer.inject(span, 'text_map', ddInfo) - params.MessageAttributes._datadog = { - DataType: 'Binary', - BinaryValue: Buffer.from(JSON.stringify(ddInfo)) // BINARY types are automatically base64 encoded + // for now, we only want to inject to the first message, this may change for batches in the future + if (injectTraceContext) { + this.tracer.inject(span, 'text_map', ddInfo) + // add ddInfo before checking DSM so we can include DD attributes in payload size + params.MessageAttributes._datadog = { + DataType: 'Binary', + BinaryValue: ddInfo + } + } + + if (this.config.dsmEnabled) { + if (!params.MessageAttributes._datadog) { + params.MessageAttributes._datadog = { + DataType: 'Binary', + BinaryValue: ddInfo + } + } + + const dataStreamsContext = this.setDSMCheckpoint(span, params, topicArn) + DsmPathwayCodec.encode(dataStreamsContext, ddInfo) + } + + if (Object.keys(ddInfo).length !== 0) { + // BINARY types are automatically base64 encoded + params.MessageAttributes._datadog.BinaryValue = Buffer.from(JSON.stringify(ddInfo)) + } else if (params.MessageAttributes._datadog) { + // let's avoid adding any additional information to payload if we failed to inject + delete params.MessageAttributes._datadog + } + } + + setDSMCheckpoint (span, params, topicArn) { + // only set a checkpoint if publishing to a topic + if (topicArn) { + const payloadSize = getHeadersSize(params) + const dataStreamsContext = this.tracer + .setCheckpoint(['direction:out', `topic:${topicArn}`, 'type:sns'], span, payloadSize) + return dataStreamsContext } } } diff --git a/packages/datadog-plugin-aws-sdk/src/services/sqs.js b/packages/datadog-plugin-aws-sdk/src/services/sqs.js index 2fde8bf5214..54a3e7e756c 100644 --- a/packages/datadog-plugin-aws-sdk/src/services/sqs.js +++ b/packages/datadog-plugin-aws-sdk/src/services/sqs.js @@ -3,6 +3,8 @@ const log = require('../../../dd-trace/src/log') const BaseAwsSdkPlugin = require('../base') const { storage } = require('../../../datadog-core') +const { getHeadersSize } = require('../../../dd-trace/src/datastreams/processor') +const { DsmPathwayCodec } = require('../../../dd-trace/src/datastreams/pathway') class Sqs extends BaseAwsSdkPlugin { static get id () { return 'sqs' } @@ -19,20 +21,28 @@ class Sqs extends BaseAwsSdkPlugin { const { request, response } = obj const store = storage.getStore() const plugin = this - const maybeChildOf = this.responseExtract(request.params, request.operation, response) - if (maybeChildOf) { + const contextExtraction = this.responseExtract(request.params, request.operation, response) + let span + let parsedMessageAttributes = null + if (contextExtraction && contextExtraction.datadogContext) { obj.needsFinish = true const options = { - childOf: maybeChildOf, + childOf: contextExtraction.datadogContext, tags: Object.assign( {}, this.requestTags.get(request) || {}, { 'span.kind': 'server' } ) } - const span = plugin.tracer.startSpan('aws.response', options) + parsedMessageAttributes = contextExtraction.parsedAttributes + span = plugin.tracer.startSpan('aws.response', options) this.enter(span, store) } + // extract DSM context after as we might not have a parent-child but may have a DSM context + + this.responseExtractDSMContext( + request.operation, request.params, response, span || null, { parsedMessageAttributes } + ) }) this.addSub('apm:aws:response:finish:sqs', err => { @@ -92,7 +102,7 @@ class Sqs extends BaseAwsSdkPlugin { Object.assign(tags, { 'resource.name': `${operation} ${params.QueueName || params.QueueUrl}`, 'aws.sqs.queue_name': params.QueueName || params.QueueUrl, - 'queuename': queueName + queuename: queueName }) switch (operation) { @@ -133,38 +143,153 @@ class Sqs extends BaseAwsSdkPlugin { const datadogAttribute = message.MessageAttributes._datadog + const parsedAttributes = this.parseDatadogAttributes(datadogAttribute) + if (parsedAttributes) { + return { + datadogContext: this.tracer.extract('text_map', parsedAttributes), + parsedAttributes + } + } + } + + parseDatadogAttributes (attributes) { try { - if (datadogAttribute.StringValue) { - const textMap = datadogAttribute.StringValue - return this.tracer.extract('text_map', JSON.parse(textMap)) - } else if (datadogAttribute.Type === 'Binary') { - const buffer = Buffer.from(datadogAttribute.Value, 'base64') - return this.tracer.extract('text_map', JSON.parse(buffer)) + if (attributes.StringValue) { + const textMap = attributes.StringValue + return JSON.parse(textMap) + } else if (attributes.Type === 'Binary' || attributes.DataType === 'Binary') { + const buffer = Buffer.from(attributes.Value ?? attributes.BinaryValue, 'base64') + return JSON.parse(buffer) } } catch (e) { log.error(e) } } - requestInject (span, request) { - const operation = request.operation - if (operation === 'sendMessage') { - if (!request.params) { - request.params = {} + responseExtractDSMContext (operation, params, response, span, kwargs = {}) { + let { parsedAttributes } = kwargs + if (!this.config.dsmEnabled) return + if (operation !== 'receiveMessage') return + if (!response || !response.Messages || !response.Messages[0]) return + + // we only want to set the payloadSize on the span if we have one message + span = response.Messages.length > 1 ? null : span + + response.Messages.forEach(message => { + // we may have already parsed the message attributes when extracting trace context + if (!parsedAttributes) { + if (message.Body) { + try { + const body = JSON.parse(message.Body) + + // SNS to SQS + if (body.Type === 'Notification') { + message = body + } + } catch (e) { + // SQS to SQS + } + } + if (!parsedAttributes && message.MessageAttributes && message.MessageAttributes._datadog) { + parsedAttributes = this.parseDatadogAttributes(message.MessageAttributes._datadog) + } } - if (!request.params.MessageAttributes) { - request.params.MessageAttributes = {} - } else if (Object.keys(request.params.MessageAttributes).length >= 10) { // SQS quota - // TODO: add test when the test suite is fixed - return + if (parsedAttributes) { + const payloadSize = getHeadersSize({ + Body: message.Body, + MessageAttributes: message.MessageAttributes + }) + const queue = params.QueueUrl.split('/').pop() + this.tracer.decodeDataStreamsContext(parsedAttributes) + this.tracer + .setCheckpoint(['direction:in', `topic:${queue}`, 'type:sqs'], span, payloadSize) } - const ddInfo = {} + }) + } + + requestInject (span, request) { + const { operation, params } = request + + if (!params) return + + switch (operation) { + case 'sendMessage': + this.injectToMessage(span, params, params.QueueUrl, true) + break + case 'sendMessageBatch': + for (let i = 0; i < params.Entries.length; i++) { + this.injectToMessage( + span, + params.Entries[i], + params.QueueUrl, + i === 0 || (this.config.batchPropagationEnabled) + ) + } + break + case 'receiveMessage': + if (!params.MessageAttributeNames) { + params.MessageAttributeNames = ['_datadog'] + } else if ( + !params.MessageAttributeNames.includes('_datadog') && + !params.MessageAttributeNames.includes('.*') && + !params.MessageAttributeNames.includes('All') + ) { + params.MessageAttributeNames.push('_datadog') + } + break + } + } + + injectToMessage (span, params, queueUrl, injectTraceContext) { + if (!params) { + params = {} + } + if (!params.MessageAttributes) { + params.MessageAttributes = {} + } else if (Object.keys(params.MessageAttributes).length >= 10) { // SQS quota + // TODO: add test when the test suite is fixed + return + } + const ddInfo = {} + // for now, we only want to inject to the first message, this may change for batches in the future + if (injectTraceContext) { this.tracer.inject(span, 'text_map', ddInfo) - request.params.MessageAttributes._datadog = { + params.MessageAttributes._datadog = { DataType: 'String', StringValue: JSON.stringify(ddInfo) } } + + if (this.config.dsmEnabled) { + if (!params.MessageAttributes._datadog) { + params.MessageAttributes._datadog = { + DataType: 'String', + StringValue: JSON.stringify(ddInfo) + } + } + + const dataStreamsContext = this.setDSMCheckpoint(span, params, queueUrl) + if (dataStreamsContext) { + DsmPathwayCodec.encode(dataStreamsContext, ddInfo) + params.MessageAttributes._datadog.StringValue = JSON.stringify(ddInfo) + } + } + + if (params.MessageAttributes._datadog && Object.keys(ddInfo).length === 0) { + // let's avoid adding any additional information to payload if we failed to inject + delete params.MessageAttributes._datadog + } + } + + setDSMCheckpoint (span, params, queueUrl) { + const payloadSize = getHeadersSize({ + Body: params.MessageBody, + MessageAttributes: params.MessageAttributes + }) + const queue = queueUrl.split('/').pop() + const dataStreamsContext = this.tracer + .setCheckpoint(['direction:out', `topic:${queue}`, 'type:sqs'], span, payloadSize) + return dataStreamsContext } } diff --git a/packages/datadog-plugin-aws-sdk/src/services/states.js b/packages/datadog-plugin-aws-sdk/src/services/states.js new file mode 100644 index 00000000000..4c12c865622 --- /dev/null +++ b/packages/datadog-plugin-aws-sdk/src/services/states.js @@ -0,0 +1,7 @@ +'use strict' +const Stepfunctions = require('./stepfunctions') +class States extends Stepfunctions { + static get id () { return 'states' } +} + +module.exports = States diff --git a/packages/datadog-plugin-aws-sdk/src/services/stepfunctions.js b/packages/datadog-plugin-aws-sdk/src/services/stepfunctions.js new file mode 100644 index 00000000000..0712d7e9f1a --- /dev/null +++ b/packages/datadog-plugin-aws-sdk/src/services/stepfunctions.js @@ -0,0 +1,64 @@ +'use strict' +const log = require('../../../dd-trace/src/log') +const BaseAwsSdkPlugin = require('../base') + +class Stepfunctions extends BaseAwsSdkPlugin { + static get id () { return 'stepfunctions' } + + // This is the shape of StartExecutionInput, as defined in + // https://github.com/aws/aws-sdk-js/blob/master/apis/states-2016-11-23.normal.json + // "StartExecutionInput": { + // "type": "structure", + // "required": [ + // "stateMachineArn" + // ], + // "members": { + // "stateMachineArn": { + // "shape": "Arn", + // }, + // "name": { + // "shape": "Name", + // }, + // "input": { + // "shape": "SensitiveData", + // }, + // "traceHeader": { + // "shape": "TraceHeader", + // } + // } + + generateTags (params, operation, response) { + if (!params) return {} + const tags = { 'resource.name': params.name ? `${operation} ${params.name}` : `${operation}` } + if (operation === 'startExecution' || operation === 'startSyncExecution') { + tags.statemachinearn = `${params.stateMachineArn}` + } + return tags + } + + requestInject (span, request) { + const operation = request.operation + if (operation === 'startExecution' || operation === 'startSyncExecution') { + if (!request.params || !request.params.input) { + return + } + + const input = request.params.input + + try { + const inputObj = JSON.parse(input) + if (inputObj !== null && typeof inputObj === 'object') { + // We've parsed the input JSON string + inputObj._datadog = {} + this.tracer.inject(span, 'text_map', inputObj._datadog) + const newInput = JSON.stringify(inputObj) + request.params.input = newInput + } + } catch (e) { + log.info('Unable to treat input as JSON') + } + } + } +} + +module.exports = Stepfunctions diff --git a/packages/datadog-plugin-aws-sdk/test/aws-sdk.spec.js b/packages/datadog-plugin-aws-sdk/test/aws-sdk.spec.js index aeb5d5b81fd..4f68f5fbf94 100644 --- a/packages/datadog-plugin-aws-sdk/test/aws-sdk.spec.js +++ b/packages/datadog-plugin-aws-sdk/test/aws-sdk.spec.js @@ -35,11 +35,11 @@ describe('Plugin', () => { }) expect(span.meta).to.include({ - 'component': 'aws-sdk', + component: 'aws-sdk', 'aws.region': 'us-east-1', - 'region': 'us-east-1', + region: 'us-east-1', 'aws.service': 'S3', - 'aws_service': 'S3', + aws_service: 'S3', 'aws.operation': 'listBuckets' }) }).then(done, done) @@ -102,11 +102,11 @@ describe('Plugin', () => { }) expect(span.meta).to.include({ - 'component': 'aws-sdk', + component: 'aws-sdk', 'aws.region': 'us-east-1', - 'region': 'us-east-1', + region: 'us-east-1', 'aws.service': 'S3', - 'aws_service': 'S3', + aws_service: 'S3', 'aws.operation': 'listBuckets' }) }).then(done, done) @@ -114,6 +114,28 @@ describe('Plugin', () => { s3.listBuckets({}, e => e && done(e)) }) + // different versions of aws-sdk use different casings and different AWS headers + it('should include tracing headers and not cause a 403 error', (done) => { + const HttpClientPlugin = require('../../datadog-plugin-http/src/client.js') + const spy = sinon.spy(HttpClientPlugin.prototype, 'bindStart') + agent.use(traces => { + const headers = new Set( + Object.keys(spy.firstCall.firstArg.args.options.headers) + .map(x => x.toLowerCase()) + ) + spy.restore() + + expect(headers).to.include('authorization') + expect(headers).to.include('x-amz-date') + expect(headers).to.include('x-datadog-trace-id') + expect(headers).to.include('x-datadog-parent-id') + expect(headers).to.include('x-datadog-sampling-priority') + expect(headers).to.include('x-datadog-tags') + }).then(done, done) + + s3.listBuckets({}, e => e && done(e)) + }) + it('should mark error responses', (done) => { let error @@ -122,7 +144,7 @@ describe('Plugin', () => { expect(span).to.include({ name: 'aws.request', - resource: 'completeMultipartUpload', + resource: 'completeMultipartUpload my-bucket', service: 'test-aws-s3' }) @@ -130,11 +152,18 @@ describe('Plugin', () => { [ERROR_TYPE]: error.name, [ERROR_MESSAGE]: error.message, [ERROR_STACK]: error.stack, - 'component': 'aws-sdk' + component: 'aws-sdk' }) + if (semver.intersects(version, '>=2.3.4')) { + expect(span.meta['aws.response.request_id']).to.match(/[\w]{8}(-[\w]{4}){3}-[\w]{12}/) + } }).then(done, done) - s3.completeMultipartUpload('invalid', e => { + s3.completeMultipartUpload({ + Bucket: 'my-bucket', + Key: 'my-key', + UploadId: 'my-upload-id' + }, e => { error = e }) }) @@ -210,7 +239,7 @@ describe('Plugin', () => { request (span, response) { span.setTag('hook.operation', response.request.operation) span.addTags({ - 'error': 0 + error: 0 }) } } @@ -238,7 +267,7 @@ describe('Plugin', () => { expect(span).to.have.property('error', 0) expect(span.meta).to.include({ 'hook.operation': 'listBuckets', - 'component': 'aws-sdk' + component: 'aws-sdk' }) }).then(done, done) @@ -307,6 +336,65 @@ describe('Plugin', () => { }, 250) }) }) + + describe('with programmatic batchPropagationEnabled configuration', () => { + before(() => { + return agent.load(['aws-sdk'], [{ + service: 'test', + batchPropagationEnabled: true, + kinesis: { + batchPropagationEnabled: false + }, + sns: false, + sqs: { + batchPropagationEnabled: false + } + }]) + }) + + before(() => { + tracer = require('../../dd-trace') + }) + + after(() => { + return agent.close({ ritmReset: false }) + }) + + it('should be configurable on a per-service basis', () => { + const { kinesis, sns, sqs } = tracer._pluginManager._pluginsByName['aws-sdk'].services + + expect(kinesis.config.batchPropagationEnabled).to.equal(false) + expect(sns.config.batchPropagationEnabled).to.equal(true) + expect(sns.config.enabled).to.equal(false) + expect(sqs.config.batchPropagationEnabled).to.equal(false) + }) + }) + + describe('with env variable _BATCH_PROPAGATION_ENABLED configuration', () => { + before(() => { + process.env.DD_TRACE_AWS_SDK_BATCH_PROPAGATION_ENABLED = true + process.env.DD_TRACE_AWS_SDK_KINESIS_BATCH_PROPAGATION_ENABLED = false + process.env.DD_TRACE_AWS_SDK_SQS_BATCH_PROPAGATION_ENABLED = true + + return agent.load(['aws-sdk']) + }) + + before(() => { + tracer = require('../../dd-trace') + }) + + after(() => { + return agent.close({ ritmReset: false }) + }) + + it('should be configurable on a per-service basis', () => { + const { kinesis, sns, sqs } = tracer._pluginManager._pluginsByName['aws-sdk'].services + + expect(kinesis.config.batchPropagationEnabled).to.equal(false) + expect(sns.config.batchPropagationEnabled).to.equal(true) + expect(sqs.config.batchPropagationEnabled).to.equal(true) + }) + }) }) }) }) diff --git a/packages/datadog-plugin-aws-sdk/test/eventbridge.spec.js b/packages/datadog-plugin-aws-sdk/test/eventbridge.spec.js index 98572b3f833..fbe77151d4c 100644 --- a/packages/datadog-plugin-aws-sdk/test/eventbridge.spec.js +++ b/packages/datadog-plugin-aws-sdk/test/eventbridge.spec.js @@ -55,7 +55,7 @@ describe('EventBridge', () => { expect(eventbridge.generateTags(params, 'putEvent', {})).to.deep.equal({ 'aws.eventbridge.source': 'my.event', 'resource.name': 'putEvent my.event', - 'rulename': 'my-rule-name' + rulename: 'my-rule-name' }) }) it('won\'t create tags for a malformed event', () => { @@ -88,7 +88,7 @@ describe('EventBridge', () => { parentId = '0000000000000000' eventbridge.requestInject(span.context(), request) - expect(request.params).to.deep.equal({ 'Entries': [{ 'Detail': '{"custom":"data","for":"my users","from":"Aaron Stuyvenberg","_datadog":{"x-datadog-trace-id":"456853219676779160","x-datadog-parent-id":"456853219676779160","x-datadog-sampling-priority":"1"}}' }] }) + expect(request.params).to.deep.equal({ Entries: [{ Detail: '{"custom":"data","for":"my users","from":"Aaron Stuyvenberg","_datadog":{"x-datadog-trace-id":"456853219676779160","x-datadog-parent-id":"456853219676779160","x-datadog-sampling-priority":"1"}}' }] }) }) it('skips injecting trace context to Eventbridge if message is full', () => { @@ -133,7 +133,7 @@ describe('EventBridge', () => { expect(eventbridge.generateTags(params, 'putEvent', {})).to.deep.equal({ 'aws.eventbridge.source': 'my.event', 'resource.name': 'putEvent my.event', - 'rulename': '' + rulename: '' }) }) @@ -146,7 +146,7 @@ describe('EventBridge', () => { expect(eventbridge.generateTags(params, null, {})).to.deep.equal({ 'aws.eventbridge.source': 'my.event', 'resource.name': 'my.event', - 'rulename': 'my-rule-name' + rulename: 'my-rule-name' }) }) it('handles null response gracefully', () => { @@ -158,7 +158,7 @@ describe('EventBridge', () => { expect(eventbridge.generateTags(params, 'putEvent', null)).to.deep.equal({ 'aws.eventbridge.source': 'my.event', 'resource.name': 'putEvent my.event', - 'rulename': 'my-rule-name' + rulename: 'my-rule-name' }) }) }) diff --git a/packages/datadog-plugin-aws-sdk/test/fixtures/cloudwatchlogs.js b/packages/datadog-plugin-aws-sdk/test/fixtures/cloudwatchlogs.js index c15270a6470..12e8e1b73a7 100644 --- a/packages/datadog-plugin-aws-sdk/test/fixtures/cloudwatchlogs.js +++ b/packages/datadog-plugin-aws-sdk/test/fixtures/cloudwatchlogs.js @@ -2,7 +2,7 @@ const cloudwatchlogs = {} -cloudwatchlogs['create'] = { +cloudwatchlogs.create = { logGroupName: 'example_cw_log_group' } diff --git a/packages/datadog-plugin-aws-sdk/test/fixtures/dynamodb.js b/packages/datadog-plugin-aws-sdk/test/fixtures/dynamodb.js index ea40d9630a0..50ec28ef6a4 100644 --- a/packages/datadog-plugin-aws-sdk/test/fixtures/dynamodb.js +++ b/packages/datadog-plugin-aws-sdk/test/fixtures/dynamodb.js @@ -2,7 +2,7 @@ const ddb = {} -ddb['create'] = { +ddb.create = { AttributeDefinitions: [ { AttributeName: 'CUSTOMER_ID', @@ -30,29 +30,29 @@ ddb['create'] = { TableName: 'CUSTOMER_LIST' } -ddb['put'] = { +ddb.put = { TableName: 'CUSTOMER_LIST', Item: { - 'CUSTOMER_ID': { N: '001' }, - 'CUSTOMER_NAME': { S: 'Richard Roe' } + CUSTOMER_ID: { N: '001' }, + CUSTOMER_NAME: { S: 'Richard Roe' } } } -ddb['get'] = { +ddb.get = { TableName: 'CUSTOMER_LIST', Key: { - 'CUSTOMER_ID': { N: '001' }, - 'CUSTOMER_NAME': { S: 'Richard Roe' } + CUSTOMER_ID: { N: '001' }, + CUSTOMER_NAME: { S: 'Richard Roe' } } } -ddb['batch'] = { +ddb.batch = { RequestItems: { - 'CUSTOMER_LIST': { + CUSTOMER_LIST: { Keys: [ { - 'CUSTOMER_ID': { N: '001' }, - 'CUSTOMER_NAME': { S: 'Richard Roe' } + CUSTOMER_ID: { N: '001' }, + CUSTOMER_NAME: { S: 'Richard Roe' } } ], ConsistentRead: true diff --git a/packages/datadog-plugin-aws-sdk/test/fixtures/kinesis.js b/packages/datadog-plugin-aws-sdk/test/fixtures/kinesis.js index 5860153a59f..aa993752903 100644 --- a/packages/datadog-plugin-aws-sdk/test/fixtures/kinesis.js +++ b/packages/datadog-plugin-aws-sdk/test/fixtures/kinesis.js @@ -2,7 +2,7 @@ const kinesis = {} -kinesis['describe'] = { +kinesis.describe = { StreamName: 'test_aws_stream' } diff --git a/packages/datadog-plugin-aws-sdk/test/fixtures/redshift.js b/packages/datadog-plugin-aws-sdk/test/fixtures/redshift.js index e736afa5b60..33a37d3488b 100644 --- a/packages/datadog-plugin-aws-sdk/test/fixtures/redshift.js +++ b/packages/datadog-plugin-aws-sdk/test/fixtures/redshift.js @@ -2,14 +2,14 @@ const redshift = {} -redshift['create'] = { +redshift.create = { ClusterIdentifier: 'example_redshift_cluster', MasterUserPassword: 'example_user_password', MasterUsername: 'example_username', NodeType: 'ds2.large' } -redshift['get'] = { +redshift.get = { ClusterIdentifier: 'example_redshift_cluster' } diff --git a/packages/datadog-plugin-aws-sdk/test/fixtures/s3.js b/packages/datadog-plugin-aws-sdk/test/fixtures/s3.js index 026de1204d9..3314ca5209a 100644 --- a/packages/datadog-plugin-aws-sdk/test/fixtures/s3.js +++ b/packages/datadog-plugin-aws-sdk/test/fixtures/s3.js @@ -2,13 +2,13 @@ const s3 = {} -s3['put'] = { +s3.put = { Bucket: 'test-aws-bucket-9bd88aa3-6fc1-44bd-ae3a-ba25f49c3eef', Key: 'test.txt', Body: 'Hello World!' } -s3['create'] = { +s3.create = { Bucket: 'test-aws-bucket-9bd88aa3-6fc1-44bd-ae3a-ba25f49c3eef' } diff --git a/packages/datadog-plugin-aws-sdk/test/fixtures/sns.js b/packages/datadog-plugin-aws-sdk/test/fixtures/sns.js index 9928346b095..8bff110115e 100644 --- a/packages/datadog-plugin-aws-sdk/test/fixtures/sns.js +++ b/packages/datadog-plugin-aws-sdk/test/fixtures/sns.js @@ -2,11 +2,11 @@ const sns = {} -sns['create'] = { +sns.create = { Name: 'example_aws_topic' } -sns['get'] = { +sns.get = { TopicArn: undefined } diff --git a/packages/datadog-plugin-aws-sdk/test/fixtures/sqs.js b/packages/datadog-plugin-aws-sdk/test/fixtures/sqs.js index 0f0030a4735..8680d57d7fa 100644 --- a/packages/datadog-plugin-aws-sdk/test/fixtures/sqs.js +++ b/packages/datadog-plugin-aws-sdk/test/fixtures/sqs.js @@ -2,14 +2,14 @@ const sqs = {} -sqs['create'] = { +sqs.create = { QueueName: 'SQS_QUEUE_NAME', Attributes: { - 'MessageRetentionPeriod': '86400' + MessageRetentionPeriod: '86400' } } -sqs['get'] = { +sqs.get = { QueueUrl: undefined } diff --git a/packages/datadog-plugin-aws-sdk/test/integration-test/client.spec.js b/packages/datadog-plugin-aws-sdk/test/integration-test/client.spec.js index db495f20234..e077c0b64b2 100644 --- a/packages/datadog-plugin-aws-sdk/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-aws-sdk/test/integration-test/client.spec.js @@ -15,9 +15,9 @@ describe('esm', () => { withVersions('aws-sdk', ['aws-sdk'], version => { before(async function () { - this.timeout(20000) + this.timeout(60000) sandbox = await createSandbox([`'aws-sdk@${version}'`], false, [ - `./packages/datadog-plugin-aws-sdk/test/integration-test/*`]) + './packages/datadog-plugin-aws-sdk/test/integration-test/*']) }) after(async () => { @@ -42,8 +42,8 @@ describe('esm', () => { proc = await spawnPluginIntegrationTestProc(sandbox.folder, 'server.mjs', agent.port, undefined, { - 'AWS_SECRET_ACCESS_KEY': '0000000000/00000000000000000000000000000', - 'AWS_ACCESS_KEY_ID': '00000000000000000000' + AWS_SECRET_ACCESS_KEY: '0000000000/00000000000000000000000000000', + AWS_ACCESS_KEY_ID: '00000000000000000000' } ) diff --git a/packages/datadog-plugin-aws-sdk/test/kinesis.spec.js b/packages/datadog-plugin-aws-sdk/test/kinesis.spec.js index 41d76d61236..cedeb14f000 100644 --- a/packages/datadog-plugin-aws-sdk/test/kinesis.spec.js +++ b/packages/datadog-plugin-aws-sdk/test/kinesis.spec.js @@ -1,25 +1,26 @@ /* eslint-disable max-len */ 'use strict' +const sinon = require('sinon') const agent = require('../../dd-trace/test/plugins/agent') const { setup } = require('./spec_helpers') const helpers = require('./kinesis_helpers') const { rawExpectedSchema } = require('./kinesis-naming') -describe('Kinesis', () => { +describe('Kinesis', function () { + this.timeout(10000) setup() withVersions('aws-sdk', ['aws-sdk', '@aws-sdk/smithy-client'], (version, moduleName) => { let AWS let kinesis + let tracer + const streamName = 'MyStream' + const streamNameDSM = 'MyStreamDSM' const kinesisClientName = moduleName === '@aws-sdk/smithy-client' ? '@aws-sdk/client-kinesis' : 'aws-sdk' - before(() => { - return agent.load('aws-sdk') - }) - - before(done => { + function createResources (streamName, cb) { AWS = require(`../../../versions/${kinesisClientName}@${version}`).get() const params = { @@ -34,116 +35,302 @@ describe('Kinesis', () => { } kinesis = new AWS.Kinesis(params) + kinesis.createStream({ - StreamName: 'MyStream', + StreamName: streamName, ShardCount: 1 }, (err, res) => { - if (err) return done(err) + if (err) return cb(err) - helpers.waitForActiveStream(kinesis, done) + helpers.waitForActiveStream(kinesis, streamName, cb) }) + } + + before(() => { + process.env.DD_DATA_STREAMS_ENABLED = 'true' }) - after(done => { - kinesis.deleteStream({ - StreamName: 'MyStream' - }, (err, res) => { - if (err) return done(err) + describe('no configuration', () => { + before(() => { + return agent.load('aws-sdk', { kinesis: { dsmEnabled: false, batchPropagationEnabled: true } }, { dsmEnabled: true }) + }) - helpers.waitForDeletedStream(kinesis, done) + before(done => { + createResources(streamName, done) }) - }) - withNamingSchema( - (done) => kinesis.describeStream({ - StreamName: 'MyStream' - }, (err) => err && done(err)), - rawExpectedSchema.outbound - ) + after(done => { + kinesis.deleteStream({ + StreamName: streamName + }, (err, res) => { + if (err) return done(err) + + helpers.waitForDeletedStream(kinesis, streamName, done) + }) + }) - it('injects trace context to Kinesis putRecord', done => { - helpers.putTestRecord(kinesis, helpers.dataBuffer, (err, data) => { - if (err) return done(err) + withNamingSchema( + (done) => kinesis.describeStream({ + StreamName: streamName + }, (err) => err && done(err)), + rawExpectedSchema.outbound + ) - helpers.getTestData(kinesis, data, (err, data) => { + it('injects trace context to Kinesis putRecord', done => { + helpers.putTestRecord(kinesis, streamName, helpers.dataBuffer, (err, data) => { if (err) return done(err) - expect(data).to.have.property('_datadog') - expect(data._datadog).to.have.property('x-datadog-trace-id') + helpers.getTestData(kinesis, streamName, data, (err, data) => { + if (err) return done(err) + + expect(data).to.have.property('_datadog') + expect(data._datadog).to.have.property('x-datadog-trace-id') - done() + done() + }) }) }) - }) - - it('handles already b64 encoded data', done => { - helpers.putTestRecord(kinesis, helpers.dataBuffer.toString('base64'), (err, data) => { - if (err) return done(err) - helpers.getTestData(kinesis, data, (err, data) => { + it('injects trace context to each message during Kinesis putRecord and batchPropagationEnabled', done => { + helpers.putTestRecords(kinesis, streamName, (err, data) => { if (err) return done(err) - expect(data).to.have.property('_datadog') - expect(data._datadog).to.have.property('x-datadog-trace-id') + helpers.getTestRecord(kinesis, streamName, data.Records[0], (err, data) => { + if (err) return done(err) + + for (const record in data.Records) { + const recordData = JSON.parse(Buffer.from(data.Records[record].Data).toString()) + expect(recordData).to.have.property('_datadog') + expect(recordData._datadog).to.have.property('x-datadog-trace-id') + } - done() + done() + }) }) }) - }) - it('skips injecting trace context to Kinesis if message is full', done => { - const dataBuffer = Buffer.from(JSON.stringify({ - myData: Array(1048576 - 100).join('a') - })) + it('handles already b64 encoded data', done => { + helpers.putTestRecord(kinesis, streamName, helpers.dataBuffer.toString('base64'), (err, data) => { + if (err) return done(err) + + helpers.getTestData(kinesis, streamName, data, (err, data) => { + if (err) return done(err) + + expect(data).to.have.property('_datadog') + expect(data._datadog).to.have.property('x-datadog-trace-id') + + done() + }) + }) + }) - helpers.putTestRecord(kinesis, dataBuffer, (err, data) => { - if (err) return done(err) + it('skips injecting trace context to Kinesis if message is full', done => { + const dataBuffer = Buffer.from(JSON.stringify({ + myData: Array(1048576 - 100).join('a') + })) - helpers.getTestData(kinesis, data, (err, data) => { + helpers.putTestRecord(kinesis, streamName, dataBuffer, (err, data) => { if (err) return done(err) - expect(data).to.not.have.property('_datadog') + helpers.getTestData(kinesis, streamName, data, (err, data) => { + if (err) return done(err) + + expect(data).to.not.have.property('_datadog') - done() + done() + }) }) }) - }) - it('generates tags for proper input', done => { - agent.use(traces => { - const span = traces[0][0] - expect(span.meta).to.include({ - 'streamname': 'MyStream', - 'aws_service': 'Kinesis', - 'region': 'us-east-1' + it('generates tags for proper input', done => { + agent.use(traces => { + const span = traces[0][0] + expect(span.meta).to.include({ + streamname: streamName, + aws_service: 'Kinesis', + region: 'us-east-1' + }) + expect(span.resource).to.equal(`putRecord ${streamName}`) + expect(span.meta).to.have.property('streamname', streamName) + }).then(done, done) + + helpers.putTestRecord(kinesis, streamName, helpers.dataBuffer, e => e && done(e)) + }) + + describe('Disabled', () => { + before(() => { + process.env.DD_TRACE_AWS_SDK_KINESIS_ENABLED = 'false' + }) + + after(() => { + delete process.env.DD_TRACE_AWS_SDK_KINESIS_ENABLED }) - expect(span.resource).to.equal('putRecord MyStream') - expect(span.meta).to.have.property('streamname', 'MyStream') - }).then(done, done) - helpers.putTestRecord(kinesis, helpers.dataBuffer, e => e && done(e)) + it('skip injects trace context to Kinesis putRecord when disabled', done => { + helpers.putTestRecord(kinesis, streamName, helpers.dataBuffer, (err, data) => { + if (err) return done(err) + + helpers.getTestData(kinesis, streamName, data, (err, data) => { + if (err) return done(err) + + expect(data).not.to.have.property('_datadog') + + done() + }) + }) + }) + }) }) - describe('Disabled', () => { + describe('DSM Context Propagation', () => { + const expectedProducerHash = '15481393933680799703' + const expectedConsumerHash = '10538746554122257118' + let nowStub + before(() => { - process.env.DD_TRACE_AWS_SDK_KINESIS_ENABLED = 'false' + return agent.load('aws-sdk', { kinesis: { dsmEnabled: true } }, { dsmEnabled: true }) + }) + + before(done => { + tracer = require('../../dd-trace') + tracer.use('aws-sdk', { kinesis: { dsmEnabled: true } }, { dsmEnabled: true }) + + createResources(streamNameDSM, done) + }) + + after(done => { + kinesis.deleteStream({ + StreamName: streamNameDSM + }, (err, res) => { + if (err) return done(err) + + helpers.waitForDeletedStream(kinesis, streamNameDSM, done) + }) }) - after(() => { - delete process.env.DD_TRACE_AWS_SDK_KINESIS_ENABLED + afterEach(() => { + try { + nowStub.restore() + } catch { + // pass + } + agent.reload('aws-sdk', { kinesis: { dsmEnabled: true } }, { dsmEnabled: true }) }) - it('skip injects trace context to Kinesis putRecord when disabled', done => { - helpers.putTestRecord(kinesis, helpers.dataBuffer, (err, data) => { + it('injects DSM pathway hash during Kinesis getRecord to the span', done => { + let getRecordSpanMeta = {} + agent.use(traces => { + const span = traces[0][0] + + if (span.name === 'aws.response') { + getRecordSpanMeta = span.meta + } + + expect(getRecordSpanMeta).to.include({ + 'pathway.hash': expectedConsumerHash + }) + }, { timeoutMs: 10000 }).then(done, done) + + helpers.putTestRecord(kinesis, streamNameDSM, helpers.dataBuffer, (err, data) => { if (err) return done(err) - helpers.getTestData(kinesis, data, (err, data) => { + helpers.getTestData(kinesis, streamNameDSM, data, (err) => { if (err) return done(err) + }) + }) + }) - expect(data).not.to.have.property('_datadog') + it('injects DSM pathway hash during Kinesis putRecord to the span', done => { + let putRecordSpanMeta = {} + agent.use(traces => { + const span = traces[0][0] - done() + if (span.resource.startsWith('putRecord')) { + putRecordSpanMeta = span.meta + } + + expect(putRecordSpanMeta).to.include({ + 'pathway.hash': expectedProducerHash }) + }).then(done, done) + + helpers.putTestRecord(kinesis, streamNameDSM, helpers.dataBuffer, (err, data) => { + if (err) return done(err) + }) + }) + + it('emits DSM stats to the agent during Kinesis putRecord', done => { + agent.expectPipelineStats(dsmStats => { + let statsPointsReceived = 0 + // we should have only have 1 stats point since we only had 1 put operation + dsmStats.forEach((timeStatsBucket) => { + if (timeStatsBucket && timeStatsBucket.Stats) { + timeStatsBucket.Stats.forEach((statsBuckets) => { + statsPointsReceived += statsBuckets.Stats.length + }) + } + }) + expect(statsPointsReceived).to.be.at.least(1) + expect(agent.dsmStatsExist(agent, expectedProducerHash)).to.equal(true) + }).then(done, done) + + helpers.putTestRecord(kinesis, streamNameDSM, helpers.dataBuffer, (err, data) => { + if (err) return done(err) + }) + }) + + it('emits DSM stats to the agent during Kinesis getRecord', done => { + agent.expectPipelineStats(dsmStats => { + let statsPointsReceived = 0 + // we should have only have 1 stats point since we only had 1 put operation + dsmStats.forEach((timeStatsBucket) => { + if (timeStatsBucket && timeStatsBucket.Stats) { + timeStatsBucket.Stats.forEach((statsBuckets) => { + statsPointsReceived += statsBuckets.Stats.length + }) + } + }, { timeoutMs: 10000 }) + expect(statsPointsReceived).to.be.at.least(2) + expect(agent.dsmStatsExist(agent, expectedConsumerHash)).to.equal(true) + }, { timeoutMs: 10000 }).then(done, done) + + helpers.putTestRecord(kinesis, streamNameDSM, helpers.dataBuffer, (err, data) => { + if (err) return done(err) + + helpers.getTestData(kinesis, streamNameDSM, data, (err) => { + if (err) return done(err) + }) + }) + }) + + it('emits DSM stats to the agent during Kinesis putRecords', done => { + // we need to stub Date.now() to ensure a new stats bucket is created for each call + // otherwise, all stats checkpoints will be combined into a single stats points + let now = Date.now() + nowStub = sinon.stub(Date, 'now') + nowStub.callsFake(() => { + now += 1000000 + return now + }) + + agent.expectPipelineStats(dsmStats => { + let statsPointsReceived = 0 + // we should have only have 3 stats points since we only had 3 records published + dsmStats.forEach((timeStatsBucket) => { + if (timeStatsBucket && timeStatsBucket.Stats) { + timeStatsBucket.Stats.forEach((statsBuckets) => { + statsPointsReceived += statsBuckets.Stats.length + }) + } + }) + expect(statsPointsReceived).to.be.at.least(3) + expect(agent.dsmStatsExist(agent, expectedProducerHash)).to.equal(true) + }, { timeoutMs: 10000 }).then(done, done) + + helpers.putTestRecords(kinesis, streamNameDSM, (err, data) => { + if (err) return done(err) + + nowStub.restore() }) }) }) diff --git a/packages/datadog-plugin-aws-sdk/test/kinesis_helpers.js b/packages/datadog-plugin-aws-sdk/test/kinesis_helpers.js index f9f61ada0bf..72784572618 100644 --- a/packages/datadog-plugin-aws-sdk/test/kinesis_helpers.js +++ b/packages/datadog-plugin-aws-sdk/test/kinesis_helpers.js @@ -8,8 +8,17 @@ const dataBuffer = Buffer.from(JSON.stringify({ from: 'Aaron Stuyvenberg' })) -function getTestData (kinesis, input, cb) { - getTestRecord(kinesis, input, (err, data) => { +const dataBufferCustom = (n) => { + return Buffer.from(JSON.stringify({ + number: n, + custom: 'data', + for: 'my users', + from: 'Aaron Stuyvenberg' + })) +} + +function getTestData (kinesis, streamName, input, cb) { + getTestRecord(kinesis, streamName, input, (err, data) => { if (err) return cb(err) const dataBuffer = Buffer.from(data.Records[0].Data).toString() @@ -22,12 +31,12 @@ function getTestData (kinesis, input, cb) { }) } -function getTestRecord (kinesis, { ShardId, SequenceNumber }, cb) { +function getTestRecord (kinesis, streamName, { ShardId, SequenceNumber }, cb) { kinesis.getShardIterator({ ShardId, ShardIteratorType: 'AT_SEQUENCE_NUMBER', StartingSequenceNumber: SequenceNumber, - StreamName: 'MyStream' + StreamName: streamName }, (err, { ShardIterator } = {}) => { if (err) return cb(err) @@ -37,32 +46,54 @@ function getTestRecord (kinesis, { ShardId, SequenceNumber }, cb) { }) } -function putTestRecord (kinesis, data, cb) { +function putTestRecord (kinesis, streamName, data, cb) { kinesis.putRecord({ PartitionKey: id().toString(), Data: data, - StreamName: 'MyStream' + StreamName: streamName + }, cb) +} + +function putTestRecords (kinesis, streamName, cb) { + kinesis.putRecords({ + Records: [ + { + PartitionKey: id().toString(), + Data: dataBufferCustom(1) + }, + { + PartitionKey: id().toString(), + Data: dataBufferCustom(2) + }, + { + PartitionKey: id().toString(), + Data: dataBufferCustom(3) + } + ], + StreamName: streamName }, cb) } -function waitForActiveStream (kinesis, cb) { +function waitForActiveStream (kinesis, streamName, cb) { kinesis.describeStream({ - StreamName: 'MyStream' + StreamName: streamName }, (err, data) => { - if (err) return waitForActiveStream(kinesis, cb) + if (err) { + return waitForActiveStream(kinesis, streamName, cb) + } if (data.StreamDescription.StreamStatus !== 'ACTIVE') { - return waitForActiveStream(kinesis, cb) + return waitForActiveStream(kinesis, streamName, cb) } cb() }) } -function waitForDeletedStream (kinesis, cb) { +function waitForDeletedStream (kinesis, streamName, cb) { kinesis.describeStream({ - StreamName: 'MyStream' + StreamName: streamName }, (err, data) => { - if (!err) return waitForDeletedStream(kinesis, cb) + if (!err) return waitForDeletedStream(kinesis, streamName, cb) cb() }) } @@ -72,6 +103,7 @@ module.exports = { getTestData, getTestRecord, putTestRecord, + putTestRecords, waitForActiveStream, waitForDeletedStream } diff --git a/packages/datadog-plugin-aws-sdk/test/lambda.spec.js b/packages/datadog-plugin-aws-sdk/test/lambda.spec.js index 2bfafed17e7..5529a84677d 100644 --- a/packages/datadog-plugin-aws-sdk/test/lambda.spec.js +++ b/packages/datadog-plugin-aws-sdk/test/lambda.spec.js @@ -40,13 +40,13 @@ describe('Plugin', () => { before(done => { AWS = require(`../../../versions/${lambdaClientName}@${version}`).get() - lambda = new AWS.Lambda({ endpoint: 'http://127.0.0.1:4566', region: 'us-east-1' }) + lambda = new AWS.Lambda({ endpoint: 'http://127.0.0.1:4567', region: 'us-east-1' }) lambda.createFunction({ FunctionName: 'ironmaiden', Code: { ZipFile }, Handler: 'handler.handle', Role: 'arn:aws:iam::123456:role/test', - Runtime: 'nodejs16.x' + Runtime: 'nodejs18.x' }, (err, res) => { if (err) return done(err) @@ -92,9 +92,9 @@ describe('Plugin', () => { expect(span.resource.startsWith('invoke')).to.equal(true) expect(span.meta).to.include({ - 'functionname': 'ironmaiden', - 'aws_service': 'Lambda', - 'region': 'us-east-1' + functionname: 'ironmaiden', + aws_service: 'Lambda', + region: 'us-east-1' }) const parentId = span.span_id.toString() const traceId = span.trace_id.toString() diff --git a/packages/datadog-plugin-aws-sdk/test/s3.spec.js b/packages/datadog-plugin-aws-sdk/test/s3.spec.js index 21165ce7b3f..9ffb9a67215 100644 --- a/packages/datadog-plugin-aws-sdk/test/s3.spec.js +++ b/packages/datadog-plugin-aws-sdk/test/s3.spec.js @@ -37,7 +37,7 @@ describe('Plugin', () => { before(done => { AWS = require(`../../../versions/${s3ClientName}@${version}`).get() - s3 = new AWS.S3({ endpoint: 'http://127.0.0.1:4566', s3ForcePathStyle: true, region: 'us-east-1' }) + s3 = new AWS.S3({ endpoint: 'http://127.0.0.1:4567', s3ForcePathStyle: true, region: 'us-east-1' }) s3.createBucket({ Bucket: bucketName }, (err) => { if (err) return done(err) done() @@ -85,9 +85,9 @@ describe('Plugin', () => { }) expect(span.meta).to.include({ - 'bucketname': bucketName, - 'aws_service': 'S3', - 'region': 'us-east-1' + bucketname: bucketName, + aws_service: 'S3', + region: 'us-east-1' }) total++ diff --git a/packages/datadog-plugin-aws-sdk/test/sns.spec.js b/packages/datadog-plugin-aws-sdk/test/sns.spec.js index 966e5ea4bbc..7b62156f06c 100644 --- a/packages/datadog-plugin-aws-sdk/test/sns.spec.js +++ b/packages/datadog-plugin-aws-sdk/test/sns.spec.js @@ -1,12 +1,13 @@ /* eslint-disable max-len */ 'use strict' +const sinon = require('sinon') const semver = require('semver') const agent = require('../../dd-trace/test/plugins/agent') const { setup } = require('./spec_helpers') const { rawExpectedSchema } = require('./sns-naming') -describe('Sns', () => { +describe('Sns', function () { setup() withVersions('aws-sdk', ['aws-sdk', '@aws-sdk/smithy-client'], (version, moduleName) => { @@ -24,7 +25,8 @@ describe('Sns', () => { const snsClientName = moduleName === '@aws-sdk/smithy-client' ? '@aws-sdk/client-sns' : 'aws-sdk' const sqsClientName = moduleName === '@aws-sdk/smithy-client' ? '@aws-sdk/client-sqs' : 'aws-sdk' - const assertPropagation = done => { + let childSpansFound = 0 + const assertPropagation = (done, childSpans = 1) => { agent.use(traces => { const span = traces[0][0] @@ -36,39 +38,31 @@ describe('Sns', () => { expect(parentId).to.not.equal('0') expect(parentId).to.equal(spanId) - }).then(done, done) + childSpansFound += 1 + expect(childSpansFound).to.equal(childSpans) + childSpansFound = 0 + }, { timeoutMs: 10000 }).then(done, done) } - beforeEach(() => { - tracer = require('../../dd-trace') - }) - - before(() => { - parentId = '0' - spanId = '0' - - return agent.load('aws-sdk') - }) - - before(done => { + function createResources (queueName, topicName, cb) { const { SNS } = require(`../../../versions/${snsClientName}@${version}`).get() const { SQS } = require(`../../../versions/${sqsClientName}@${version}`).get() sns = new SNS({ endpoint: 'http://127.0.0.1:4566', region: 'us-east-1' }) sqs = new SQS({ endpoint: 'http://127.0.0.1:4566', region: 'us-east-1' }) - sns.createTopic({ Name: 'TestTopic' }, (err, data) => { - if (err) return done(err) + sns.createTopic({ Name: topicName }, (err, data) => { + if (err) return cb(err) TopicArn = data.TopicArn - sqs.createQueue({ QueueName: 'TestQueue' }, (err, data) => { - if (err) return done(err) + sqs.createQueue({ QueueName: queueName }, (err, data) => { + if (err) return cb(err) QueueUrl = data.QueueUrl sqs.getQueueAttributes({ QueueUrl, AttributeNames: ['All'] }, (err, data) => { - if (err) return done(err) + if (err) return cb(err) QueueArn = data.Attributes.QueueArn @@ -84,137 +78,615 @@ describe('Sns', () => { WaitTimeSeconds: 1 } - done() + cb() }) }) }) - }) + } - after(done => { - sns.deleteTopic({ TopicArn }, done) - }) + describe('with payload tagging', () => { + before(async () => { + await agent.load('aws-sdk') + await agent.close({ ritmReset: false, wipe: true }) + await agent.load('aws-sdk', {}, { + cloudPayloadTagging: { + request: '$.MessageAttributes.foo,$.MessageAttributes.redacted.StringValue.foo', + response: '$.MessageId,$.Attributes.DisplayName', + maxDepth: 5 + } + }) + }) - after(done => { - sqs.deleteQueue({ QueueUrl }, done) - }) + after(() => agent.close({ ritmReset: false, wipe: true })) + + before(done => { + createResources('TestQueue', 'TestTopic', done) + }) + + after(done => { + sns.deleteTopic({ TopicArn }, done) + }) + + after(done => { + sqs.deleteQueue({ QueueUrl }, done) + }) + + it('adds request and response payloads as flattened tags', done => { + agent.use(traces => { + const span = traces[0][0] + + expect(span.resource).to.equal(`publish ${TopicArn}`) + expect(span.meta).to.include({ + 'aws.sns.topic_arn': TopicArn, + topicname: 'TestTopic', + aws_service: 'SNS', + region: 'us-east-1', + 'aws.request.body.TopicArn': TopicArn, + 'aws.request.body.Message': 'message 1', + 'aws.request.body.MessageAttributes.baz.DataType': 'String', + 'aws.request.body.MessageAttributes.baz.StringValue': 'bar', + 'aws.request.body.MessageAttributes.keyOne.DataType': 'String', + 'aws.request.body.MessageAttributes.keyOne.StringValue': 'keyOne', + 'aws.request.body.MessageAttributes.keyTwo.DataType': 'String', + 'aws.request.body.MessageAttributes.keyTwo.StringValue': 'keyTwo', + 'aws.response.body.MessageId': 'redacted' + }) + }).then(done, done) + + sns.publish({ + TopicArn, + Message: 'message 1', + MessageAttributes: { + baz: { DataType: 'String', StringValue: 'bar' }, + keyOne: { DataType: 'String', StringValue: 'keyOne' }, + keyTwo: { DataType: 'String', StringValue: 'keyTwo' } + } + }, e => e && done(e)) + }) + + it('expands and redacts keys identified as expandable', done => { + agent.use(traces => { + const span = traces[0][0] + + expect(span.resource).to.equal(`publish ${TopicArn}`) + expect(span.meta).to.include({ + 'aws.sns.topic_arn': TopicArn, + topicname: 'TestTopic', + aws_service: 'SNS', + region: 'us-east-1', + 'aws.request.body.TopicArn': TopicArn, + 'aws.request.body.Message': 'message 1', + 'aws.request.body.MessageAttributes.redacted.StringValue.foo': 'redacted', + 'aws.request.body.MessageAttributes.unredacted.StringValue.foo': 'bar', + 'aws.request.body.MessageAttributes.unredacted.StringValue.baz': 'yup', + 'aws.response.body.MessageId': 'redacted' + }) + }).then(done, done) + + sns.publish({ + TopicArn, + Message: 'message 1', + MessageAttributes: { + unredacted: { DataType: 'String', StringValue: '{"foo": "bar", "baz": "yup"}' }, + redacted: { DataType: 'String', StringValue: '{"foo": "bar"}' } + } + }, e => e && done(e)) + }) + + describe('user-defined redaction', () => { + it('redacts user-defined keys to suppress in request', done => { + agent.use(traces => { + const span = traces[0][0] + + expect(span.resource).to.equal(`publish ${TopicArn}`) + expect(span.meta).to.include({ + 'aws.sns.topic_arn': TopicArn, + topicname: 'TestTopic', + aws_service: 'SNS', + region: 'us-east-1', + 'aws.request.body.TopicArn': TopicArn, + 'aws.request.body.Message': 'message 1', + 'aws.request.body.MessageAttributes.foo': 'redacted', + 'aws.request.body.MessageAttributes.keyOne.DataType': 'String', + 'aws.request.body.MessageAttributes.keyOne.StringValue': 'keyOne', + 'aws.request.body.MessageAttributes.keyTwo.DataType': 'String', + 'aws.request.body.MessageAttributes.keyTwo.StringValue': 'keyTwo' + }) + expect(span.meta).to.have.property('aws.response.body.MessageId') + }).then(done, done) + + sns.publish({ + TopicArn, + Message: 'message 1', + MessageAttributes: { + foo: { DataType: 'String', StringValue: 'bar' }, + keyOne: { DataType: 'String', StringValue: 'keyOne' }, + keyTwo: { DataType: 'String', StringValue: 'keyTwo' } + } + }, e => e && done(e)) + }) + + // TODO add response tests + it('redacts user-defined keys to suppress in response', done => { + agent.use(traces => { + const span = traces[0][0] + expect(span.resource).to.equal(`getTopicAttributes ${TopicArn}`) + expect(span.meta).to.include({ + 'aws.sns.topic_arn': TopicArn, + topicname: 'TestTopic', + aws_service: 'SNS', + region: 'us-east-1', + 'aws.request.body.TopicArn': TopicArn, + 'aws.response.body.Attributes.DisplayName': 'redacted' + }) + }).then(done, done) + + sns.getTopicAttributes({ TopicArn }, e => e && done(e)) + }) + }) + + describe('redaction of internally suppressed keys', () => { + const supportsSMSNotification = (moduleName, version) => { + switch (moduleName) { + case 'aws-sdk': + // aws-sdk-js phone notifications introduced in c6d1bb1a + return semver.intersects(version, '>=2.10.0') + case '@aws-sdk/smithy-client': + return true + default: + return false + } + } + + if (supportsSMSNotification(moduleName, version)) { + // TODO + describe.skip('phone number', () => { + before(done => { + sns.createSMSSandboxPhoneNumber({ PhoneNumber: '+33628606135' }, err => err && done(err)) + sns.createSMSSandboxPhoneNumber({ PhoneNumber: '+33628606136' }, err => err && done(err)) + }) + + after(done => { + sns.deleteSMSSandboxPhoneNumber({ PhoneNumber: '+33628606135' }, err => err && done(err)) + sns.deleteSMSSandboxPhoneNumber({ PhoneNumber: '+33628606136' }, err => err && done(err)) + }) + + it('redacts phone numbers in request', done => { + agent.use(traces => { + const span = traces[0][0] + + expect(span.resource).to.equal('publish') + expect(span.meta).to.include({ + aws_service: 'SNS', + region: 'us-east-1', + 'aws.request.body.PhoneNumber': 'redacted', + 'aws.request.body.Message': 'message 1' + }) + }).then(done, done) + + sns.publish({ + PhoneNumber: '+33628606135', + Message: 'message 1' + }, e => e && done(e)) + }) + + it('redacts phone numbers in response', done => { + agent.use(traces => { + const span = traces[0][0] + + expect(span.resource).to.equal('publish') + expect(span.meta).to.include({ + aws_service: 'SNS', + region: 'us-east-1', + 'aws.response.body.PhoneNumber': 'redacted' + }) + }).then(done, done) + + sns.listSMSSandboxPhoneNumbers({ + PhoneNumber: '+33628606135', + Message: 'message 1' + }, e => e && done(e)) + }) + }) + } - after(() => { - return agent.close({ ritmReset: false }) + describe('subscription confirmation tokens', () => { + it('redacts tokens in request', done => { + agent.use(traces => { + const span = traces[0][0] + + expect(span.resource).to.equal(`confirmSubscription ${TopicArn}`) + expect(span.meta).to.include({ + aws_service: 'SNS', + 'aws.sns.topic_arn': TopicArn, + topicname: 'TestTopic', + region: 'us-east-1', + 'aws.request.body.Token': 'redacted', + 'aws.request.body.TopicArn': TopicArn + }) + }).then(done, done) + + sns.confirmSubscription({ + TopicArn, + Token: '1234' + }, () => {}) + }) + + // TODO + it.skip('redacts tokens in response', () => { + + }) + }) + }) }) - withPeerService( - () => tracer, - 'aws-sdk', - (done) => sns.publish({ - TopicArn, - Message: 'message 1' - }, (err) => err && done()), - 'TestTopic', 'topicname') - - withNamingSchema( - (done) => sns.publish({ - TopicArn, - Message: 'message 1' - }, (err) => err && done()), - rawExpectedSchema.producer, - { - desc: 'producer' + describe('no configuration', () => { + before(() => { + parentId = '0' + spanId = '0' + + return agent.load('aws-sdk', { sns: { dsmEnabled: false, batchPropagationEnabled: true } }, { dsmEnabled: true }) + }) + + before(done => { + process.env.DD_DATA_STREAMS_ENABLED = 'true' + tracer = require('../../dd-trace') + tracer.use('aws-sdk', { sns: { dsmEnabled: false, batchPropagationEnabled: true } }) + + createResources('TestQueue', 'TestTopic', done) + }) + + after(done => { + sns.deleteTopic({ TopicArn }, done) + }) + + after(done => { + sqs.deleteQueue({ QueueUrl }, done) + }) + + after(() => { + return agent.close({ ritmReset: false }) + }) + + withPeerService( + () => tracer, + 'aws-sdk', + (done) => sns.publish({ + TopicArn, + Message: 'message 1' + }, (err) => err && done()), + 'TestTopic', 'topicname') + + withNamingSchema( + (done) => sns.publish({ + TopicArn, + Message: 'message 1' + }, (err) => err && done()), + rawExpectedSchema.producer, + { + desc: 'producer' + } + ) + + withNamingSchema( + (done) => sns.getTopicAttributes({ + TopicArn + }, (err) => err && done(err)), + rawExpectedSchema.client, + { + desc: 'client' + } + ) + + it('injects trace context to SNS publish', done => { + assertPropagation(done) + + sns.subscribe(subParams, (err, data) => { + if (err) return done(err) + + sqs.receiveMessage(receiveParams, e => e && done(e)) + sns.publish({ TopicArn, Message: 'message 1' }, (e) => { + if (e) done(e) + }) + }) + }) + + // There is a bug in 3.x (but not 3.0.0) that will be fixed in 3.261 + // https://github.com/aws/aws-sdk-js-v3/issues/2861 + if (!semver.intersects(version, '<3 || >3.0.0')) { + it('injects trace context to SNS publishBatch', done => { + assertPropagation(done) + + sns.subscribe(subParams, (err, data) => { + if (err) return done(err) + + sqs.receiveMessage(receiveParams, e => e && done(e)) + sns.publishBatch({ + TopicArn, + PublishBatchRequestEntries: [ + { Id: '1', Message: 'message 1' }, + { Id: '2', Message: 'message 2' } + ] + }, e => e && done(e)) + }) + }) + + it('injects trace context to each message SNS publishBatch with batch propagation enabled', done => { + assertPropagation(done, 3) + + sns.subscribe(subParams, (err, data) => { + if (err) return done(err) + + sqs.receiveMessage(receiveParams, (err, data) => { + if (err) done(err) + + for (const message in data.Messages) { + const recordData = JSON.parse(data.Messages[message].Body) + expect(recordData.MessageAttributes).to.have.property('_datadog') + + const attributes = JSON.parse(Buffer.from(recordData.MessageAttributes._datadog.Value, 'base64')) + expect(attributes).to.have.property('x-datadog-trace-id') + } + }) + sns.publishBatch({ + TopicArn, + PublishBatchRequestEntries: [ + { Id: '1', Message: 'message 1' }, + { Id: '2', Message: 'message 2' }, + { Id: '3', Message: 'message 3' } + ] + }, e => e && done(e)) + }) + }) } - ) - - withNamingSchema( - (done) => sns.getTopicAttributes({ - TopicArn - }, (err) => err && done(err)), - rawExpectedSchema.client, - { - desc: 'client' + + // TODO: Figure out why this fails only in 3.0.0 + if (version !== '3.0.0') { + it('skips injecting trace context to SNS if message attributes are full', done => { + sns.subscribe(subParams, (err, data) => { + if (err) return done(err) + + sqs.receiveMessage(receiveParams, (err, data) => { + if (err) return done(err) + + try { + expect(data.Messages[0].Body).to.not.include('datadog') + done() + } catch (e) { + done(e) + } + }) + + sns.publish({ + TopicArn, + Message: 'message 1', + MessageAttributes: { + keyOne: { DataType: 'String', StringValue: 'keyOne' }, + keyTwo: { DataType: 'String', StringValue: 'keyTwo' }, + keyThree: { DataType: 'String', StringValue: 'keyThree' }, + keyFour: { DataType: 'String', StringValue: 'keyFour' }, + keyFive: { DataType: 'String', StringValue: 'keyFive' }, + keySix: { DataType: 'String', StringValue: 'keySix' }, + keySeven: { DataType: 'String', StringValue: 'keySeven' }, + keyEight: { DataType: 'String', StringValue: 'keyEight' }, + keyNine: { DataType: 'String', StringValue: 'keyNine' }, + keyTen: { DataType: 'String', StringValue: 'keyTen' } + } + }, e => e && done(e)) + }) + }) } - ) - it('injects trace context to SNS publish', done => { - assertPropagation(done) + it('generates tags for proper publish calls', done => { + agent.use(traces => { + const span = traces[0][0] - sns.subscribe(subParams, (err, data) => { - if (err) return done(err) + expect(span.resource).to.equal(`publish ${TopicArn}`) + expect(span.meta).to.include({ + 'aws.sns.topic_arn': TopicArn, + topicname: 'TestTopic', + aws_service: 'SNS', + region: 'us-east-1' + }) + }).then(done, done) - sqs.receiveMessage(receiveParams, e => e && done(e)) sns.publish({ TopicArn, Message: 'message 1' }, e => e && done(e)) }) }) - // There is a bug in 3.x (but not 3.0.0) that will be fixed in 3.261 - // https://github.com/aws/aws-sdk-js-v3/issues/2861 - if (!semver.intersects(version, '<3 || >3.0.0')) { - it('injects trace context to SNS publishBatch', done => { - assertPropagation(done) + describe('Data Streams Monitoring', () => { + const expectedProducerHash = '5117773060236273241' + const expectedConsumerHash = '1353703578833511841' + let nowStub + + before(() => { + return agent.load('aws-sdk', { sns: { dsmEnabled: true }, sqs: { dsmEnabled: true } }, { dsmEnabled: true }) + }) + before(done => { + process.env.DD_DATA_STREAMS_ENABLED = 'true' + tracer = require('../../dd-trace') + tracer.use('aws-sdk', { sns: { dsmEnabled: true }, sqs: { dsmEnabled: true } }) + + createResources('TestQueueDSM', 'TestTopicDSM', done) + }) + + after(done => { + sns.deleteTopic({ TopicArn }, done) + }) + + after(done => { + sqs.deleteQueue({ QueueUrl }, done) + }) + + after(() => { + return agent.close({ ritmReset: false, wipe: true }) + }) + + afterEach(() => { + try { + nowStub.restore() + } catch { + // pass + } + agent.reload('aws-sdk', { sns: { dsmEnabled: true, batchPropagationEnabled: true } }, { dsmEnabled: true }) + }) + + it('injects DSM pathway hash to SNS publish span', done => { sns.subscribe(subParams, (err, data) => { if (err) return done(err) - sqs.receiveMessage(receiveParams, e => e && done(e)) - sns.publishBatch({ - TopicArn, - PublishBatchRequestEntries: [ - { Id: '1', Message: 'message 1' }, - { Id: '2', Message: 'message 2' } - ] - }, e => e && done(e)) + sns.publish( + { TopicArn, Message: 'message DSM' }, + (err) => { + if (err) return done(err) + + let publishSpanMeta = {} + agent.use(traces => { + const span = traces[0][0] + + if (span.resource.startsWith('publish')) { + publishSpanMeta = span.meta + } + + expect(publishSpanMeta).to.include({ + 'pathway.hash': expectedProducerHash + }) + }).then(done, done) + }) }) }) - } - // TODO: Figure out why this fails only in 3.0.0 - if (version !== '3.0.0') { - it('skips injecting trace context to SNS if message attributes are full', done => { + it('injects DSM pathway hash to SQS receive span from SNS topic', done => { sns.subscribe(subParams, (err, data) => { if (err) return done(err) - sqs.receiveMessage(receiveParams, (err, data) => { - if (err) return done(err) + sns.publish( + { TopicArn, Message: 'message DSM' }, + (err) => { + if (err) return done(err) + }) + + sqs.receiveMessage( + receiveParams, + (err, res) => { + if (err) return done(err) + + let consumeSpanMeta = {} + agent.use(traces => { + const span = traces[0][0] + + if (span.name === 'aws.response') { + consumeSpanMeta = span.meta + } + + expect(consumeSpanMeta).to.include({ + 'pathway.hash': expectedConsumerHash + }) + }).then(done, done) + }) + }) + }) - try { - expect(data.Messages[0].Body).to.not.include('datadog') - done() - } catch (e) { - done(e) + it('outputs DSM stats to the agent when publishing a message', done => { + agent.expectPipelineStats(dsmStats => { + let statsPointsReceived = 0 + // we should have 1 dsm stats points + dsmStats.forEach((timeStatsBucket) => { + if (timeStatsBucket && timeStatsBucket.Stats) { + timeStatsBucket.Stats.forEach((statsBuckets) => { + statsPointsReceived += statsBuckets.Stats.length + }) } }) + expect(statsPointsReceived).to.be.at.least(1) + expect(agent.dsmStatsExist(agent, expectedProducerHash)).to.equal(true) + }).then(done, done) - sns.publish({ - TopicArn, - Message: 'message 1', - MessageAttributes: { - keyOne: { DataType: 'String', StringValue: 'keyOne' }, - keyTwo: { DataType: 'String', StringValue: 'keyTwo' }, - keyThree: { DataType: 'String', StringValue: 'keyThree' }, - keyFour: { DataType: 'String', StringValue: 'keyFour' }, - keyFive: { DataType: 'String', StringValue: 'keyFive' }, - keySix: { DataType: 'String', StringValue: 'keySix' }, - keySeven: { DataType: 'String', StringValue: 'keySeven' }, - keyEight: { DataType: 'String', StringValue: 'keyEight' }, - keyNine: { DataType: 'String', StringValue: 'keyNine' }, - keyTen: { DataType: 'String', StringValue: 'keyTen' } - } - }, e => e && done(e)) + sns.subscribe(subParams, () => { + sns.publish({ TopicArn, Message: 'message DSM' }, () => {}) }) }) - } - it('generates tags for proper publish calls', done => { - agent.use(traces => { - const span = traces[0][0] + it('outputs DSM stats to the agent when consuming a message', done => { + agent.expectPipelineStats(dsmStats => { + let statsPointsReceived = 0 + // we should have 2 dsm stats points + dsmStats.forEach((timeStatsBucket) => { + if (timeStatsBucket && timeStatsBucket.Stats) { + timeStatsBucket.Stats.forEach((statsBuckets) => { + statsPointsReceived += statsBuckets.Stats.length + }) + } + }) + expect(statsPointsReceived).to.be.at.least(2) + expect(agent.dsmStatsExist(agent, expectedConsumerHash)).to.equal(true) + }).then(done, done) - expect(span.resource).to.equal(`publish ${TopicArn}`) - expect(span.meta).to.include({ - 'aws.sns.topic_arn': TopicArn, - 'topicname': 'TestTopic', - 'aws_service': 'SNS', - 'region': 'us-east-1' + sns.subscribe(subParams, () => { + sns.publish({ TopicArn, Message: 'message DSM' }, () => { + sqs.receiveMessage(receiveParams, () => {}) + }) }) - }).then(done, done) + }) + + it('outputs DSM stats to the agent when publishing batch messages', function (done) { + // publishBatch was released with version 2.1031.0 for the aws-sdk + // publishBatch does not work with smithy-client 3.0.0, unable to find compatible version it + // was released for, but works on 3.374.0 + if ( + (moduleName === '@aws-sdk/smithy-client' && semver.intersects(version, '>=3.374.0')) || + (moduleName === 'aws-sdk' && semver.intersects(version, '>=2.1031.0')) + ) { + // we need to stub Date.now() to ensure a new stats bucket is created for each call + // otherwise, all stats checkpoints will be combined into a single stats points + let now = Date.now() + nowStub = sinon.stub(Date, 'now') + nowStub.callsFake(() => { + now += 1000000 + return now + }) - sns.publish({ TopicArn, Message: 'message 1' }, e => e && done(e)) + agent.expectPipelineStats(dsmStats => { + let statsPointsReceived = 0 + // we should have 3 dsm stats points + dsmStats.forEach((timeStatsBucket) => { + if (timeStatsBucket && timeStatsBucket.Stats) { + timeStatsBucket.Stats.forEach((statsBuckets) => { + statsPointsReceived += statsBuckets.Stats.length + }) + } + }) + expect(statsPointsReceived).to.be.at.least(3) + expect(agent.dsmStatsExist(agent, expectedProducerHash)).to.equal(true) + }, { timeoutMs: 2000 }).then(done, done) + + sns.subscribe(subParams, () => { + sns.publishBatch( + { + TopicArn, + PublishBatchRequestEntries: [ + { + Id: '1', + Message: 'message DSM 1' + }, + { + Id: '2', + Message: 'message DSM 2' + }, + { + Id: '3', + Message: 'message DSM 3' + } + ] + }, () => { + nowStub.restore() + }) + }) + } else { + this.skip() + } + }) }) }) }) diff --git a/packages/datadog-plugin-aws-sdk/test/spec_helpers.js b/packages/datadog-plugin-aws-sdk/test/spec_helpers.js index 821f2486c23..f86f50acd0e 100644 --- a/packages/datadog-plugin-aws-sdk/test/spec_helpers.js +++ b/packages/datadog-plugin-aws-sdk/test/spec_helpers.js @@ -7,13 +7,13 @@ const helpers = { setup () { before(() => { - process.env['AWS_SECRET_ACCESS_KEY'] = '0000000000/00000000000000000000000000000' - process.env['AWS_ACCESS_KEY_ID'] = '00000000000000000000' + process.env.AWS_SECRET_ACCESS_KEY = '0000000000/00000000000000000000000000000' + process.env.AWS_ACCESS_KEY_ID = '00000000000000000000' }) after(() => { - delete process.env['AWS_SECRET_ACCESS_KEY'] - delete process.env['AWS_ACCESS_KEY_ID'] + delete process.env.AWS_SECRET_ACCESS_KEY + delete process.env.AWS_ACCESS_KEY_ID }) } } diff --git a/packages/datadog-plugin-aws-sdk/test/sqs.spec.js b/packages/datadog-plugin-aws-sdk/test/sqs.spec.js index 0ee09e23d24..9c0c3686f9b 100644 --- a/packages/datadog-plugin-aws-sdk/test/sqs.spec.js +++ b/packages/datadog-plugin-aws-sdk/test/sqs.spec.js @@ -1,16 +1,26 @@ 'use strict' +const sinon = require('sinon') const agent = require('../../dd-trace/test/plugins/agent') const { setup } = require('./spec_helpers') +const semver = require('semver') const { rawExpectedSchema } = require('./sqs-naming') -const queueOptions = { - QueueName: 'SQS_QUEUE_NAME', - Attributes: { - 'MessageRetentionPeriod': '86400' +const queueName = 'SQS_QUEUE_NAME' +const queueNameDSM = 'SQS_QUEUE_NAME_DSM' + +const getQueueParams = (queueName) => { + return { + QueueName: queueName, + Attributes: { + MessageRetentionPeriod: '86400' + } } } +const queueOptions = getQueueParams(queueName) +const queueOptionsDsm = getQueueParams(queueNameDSM) + describe('Plugin', () => { describe('aws-sdk (sqs)', function () { setup() @@ -18,16 +28,21 @@ describe('Plugin', () => { withVersions('aws-sdk', ['aws-sdk', '@aws-sdk/smithy-client'], (version, moduleName) => { let AWS let sqs - let QueueUrl + const QueueUrl = 'http://127.0.0.1:4566/00000000000000000000/SQS_QUEUE_NAME' + const QueueUrlDsm = 'http://127.0.0.1:4566/00000000000000000000/SQS_QUEUE_NAME_DSM' let tracer const sqsClientName = moduleName === '@aws-sdk/smithy-client' ? '@aws-sdk/client-sqs' : 'aws-sdk' describe('without configuration', () => { before(() => { + process.env.DD_DATA_STREAMS_ENABLED = 'true' tracer = require('../../dd-trace') + tracer.use('aws-sdk', { sqs: { batchPropagationEnabled: true } }) - return agent.load('aws-sdk') + return agent.load( + 'aws-sdk', { sqs: { dsmEnabled: false, batchPropagationEnabled: true } }, { dsmEnabled: true } + ) }) before(done => { @@ -37,8 +52,6 @@ describe('Plugin', () => { sqs.createQueue(queueOptions, (err, res) => { if (err) return done(err) - QueueUrl = res.QueueUrl - done() }) }) @@ -107,6 +120,9 @@ describe('Plugin', () => { const span = traces[0][0] expect(span.resource.startsWith('sendMessage')).to.equal(true) + expect(span.meta).to.include({ + queuename: 'SQS_QUEUE_NAME' + }) parentId = span.span_id.toString() traceId = span.trace_id.toString() @@ -135,6 +151,74 @@ describe('Plugin', () => { }) }) + it('should propagate the tracing context from the producer to the consumer in batch operations', (done) => { + let parentId + let traceId + + agent.use(traces => { + const span = traces[0][0] + + expect(span.resource.startsWith('sendMessageBatch')).to.equal(true) + expect(span.meta).to.include({ + queuename: 'SQS_QUEUE_NAME' + }) + + parentId = span.span_id.toString() + traceId = span.trace_id.toString() + }) + + let batchChildSpans = 0 + agent.use(traces => { + const span = traces[0][0] + + expect(parentId).to.be.a('string') + expect(span.parent_id.toString()).to.equal(parentId) + expect(span.trace_id.toString()).to.equal(traceId) + batchChildSpans += 1 + expect(batchChildSpans).to.equal(3) + }, { timeoutMs: 2000 }).then(done, done) + + sqs.sendMessageBatch( + { + Entries: [ + { + Id: '1', + MessageBody: 'test batch propagation 1' + }, + { + Id: '2', + MessageBody: 'test batch propagation 2' + }, + { + Id: '3', + MessageBody: 'test batch propagation 3' + } + ], + QueueUrl + }, (err) => { + if (err) return done(err) + + function receiveMessage () { + sqs.receiveMessage({ + QueueUrl, + MaxNumberOfMessages: 1 + }, (err, data) => { + if (err) return done(err) + + for (const message in data.Messages) { + const recordData = data.Messages[message].MessageAttributes + expect(recordData).to.have.property('_datadog') + const traceContext = JSON.parse(recordData._datadog.StringValue) + expect(traceContext).to.have.property('x-datadog-trace-id') + } + }) + } + receiveMessage() + receiveMessage() + receiveMessage() + }) + }) + it('should run the consumer in the context of its span', (done) => { sqs.sendMessage({ MessageBody: 'test body', @@ -184,6 +268,32 @@ describe('Plugin', () => { }) }) }) + + it('should propagate DSM context from producer to consumer', (done) => { + sqs.sendMessage({ + MessageBody: 'test DSM', + QueueUrl + }, (err) => { + if (err) return done(err) + + const beforeSpan = tracer.scope().active() + + sqs.receiveMessage({ + QueueUrl, + MessageAttributeNames: ['.*'] + }, (err) => { + if (err) return done(err) + + const span = tracer.scope().active() + + expect(span).to.not.equal(beforeSpan) + return Promise.resolve().then(() => { + expect(tracer.scope().active()).to.equal(span) + done() + }) + }) + }) + }) }) describe('with configuration', () => { @@ -192,9 +302,12 @@ describe('Plugin', () => { return agent.load('aws-sdk', { sqs: { - consumer: false + consumer: false, + dsmEnabled: false } - }) + }, + { dsmEnabled: true } + ) }) before(done => { @@ -204,8 +317,6 @@ describe('Plugin', () => { sqs.createQueue(queueOptions, (err, res) => { if (err) return done(err) - QueueUrl = res.QueueUrl - done() }) }) @@ -230,9 +341,9 @@ describe('Plugin', () => { }) expect(span.meta).to.include({ - 'queuename': 'SQS_QUEUE_NAME', - 'aws_service': 'SQS', - 'region': 'us-east-1' + queuename: 'SQS_QUEUE_NAME', + aws_service: 'SQS', + region: 'us-east-1' }) total++ }).catch(() => {}, { timeoutMs: 100 }) @@ -268,6 +379,219 @@ describe('Plugin', () => { }, 250) }) }) + + describe('data stream monitoring', () => { + const expectedProducerHash = '4673734031235697865' + const expectedConsumerHash = '9749472979704578383' + let nowStub + + before(() => { + process.env.DD_DATA_STREAMS_ENABLED = 'true' + tracer = require('../../dd-trace') + tracer.use('aws-sdk', { sqs: { dsmEnabled: true } }) + }) + + before(async () => { + return agent.load('aws-sdk', { + sqs: { + consumer: false, + dsmEnabled: true + } + }, + { dsmEnabled: true }) + }) + + before(done => { + AWS = require(`../../../versions/${sqsClientName}@${version}`).get() + + sqs = new AWS.SQS({ endpoint: 'http://127.0.0.1:4566', region: 'us-east-1' }) + sqs.createQueue(queueOptionsDsm, (err, res) => { + if (err) return done(err) + + done() + }) + }) + + after(done => { + sqs.deleteQueue({ QueueUrl: QueueUrlDsm }, done) + }) + + after(() => { + return agent.close({ ritmReset: false }) + }) + + afterEach(() => { + try { + nowStub.restore() + } catch { + // pass + } + agent.reload('aws-sdk', { kinesis: { dsmEnabled: true } }, { dsmEnabled: true }) + }) + + it('Should set pathway hash tag on a span when producing', (done) => { + sqs.sendMessage({ + MessageBody: 'test DSM', + QueueUrl: QueueUrlDsm + }, (err) => { + if (err) return done(err) + + let produceSpanMeta = {} + agent.use(traces => { + const span = traces[0][0] + + if (span.resource.startsWith('sendMessage')) { + produceSpanMeta = span.meta + } + + expect(produceSpanMeta).to.include({ + 'pathway.hash': expectedProducerHash + }) + }).then(done, done) + }) + }) + + it('Should set pathway hash tag on a span when consuming', (done) => { + sqs.sendMessage({ + MessageBody: 'test DSM', + QueueUrl: QueueUrlDsm + }, (err) => { + if (err) return done(err) + + sqs.receiveMessage({ + QueueUrl: QueueUrlDsm, + MessageAttributeNames: ['.*'] + }, (err) => { + if (err) return done(err) + + let consumeSpanMeta = {} + agent.use(traces => { + const span = traces[0][0] + + if (span.name === 'aws.response') { + consumeSpanMeta = span.meta + } + + expect(consumeSpanMeta).to.include({ + 'pathway.hash': expectedConsumerHash + }) + }).then(done, done) + }) + }) + }) + + if (sqsClientName === 'aws-sdk' && semver.intersects(version, '>=2.3')) { + it('Should set pathway hash tag on a span when consuming and promise() was used over a callback', + async () => { + await sqs.sendMessage({ MessageBody: 'test DSM', QueueUrl: QueueUrlDsm }) + await sqs.receiveMessage({ QueueUrl: QueueUrlDsm }).promise() + + let consumeSpanMeta = {} + return new Promise((resolve, reject) => { + agent.use(traces => { + const span = traces[0][0] + + if (span.name === 'aws.request' && span.meta['aws.operation'] === 'receiveMessage') { + consumeSpanMeta = span.meta + } + + try { + expect(consumeSpanMeta).to.include({ + 'pathway.hash': expectedConsumerHash + }) + resolve() + } catch (error) { + reject(error) + } + }) + }) + }) + } + + it('Should emit DSM stats to the agent when sending a message', done => { + agent.expectPipelineStats(dsmStats => { + let statsPointsReceived = 0 + // we should have 1 dsm stats points + dsmStats.forEach((timeStatsBucket) => { + if (timeStatsBucket && timeStatsBucket.Stats) { + timeStatsBucket.Stats.forEach((statsBuckets) => { + statsPointsReceived += statsBuckets.Stats.length + }) + } + }) + expect(statsPointsReceived).to.be.at.least(1) + expect(agent.dsmStatsExist(agent, expectedProducerHash)).to.equal(true) + }).then(done, done) + + sqs.sendMessage({ MessageBody: 'test DSM', QueueUrl: QueueUrlDsm }, () => {}) + }) + + it('Should emit DSM stats to the agent when receiving a message', done => { + agent.expectPipelineStats(dsmStats => { + let statsPointsReceived = 0 + // we should have 2 dsm stats points + dsmStats.forEach((timeStatsBucket) => { + if (timeStatsBucket && timeStatsBucket.Stats) { + timeStatsBucket.Stats.forEach((statsBuckets) => { + statsPointsReceived += statsBuckets.Stats.length + }) + } + }) + expect(statsPointsReceived).to.be.at.least(2) + expect(agent.dsmStatsExist(agent, expectedConsumerHash)).to.equal(true) + }).then(done, done) + + sqs.sendMessage({ MessageBody: 'test DSM', QueueUrl: QueueUrlDsm }, () => { + sqs.receiveMessage({ QueueUrl: QueueUrlDsm, MessageAttributeNames: ['.*'] }, () => {}) + }) + }) + + it('Should emit DSM stats to the agent when sending batch messages', done => { + // we need to stub Date.now() to ensure a new stats bucket is created for each call + // otherwise, all stats checkpoints will be combined into a single stats points + let now = Date.now() + nowStub = sinon.stub(Date, 'now') + nowStub.callsFake(() => { + now += 1000000 + return now + }) + + agent.expectPipelineStats(dsmStats => { + let statsPointsReceived = 0 + // we should have 3 dsm stats points + dsmStats.forEach((timeStatsBucket) => { + if (timeStatsBucket && timeStatsBucket.Stats) { + timeStatsBucket.Stats.forEach((statsBuckets) => { + statsPointsReceived += statsBuckets.Stats.length + }) + } + }) + expect(statsPointsReceived).to.be.at.least(3) + expect(agent.dsmStatsExist(agent, expectedProducerHash)).to.equal(true) + }).then(done, done) + + sqs.sendMessageBatch( + { + Entries: [ + { + Id: '1', + MessageBody: 'test DSM 1' + }, + { + Id: '2', + MessageBody: 'test DSM 2' + }, + { + Id: '3', + MessageBody: 'test DSM 3' + } + ], + QueueUrl: QueueUrlDsm + }, () => { + nowStub.restore() + }) + }) + }) }) }) }) diff --git a/packages/datadog-plugin-aws-sdk/test/stepfunctions.spec.js b/packages/datadog-plugin-aws-sdk/test/stepfunctions.spec.js new file mode 100644 index 00000000000..ed77ecd51b2 --- /dev/null +++ b/packages/datadog-plugin-aws-sdk/test/stepfunctions.spec.js @@ -0,0 +1,128 @@ +/* eslint-disable max-len */ +'use strict' + +const semver = require('semver') +const agent = require('../../dd-trace/test/plugins/agent') +const { setup } = require('./spec_helpers') + +const helloWorldSMD = { + Comment: 'A Hello World example of the Amazon States Language using a Pass state', + StartAt: 'HelloWorld', + States: { + HelloWorld: { + Type: 'Pass', + Result: 'Hello World!', + End: true + } + } +} + +describe('Sfn', () => { + let tracer + + withVersions('aws-sdk', ['aws-sdk', '@aws-sdk/smithy-client'], (version, moduleName) => { + let stateMachineArn + let client + + setup() + + before(() => { + client = getClient() + }) + + function getClient () { + const params = { endpoint: 'http://127.0.0.1:4566', region: 'us-east-1' } + if (moduleName === '@aws-sdk/smithy-client') { + const lib = require(`../../../versions/@aws-sdk/client-sfn@${version}`).get() + const client = new lib.SFNClient(params) + return { + client, + createStateMachine: function () { + const req = new lib.CreateStateMachineCommand(...arguments) + return client.send(req) + }, + deleteStateMachine: function () { + const req = new lib.DeleteStateMachineCommand(...arguments) + return client.send(req) + }, + startExecution: function () { + const req = new lib.StartExecutionCommand(...arguments) + return client.send(req) + }, + describeExecution: function () { + const req = new lib.DescribeExecutionCommand(...arguments) + return client.send(req) + } + } + } else { + const { StepFunctions } = require(`../../../versions/aws-sdk@${version}`).get() + const client = new StepFunctions(params) + return { + client, + createStateMachine: function () { return client.createStateMachine(...arguments).promise() }, + deleteStateMachine: function () { + return client.deleteStateMachine(...arguments).promise() + }, + startExecution: function () { return client.startExecution(...arguments).promise() }, + describeExecution: function () { return client.describeExecution(...arguments).promise() } + } + } + } + + async function createStateMachine (name, definition, xargs) { + return client.createStateMachine({ + definition: JSON.stringify(definition), + name, + roleArn: 'arn:aws:iam::123456:role/test', + ...xargs + }) + } + + async function deleteStateMachine (arn) { + return client.deleteStateMachine({ stateMachineArn: arn }) + } + + describe('Traces', () => { + before(() => { + tracer = require('../../dd-trace') + tracer.use('aws-sdk') + }) + // aws-sdk v2 doesn't support StepFunctions below 2.7.10 + // https://github.com/aws/aws-sdk-js/blob/5dba638fd/CHANGELOG.md?plain=1#L18 + if (moduleName !== 'aws-sdk' || semver.intersects(version, '>=2.7.10')) { + beforeEach(() => { return agent.load('aws-sdk') }) + beforeEach(async () => { + const data = await createStateMachine('helloWorld', helloWorldSMD, {}) + stateMachineArn = data.stateMachineArn + }) + + afterEach(() => { return agent.close({ ritmReset: false }) }) + + afterEach(async () => { + await deleteStateMachine(stateMachineArn) + }) + + it('is instrumented', async function () { + const startExecInput = { + stateMachineArn, + input: JSON.stringify({ moduleName }) + } + const expectSpanPromise = agent.use(traces => { + const span = traces[0][0] + expect(span).to.have.property('resource', 'startExecution') + expect(span.meta).to.have.property('statemachinearn', stateMachineArn) + }) + + const resp = await client.startExecution(startExecInput) + + const result = await client.describeExecution({ executionArn: resp.executionArn }) + const sfInput = JSON.parse(result.input) + expect(sfInput).to.have.property('_datadog') + expect(sfInput._datadog).to.have.property('x-datadog-trace-id') + expect(sfInput._datadog).to.have.property('x-datadog-parent-id') + return expectSpanPromise.then(() => {}) + }) + } + }) + }) +}) diff --git a/packages/datadog-plugin-axios/test/integration-test/client.spec.js b/packages/datadog-plugin-axios/test/integration-test/client.spec.js index 7a711a74a4d..018cdc7522d 100644 --- a/packages/datadog-plugin-axios/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-axios/test/integration-test/client.spec.js @@ -16,7 +16,7 @@ describe('esm', () => { before(async function () { this.timeout(20000) sandbox = await createSandbox(['axios'], false, [ - `./packages/datadog-plugin-axios/test/integration-test/*`]) + './packages/datadog-plugin-axios/test/integration-test/*']) }) after(async () => { diff --git a/packages/datadog-plugin-azure-functions/src/index.js b/packages/datadog-plugin-azure-functions/src/index.js new file mode 100644 index 00000000000..2c85403906c --- /dev/null +++ b/packages/datadog-plugin-azure-functions/src/index.js @@ -0,0 +1,77 @@ +'use strict' + +const TracingPlugin = require('../../dd-trace/src/plugins/tracing') +const { storage } = require('../../datadog-core') +const serverless = require('../../dd-trace/src/plugins/util/serverless') +const web = require('../../dd-trace/src/plugins/util/web') + +const triggerMap = { + deleteRequest: 'Http', + http: 'Http', + get: 'Http', + patch: 'Http', + post: 'Http', + put: 'Http' +} + +class AzureFunctionsPlugin extends TracingPlugin { + static get id () { return 'azure-functions' } + static get operation () { return 'invoke' } + static get kind () { return 'server' } + static get type () { return 'serverless' } + + static get prefix () { return 'tracing:datadog:azure-functions:invoke' } + + bindStart (ctx) { + const { functionName, methodName } = ctx + const store = storage.getStore() + + const span = this.startSpan(this.operationName(), { + service: this.serviceName(), + type: 'serverless', + meta: { + 'aas.function.name': functionName, + 'aas.function.trigger': mapTriggerTag(methodName) + } + }, false) + + ctx.span = span + ctx.parentStore = store + ctx.currentStore = { ...store, span } + + return ctx.currentStore + } + + error (ctx) { + this.addError(ctx.error) + ctx.currentStore.span.setTag('error.message', ctx.error) + } + + asyncEnd (ctx) { + const { httpRequest, result = {} } = ctx + const path = (new URL(httpRequest.url)).pathname + const req = { + method: httpRequest.method, + headers: Object.fromEntries(httpRequest.headers.entries()), + url: path + } + + const context = web.patch(req) + context.config = this.config + context.paths = [path] + context.res = { statusCode: result.status } + context.span = ctx.currentStore.span + + serverless.finishSpan(context) + } + + configure (config) { + return super.configure(web.normalizeConfig(config)) + } +} + +function mapTriggerTag (methodName) { + return triggerMap[methodName] || 'Unknown' +} + +module.exports = AzureFunctionsPlugin diff --git a/packages/datadog-plugin-azure-functions/test/integration-test/client.spec.js b/packages/datadog-plugin-azure-functions/test/integration-test/client.spec.js new file mode 100644 index 00000000000..8d5a0d43fdb --- /dev/null +++ b/packages/datadog-plugin-azure-functions/test/integration-test/client.spec.js @@ -0,0 +1,100 @@ +'use strict' + +const { + FakeAgent, + hookFile, + createSandbox, + curlAndAssertMessage +} = require('../../../../integration-tests/helpers') +const { spawn } = require('child_process') +const { assert } = require('chai') + +describe('esm', () => { + let agent + let proc + let sandbox + + withVersions('azure-functions', '@azure/functions', version => { + before(async function () { + this.timeout(50000) + sandbox = await createSandbox([`@azure/functions@${version}`, 'azure-functions-core-tools@4'], false, + ['./packages/datadog-plugin-azure-functions/test/integration-test/fixtures/*']) + }) + + after(async function () { + this.timeout(50000) + await sandbox.remove() + }) + + beforeEach(async () => { + agent = await new FakeAgent().start() + }) + + afterEach(async () => { + proc && proc.kill('SIGINT') + await agent.stop() + }) + + it('is instrumented', async () => { + const envArgs = { + PATH: `${sandbox.folder}/node_modules/azure-functions-core-tools/bin:${process.env.PATH}` + } + proc = await spawnPluginIntegrationTestProc(sandbox.folder, 'func', ['start'], agent.port, undefined, envArgs) + + return curlAndAssertMessage(agent, 'http://127.0.0.1:7071/api/httptest', ({ headers, payload }) => { + assert.propertyVal(headers, 'host', `127.0.0.1:${agent.port}`) + assert.isArray(payload) + assert.strictEqual(payload.length, 1) + assert.isArray(payload[0]) + assert.strictEqual(payload[0].length, 1) + assert.propertyVal(payload[0][0], 'name', 'azure-functions.invoke') + }) + }).timeout(50000) + }) +}) + +async function spawnPluginIntegrationTestProc (cwd, command, args, agentPort, stdioHandler, additionalEnvArgs = {}) { + let env = { + NODE_OPTIONS: `--loader=${hookFile}`, + DD_TRACE_AGENT_PORT: agentPort + } + env = { ...env, ...additionalEnvArgs } + return spawnProc(command, args, { + cwd, + env + }, stdioHandler) +} + +function spawnProc (command, args, options = {}, stdioHandler, stderrHandler) { + const proc = spawn(command, args, { ...options, stdio: 'pipe' }) + return new Promise((resolve, reject) => { + proc + .on('error', reject) + .on('exit', code => { + if (code !== 0) { + reject(new Error(`Process exited with status code ${code}.`)) + } + resolve() + }) + + proc.stdout.on('data', data => { + if (stdioHandler) { + stdioHandler(data) + } + // eslint-disable-next-line no-console + if (!options.silent) console.log(data.toString()) + + if (data.toString().includes('http://localhost:7071/api/httptest')) { + resolve(proc) + } + }) + + proc.stderr.on('data', data => { + if (stderrHandler) { + stderrHandler(data) + } + // eslint-disable-next-line no-console + if (!options.silent) console.error(data.toString()) + }) + }) +} diff --git a/packages/datadog-plugin-azure-functions/test/integration-test/fixtures/host.json b/packages/datadog-plugin-azure-functions/test/integration-test/fixtures/host.json new file mode 100644 index 00000000000..06d01bdaa95 --- /dev/null +++ b/packages/datadog-plugin-azure-functions/test/integration-test/fixtures/host.json @@ -0,0 +1,15 @@ +{ + "version": "2.0", + "logging": { + "applicationInsights": { + "samplingSettings": { + "isEnabled": true, + "excludedTypes": "Request" + } + } + }, + "extensionBundle": { + "id": "Microsoft.Azure.Functions.ExtensionBundle", + "version": "[4.*, 5.0.0)" + } +} diff --git a/packages/datadog-plugin-azure-functions/test/integration-test/fixtures/local.settings.json b/packages/datadog-plugin-azure-functions/test/integration-test/fixtures/local.settings.json new file mode 100644 index 00000000000..6beb0236ad6 --- /dev/null +++ b/packages/datadog-plugin-azure-functions/test/integration-test/fixtures/local.settings.json @@ -0,0 +1,8 @@ +{ + "IsEncrypted": false, + "Values": { + "FUNCTIONS_WORKER_RUNTIME": "node", + "AzureWebJobsFeatureFlags": "EnableWorkerIndexing", + "AzureWebJobsStorage": "" + } +} diff --git a/packages/datadog-plugin-azure-functions/test/integration-test/fixtures/package.json b/packages/datadog-plugin-azure-functions/test/integration-test/fixtures/package.json new file mode 100644 index 00000000000..07b0ac311ee --- /dev/null +++ b/packages/datadog-plugin-azure-functions/test/integration-test/fixtures/package.json @@ -0,0 +1,15 @@ +{ + "name": "azure-function-node-integration-test", + "version": "1.0.0", + "description": "", + "main": "src/functions/server.mjs", + "scripts": { + "start": "func start" + }, + "dependencies": { + "@azure/functions": "^4.0.0" + }, + "devDependencies": { + "azure-functions-core-tools": "^4.x" + } +} diff --git a/packages/datadog-plugin-azure-functions/test/integration-test/fixtures/src/functions/server.mjs b/packages/datadog-plugin-azure-functions/test/integration-test/fixtures/src/functions/server.mjs new file mode 100644 index 00000000000..2efdd200732 --- /dev/null +++ b/packages/datadog-plugin-azure-functions/test/integration-test/fixtures/src/functions/server.mjs @@ -0,0 +1,15 @@ +import 'dd-trace/init.js' +import { app } from '@azure/functions' + +async function handlerFunction (request, context) { + return { + status: 200, + body: 'Hello Datadog!' + } +} + +app.http('httptest', { + methods: ['GET'], + authLevel: 'anonymous', + handler: handlerFunction +}) diff --git a/packages/datadog-plugin-azure-functions/test/integration-test/fixtures/yarn.lock b/packages/datadog-plugin-azure-functions/test/integration-test/fixtures/yarn.lock new file mode 100644 index 00000000000..98c420c8953 --- /dev/null +++ b/packages/datadog-plugin-azure-functions/test/integration-test/fixtures/yarn.lock @@ -0,0 +1,269 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"@azure/functions@^4.0.0": + version "4.5.1" + resolved "https://registry.yarnpkg.com/@azure/functions/-/functions-4.5.1.tgz#70d1a99d335af87579a55d3c149ef1ae77da0a66" + integrity sha512-ikiw1IrM2W9NlQM3XazcX+4Sq3XAjZi4eeG22B5InKC2x5i7MatGF2S/Gn1ACZ+fEInwu+Ru9J8DlnBv1/hIvg== + dependencies: + cookie "^0.6.0" + long "^4.0.0" + undici "^5.13.0" + +"@fastify/busboy@^2.0.0": + version "2.1.1" + resolved "https://registry.yarnpkg.com/@fastify/busboy/-/busboy-2.1.1.tgz#b9da6a878a371829a0502c9b6c1c143ef6663f4d" + integrity sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA== + +"@types/node@*": + version "22.7.4" + resolved "https://registry.yarnpkg.com/@types/node/-/node-22.7.4.tgz#e35d6f48dca3255ce44256ddc05dee1c23353fcc" + integrity sha512-y+NPi1rFzDs1NdQHHToqeiX2TIS79SWEAw9GYhkkx8bD0ChpfqC+n2j5OXOCpzfojBEBt6DnEnnG9MY0zk1XLg== + dependencies: + undici-types "~6.19.2" + +"@types/yauzl@^2.9.1": + version "2.10.3" + resolved "https://registry.yarnpkg.com/@types/yauzl/-/yauzl-2.10.3.tgz#e9b2808b4f109504a03cda958259876f61017999" + integrity sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q== + dependencies: + "@types/node" "*" + +agent-base@6: + version "6.0.2" + resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.2.tgz#49fff58577cfee3f37176feab4c22e00f86d7f77" + integrity sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ== + dependencies: + debug "4" + +ansi-styles@^4.1.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-4.3.0.tgz#edd803628ae71c04c85ae7a0906edad34b648937" + integrity sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg== + dependencies: + color-convert "^2.0.1" + +azure-functions-core-tools@^4.x: + version "4.0.6280" + resolved "https://registry.yarnpkg.com/azure-functions-core-tools/-/azure-functions-core-tools-4.0.6280.tgz#59b4d9402846760aef3ad292355c3eeb4e5f21ad" + integrity sha512-DVSgYNnT4POLbj/YV3FKtNdo9KT/M5Dl//slWEmVwZo1y4aJEsUApn6DtkZswut76I3S9eKGC5IaC84j5OGNaA== + dependencies: + chalk "3.0.0" + extract-zip "^2.0.1" + https-proxy-agent "5.0.0" + progress "2.0.3" + rimraf "4.4.1" + +balanced-match@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee" + integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== + +brace-expansion@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-2.0.1.tgz#1edc459e0f0c548486ecf9fc99f2221364b9a0ae" + integrity sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA== + dependencies: + balanced-match "^1.0.0" + +buffer-crc32@~0.2.3: + version "0.2.13" + resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242" + integrity sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ== + +chalk@3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-3.0.0.tgz#3f73c2bf526591f574cc492c51e2456349f844e4" + integrity sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg== + dependencies: + ansi-styles "^4.1.0" + supports-color "^7.1.0" + +color-convert@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-2.0.1.tgz#72d3a68d598c9bdb3af2ad1e84f21d896abd4de3" + integrity sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ== + dependencies: + color-name "~1.1.4" + +color-name@~1.1.4: + version "1.1.4" + resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" + integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== + +cookie@^0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.6.0.tgz#2798b04b071b0ecbff0dbb62a505a8efa4e19051" + integrity sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw== + +debug@4, debug@^4.1.1: + version "4.3.7" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.7.tgz#87945b4151a011d76d95a198d7111c865c360a52" + integrity sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ== + dependencies: + ms "^2.1.3" + +end-of-stream@^1.1.0: + version "1.4.4" + resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0" + integrity sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q== + dependencies: + once "^1.4.0" + +extract-zip@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/extract-zip/-/extract-zip-2.0.1.tgz#663dca56fe46df890d5f131ef4a06d22bb8ba13a" + integrity sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg== + dependencies: + debug "^4.1.1" + get-stream "^5.1.0" + yauzl "^2.10.0" + optionalDependencies: + "@types/yauzl" "^2.9.1" + +fd-slicer@~1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/fd-slicer/-/fd-slicer-1.1.0.tgz#25c7c89cb1f9077f8891bbe61d8f390eae256f1e" + integrity sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g== + dependencies: + pend "~1.2.0" + +fs.realpath@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" + integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw== + +get-stream@^5.1.0: + version "5.2.0" + resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-5.2.0.tgz#4966a1795ee5ace65e706c4b7beb71257d6e22d3" + integrity sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA== + dependencies: + pump "^3.0.0" + +glob@^9.2.0: + version "9.3.5" + resolved "https://registry.yarnpkg.com/glob/-/glob-9.3.5.tgz#ca2ed8ca452781a3009685607fdf025a899dfe21" + integrity sha512-e1LleDykUz2Iu+MTYdkSsuWX8lvAjAcs0Xef0lNIu0S2wOAzuTxCJtcd9S3cijlwYF18EsU3rzb8jPVobxDh9Q== + dependencies: + fs.realpath "^1.0.0" + minimatch "^8.0.2" + minipass "^4.2.4" + path-scurry "^1.6.1" + +has-flag@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-4.0.0.tgz#944771fd9c81c81265c4d6941860da06bb59479b" + integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ== + +https-proxy-agent@5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz#e2a90542abb68a762e0a0850f6c9edadfd8506b2" + integrity sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA== + dependencies: + agent-base "6" + debug "4" + +long@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/long/-/long-4.0.0.tgz#9a7b71cfb7d361a194ea555241c92f7468d5bf28" + integrity sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA== + +lru-cache@^10.2.0: + version "10.4.3" + resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-10.4.3.tgz#410fc8a17b70e598013df257c2446b7f3383f119" + integrity sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ== + +minimatch@^8.0.2: + version "8.0.4" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-8.0.4.tgz#847c1b25c014d4e9a7f68aaf63dedd668a626229" + integrity sha512-W0Wvr9HyFXZRGIDgCicunpQ299OKXs9RgZfaukz4qAW/pJhcpUfupc9c+OObPOFueNy8VSrZgEmDtk6Kh4WzDA== + dependencies: + brace-expansion "^2.0.1" + +minipass@^4.2.4: + version "4.2.8" + resolved "https://registry.yarnpkg.com/minipass/-/minipass-4.2.8.tgz#f0010f64393ecfc1d1ccb5f582bcaf45f48e1a3a" + integrity sha512-fNzuVyifolSLFL4NzpF+wEF4qrgqaaKX0haXPQEdQ7NKAN+WecoKMHV09YcuL/DHxrUsYQOK3MiuDf7Ip2OXfQ== + +"minipass@^5.0.0 || ^6.0.2 || ^7.0.0": + version "7.1.2" + resolved "https://registry.yarnpkg.com/minipass/-/minipass-7.1.2.tgz#93a9626ce5e5e66bd4db86849e7515e92340a707" + integrity sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw== + +ms@^2.1.3: + version "2.1.3" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" + integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== + +once@^1.3.1, once@^1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" + integrity sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w== + dependencies: + wrappy "1" + +path-scurry@^1.6.1: + version "1.11.1" + resolved "https://registry.yarnpkg.com/path-scurry/-/path-scurry-1.11.1.tgz#7960a668888594a0720b12a911d1a742ab9f11d2" + integrity sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA== + dependencies: + lru-cache "^10.2.0" + minipass "^5.0.0 || ^6.0.2 || ^7.0.0" + +pend@~1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/pend/-/pend-1.2.0.tgz#7a57eb550a6783f9115331fcf4663d5c8e007a50" + integrity sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg== + +progress@2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8" + integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA== + +pump@^3.0.0: + version "3.0.2" + resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.2.tgz#836f3edd6bc2ee599256c924ffe0d88573ddcbf8" + integrity sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw== + dependencies: + end-of-stream "^1.1.0" + once "^1.3.1" + +rimraf@4.4.1: + version "4.4.1" + resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-4.4.1.tgz#bd33364f67021c5b79e93d7f4fa0568c7c21b755" + integrity sha512-Gk8NlF062+T9CqNGn6h4tls3k6T1+/nXdOcSZVikNVtlRdYpA7wRJJMoXmuvOnLW844rPjdQ7JgXCYM6PPC/og== + dependencies: + glob "^9.2.0" + +supports-color@^7.1.0: + version "7.2.0" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-7.2.0.tgz#1b7dcdcb32b8138801b3e478ba6a51caa89648da" + integrity sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw== + dependencies: + has-flag "^4.0.0" + +undici-types@~6.19.2: + version "6.19.8" + resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-6.19.8.tgz#35111c9d1437ab83a7cdc0abae2f26d88eda0a02" + integrity sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw== + +undici@^5.13.0: + version "5.28.4" + resolved "https://registry.yarnpkg.com/undici/-/undici-5.28.4.tgz#6b280408edb6a1a604a9b20340f45b422e373068" + integrity sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g== + dependencies: + "@fastify/busboy" "^2.0.0" + +wrappy@1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" + integrity sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ== + +yauzl@^2.10.0: + version "2.10.0" + resolved "https://registry.yarnpkg.com/yauzl/-/yauzl-2.10.0.tgz#c7eb17c93e112cb1086fa6d8e51fb0667b79a5f9" + integrity sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g== + dependencies: + buffer-crc32 "~0.2.3" + fd-slicer "~1.1.0" diff --git a/packages/datadog-plugin-bunyan/test/integration-test/client.spec.js b/packages/datadog-plugin-bunyan/test/integration-test/client.spec.js index 9f6688a0354..5ce5d8884fa 100644 --- a/packages/datadog-plugin-bunyan/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-bunyan/test/integration-test/client.spec.js @@ -15,7 +15,7 @@ describe('esm', () => { before(async function () { this.timeout(20000) sandbox = await createSandbox([`'bunyan@${version}'`], false, - [`./packages/datadog-plugin-bunyan/test/integration-test/*`]) + ['./packages/datadog-plugin-bunyan/test/integration-test/*']) }) after(async () => { diff --git a/packages/datadog-plugin-cassandra-driver/test/index.spec.js b/packages/datadog-plugin-cassandra-driver/test/index.spec.js index ddca9a999a7..d10433a04b5 100644 --- a/packages/datadog-plugin-cassandra-driver/test/index.spec.js +++ b/packages/datadog-plugin-cassandra-driver/test/index.spec.js @@ -146,7 +146,7 @@ describe('Plugin', () => { const childOf = tracer.startSpan('test') scope.activate(childOf, () => { - client.batch([`UPDATE test.test SET test='test' WHERE id='1234';`], () => { + client.batch(['UPDATE test.test SET test=\'test\' WHERE id=\'1234\';'], () => { expect(tracer.scope().active()).to.equal(childOf) done() }) diff --git a/packages/datadog-plugin-cassandra-driver/test/integration-test/client.spec.js b/packages/datadog-plugin-cassandra-driver/test/integration-test/client.spec.js index 4b58ba299a1..da680b7fa25 100644 --- a/packages/datadog-plugin-cassandra-driver/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-cassandra-driver/test/integration-test/client.spec.js @@ -22,7 +22,7 @@ describe('esm', () => { before(async function () { this.timeout(20000) sandbox = await createSandbox([`'cassandra-driver@${version}'`], false, [ - `./packages/datadog-plugin-cassandra-driver/test/integration-test/*`]) + './packages/datadog-plugin-cassandra-driver/test/integration-test/*']) }) after(async () => { diff --git a/packages/datadog-plugin-cassandra-driver/test/naming.js b/packages/datadog-plugin-cassandra-driver/test/naming.js index a4a3929ecb5..10d93008b8a 100644 --- a/packages/datadog-plugin-cassandra-driver/test/naming.js +++ b/packages/datadog-plugin-cassandra-driver/test/naming.js @@ -14,6 +14,6 @@ const rawExpectedSchema = { } module.exports = { - rawExpectedSchema: rawExpectedSchema, + rawExpectedSchema, expectedSchema: resolveNaming(rawExpectedSchema) } diff --git a/packages/datadog-plugin-child_process/src/index.js b/packages/datadog-plugin-child_process/src/index.js new file mode 100644 index 00000000000..7209f44b97a --- /dev/null +++ b/packages/datadog-plugin-child_process/src/index.js @@ -0,0 +1,91 @@ +'use strict' + +const TracingPlugin = require('../../dd-trace/src/plugins/tracing') +const scrubChildProcessCmd = require('./scrub-cmd-params') + +const MAX_ARG_SIZE = 4096 // 4kB + +function truncateCommand (cmdFields) { + let size = cmdFields[0].length + let truncated = false + for (let i = 1; i < cmdFields.length; i++) { + if (size >= MAX_ARG_SIZE) { + truncated = true + cmdFields[i] = '' + continue + } + + const argLen = cmdFields[i].length + if (size < MAX_ARG_SIZE && size + argLen > MAX_ARG_SIZE) { + cmdFields[i] = cmdFields[i].substring(0, 2) + truncated = true + } + + size += argLen + } + + return truncated +} + +class ChildProcessPlugin extends TracingPlugin { + static get id () { return 'child_process' } + static get prefix () { return 'tracing:datadog:child_process:execution' } + + get tracer () { + return this._tracer + } + + start ({ command, shell }) { + if (typeof command !== 'string') { + return + } + + const cmdFields = scrubChildProcessCmd(command) + const truncated = truncateCommand(cmdFields) + const property = (shell === true) ? 'cmd.shell' : 'cmd.exec' + + const meta = { + component: 'subprocess', + [property]: (shell === true) ? cmdFields.join(' ') : JSON.stringify(cmdFields) + } + + if (truncated) { + meta['cmd.truncated'] = `${truncated}` + } + + this.startSpan('command_execution', { + service: this.config.service || this._tracerConfig.service, + resource: (shell === true) ? 'sh' : cmdFields[0], + type: 'system', + meta + }) + } + + end ({ result, error }) { + let exitCode + + if (result !== undefined) { + exitCode = result?.status || 0 + } else if (error !== undefined) { + exitCode = error?.status || error?.code || 0 + } else { + // TracingChannels call start, end synchronously. Later when the promise is resolved then asyncStart asyncEnd. + // Therefore in the case of calling end with neither result nor error means that they will come in the asyncEnd. + return + } + + this.activeSpan?.setTag('cmd.exit_code', `${exitCode}`) + this.activeSpan?.finish() + } + + error (error) { + this.addError(error) + } + + asyncEnd ({ result }) { + this.activeSpan?.setTag('cmd.exit_code', `${result}`) + this.activeSpan?.finish() + } +} + +module.exports = ChildProcessPlugin diff --git a/packages/datadog-plugin-child_process/src/scrub-cmd-params.js b/packages/datadog-plugin-child_process/src/scrub-cmd-params.js new file mode 100644 index 00000000000..b5fb59bb781 --- /dev/null +++ b/packages/datadog-plugin-child_process/src/scrub-cmd-params.js @@ -0,0 +1,127 @@ +'use strict' + +const shellParser = require('shell-quote/parse') + +const ALLOWED_ENV_VARIABLES = ['LD_PRELOAD', 'LD_LIBRARY_PATH', 'PATH'] +const PROCESS_DENYLIST = ['md5'] + +const VARNAMES_REGEX = /\$([\w\d_]*)(?:[^\w\d_]|$)/gmi +// eslint-disable-next-line max-len +const PARAM_PATTERN = '^-{0,2}(?:p(?:ass(?:w(?:or)?d)?)?|address|api[-_]?key|e?mail|secret(?:[-_]?key)?|a(?:ccess|uth)[-_]?token|mysql_pwd|credentials|(?:stripe)?token)$' +const regexParam = new RegExp(PARAM_PATTERN, 'i') +const ENV_PATTERN = '^(\\w+=\\w+;)*\\w+=\\w+;?$' +const envVarRegex = new RegExp(ENV_PATTERN) +const REDACTED = '?' + +function extractVarNames (expression) { + const varNames = new Set() + let match + + while ((match = VARNAMES_REGEX.exec(expression))) { + varNames.add(match[1]) + } + + const varNamesObject = {} + for (const varName of varNames.keys()) { + varNamesObject[varName] = `$${varName}` + } + return varNamesObject +} + +function getTokensByExpression (expressionTokens) { + const expressionListTokens = [] + let wipExpressionTokens = [] + let isNewExpression = true + + expressionTokens.forEach(token => { + if (isNewExpression) { + expressionListTokens.push(wipExpressionTokens) + isNewExpression = false + } + + wipExpressionTokens.push(token) + + if (token.op) { + wipExpressionTokens = [] + isNewExpression = true + } + }) + return expressionListTokens +} + +function scrubChildProcessCmd (expression) { + const varNames = extractVarNames(expression) + const expressionTokens = shellParser(expression, varNames) + + const expressionListTokens = getTokensByExpression(expressionTokens) + + const result = [] + expressionListTokens.forEach((expressionTokens) => { + let foundBinary = false + for (let index = 0; index < expressionTokens.length; index++) { + const token = expressionTokens[index] + + if (token === null) { + continue + } else if (typeof token === 'object') { + if (token.pattern) { + result.push(token.pattern) + } else if (token.op) { + result.push(token.op) + } else if (token.comment) { + result.push(`#${token.comment}`) + } + } else if (!foundBinary) { + if (envVarRegex.test(token)) { + const envSplit = token.split('=') + + if (!ALLOWED_ENV_VARIABLES.includes(envSplit[0])) { + envSplit[1] = REDACTED + + const newToken = envSplit.join('=') + expressionTokens[index] = newToken + + result.push(newToken) + } else { + result.push(token) + } + } else { + foundBinary = true + result.push(token) + + if (PROCESS_DENYLIST.includes(token)) { + for (index++; index < expressionTokens.length; index++) { + const token = expressionTokens[index] + + if (token.op) { + result.push(token.op) + } else { + expressionTokens[index] = REDACTED + result.push(REDACTED) + } + } + break + } + } + } else { + const paramKeyValue = token.split('=') + const paramKey = paramKeyValue[0] + + if (regexParam.test(paramKey)) { + if (paramKeyValue.length === 1) { + expressionTokens[index + 1] = REDACTED + result.push(token) + } else { + result.push(`${paramKey}=${REDACTED}`) + } + } else { + result.push(token) + } + } + } + }) + + return result +} + +module.exports = scrubChildProcessCmd diff --git a/packages/datadog-plugin-child_process/test/index.spec.js b/packages/datadog-plugin-child_process/test/index.spec.js new file mode 100644 index 00000000000..33624eab4d8 --- /dev/null +++ b/packages/datadog-plugin-child_process/test/index.spec.js @@ -0,0 +1,691 @@ +'use strict' + +const ChildProcessPlugin = require('../src') +const { storage } = require('../../datadog-core') +const agent = require('../../dd-trace/test/plugins/agent') +const { expectSomeSpan } = require('../../dd-trace/test/plugins/helpers') +const { NODE_MAJOR } = require('../../../version') + +function noop () {} + +function normalizeArgs (methodName, command, options) { + const args = [] + if (methodName === 'exec' || methodName === 'execSync') { + args.push(command.join(' ')) + } else { + args.push(command[0], command.slice(1)) + } + + args.push(options) + + return args +} + +describe('Child process plugin', () => { + describe('unit tests', () => { + let tracerStub, configStub, spanStub + + beforeEach(() => { + spanStub = { + setTag: sinon.stub(), + finish: sinon.stub() + } + + tracerStub = { + startSpan: sinon.stub() + } + + configStub = { + service: 'test-service' + } + }) + + afterEach(() => { + sinon.restore() + }) + + describe('start', () => { + it('should call startSpan with proper parameters', () => { + const shellPlugin = new ChildProcessPlugin(tracerStub, configStub) + + shellPlugin.start({ command: 'ls -l' }) + + expect(tracerStub.startSpan).to.have.been.calledOnceWithExactly( + 'command_execution', + { + childOf: undefined, + tags: { + component: 'subprocess', + 'service.name': 'test-service', + 'resource.name': 'ls', + 'span.kind': undefined, + 'span.type': 'system', + 'cmd.exec': JSON.stringify(['ls', '-l']) + }, + integrationName: 'system' + } + ) + }) + + it('should call startSpan with cmd.shell property', () => { + const shellPlugin = new ChildProcessPlugin(tracerStub, configStub) + + shellPlugin.start({ command: 'ls -l', shell: true }) + + expect(tracerStub.startSpan).to.have.been.calledOnceWithExactly( + 'command_execution', + { + childOf: undefined, + tags: { + component: 'subprocess', + 'service.name': 'test-service', + 'resource.name': 'sh', + 'span.kind': undefined, + 'span.type': 'system', + 'cmd.shell': 'ls -l' + }, + integrationName: 'system' + } + ) + }) + + it('should truncate last argument', () => { + const shellPlugin = new ChildProcessPlugin(tracerStub, configStub) + const arg = 'a'.padEnd(4092, 'a') + const command = 'echo' + ' ' + arg + ' arg2' + + shellPlugin.start({ command }) + + expect(tracerStub.startSpan).to.have.been.calledOnceWithExactly( + 'command_execution', + { + childOf: undefined, + tags: { + component: 'subprocess', + 'service.name': 'test-service', + 'resource.name': 'echo', + 'span.kind': undefined, + 'span.type': 'system', + 'cmd.exec': JSON.stringify(['echo', arg, '']), + 'cmd.truncated': 'true' + }, + integrationName: 'system' + } + ) + }) + + it('should truncate path and blank last argument', () => { + const shellPlugin = new ChildProcessPlugin(tracerStub, configStub) + const path = '/home/'.padEnd(4096, '/') + const command = 'ls -l' + ' ' + path + ' -t' + + shellPlugin.start({ command, shell: true }) + + expect(tracerStub.startSpan).to.have.been.calledOnceWithExactly( + 'command_execution', + { + childOf: undefined, + tags: { + component: 'subprocess', + 'service.name': 'test-service', + 'resource.name': 'sh', + 'span.kind': undefined, + 'span.type': 'system', + 'cmd.shell': 'ls -l /h ', + 'cmd.truncated': 'true' + }, + integrationName: 'system' + } + ) + }) + + it('should truncate first argument and blank the rest', () => { + const shellPlugin = new ChildProcessPlugin(tracerStub, configStub) + const option = '-l'.padEnd(4096, 't') + const path = '/home' + const command = `ls ${option} ${path} -t` + + shellPlugin.start({ command }) + + expect(tracerStub.startSpan).to.have.been.calledOnceWithExactly( + 'command_execution', + { + childOf: undefined, + tags: { + component: 'subprocess', + 'service.name': 'test-service', + 'resource.name': 'ls', + 'span.kind': undefined, + 'span.type': 'system', + 'cmd.exec': JSON.stringify(['ls', '-l', '', '']), + 'cmd.truncated': 'true' + }, + integrationName: 'system' + } + ) + }) + + it('should truncate last argument', () => { + const shellPlugin = new ChildProcessPlugin(tracerStub, configStub) + const option = '-t'.padEnd(4000 * 8, 'u') + const path = '/home' + const command = 'ls' + ' -l' + ' ' + path + ' ' + option + + shellPlugin.start({ command, shell: true }) + + expect(tracerStub.startSpan).to.have.been.calledOnceWithExactly( + 'command_execution', + { + childOf: undefined, + tags: { + component: 'subprocess', + 'service.name': 'test-service', + 'resource.name': 'sh', + 'span.kind': undefined, + 'span.type': 'system', + 'cmd.shell': 'ls -l /home -t', + 'cmd.truncated': 'true' + }, + integrationName: 'system' + } + ) + }) + + it('should not crash if command is not a string', () => { + const shellPlugin = new ChildProcessPlugin(tracerStub, configStub) + + shellPlugin.start({ command: undefined }) + + expect(tracerStub.startSpan).not.to.have.been.called + }) + + it('should not crash if command does not exist', () => { + const shellPlugin = new ChildProcessPlugin(tracerStub, configStub) + + shellPlugin.start({}) + + expect(tracerStub.startSpan).not.to.have.been.called + }) + }) + + describe('end', () => { + it('should not call setTag if neither error nor result is passed', () => { + sinon.stub(storage, 'getStore').returns({ span: spanStub }) + const shellPlugin = new ChildProcessPlugin(tracerStub, configStub) + + shellPlugin.end({}) + + expect(spanStub.setTag).not.to.have.been.called + expect(spanStub.finish).not.to.have.been.called + }) + + it('should call setTag with proper code when result is a buffer', () => { + sinon.stub(storage, 'getStore').returns({ span: spanStub }) + const shellPlugin = new ChildProcessPlugin(tracerStub, configStub) + + shellPlugin.end({ result: Buffer.from('test') }) + + expect(spanStub.setTag).to.have.been.calledOnceWithExactly('cmd.exit_code', '0') + expect(spanStub.finish).to.have.been.calledOnceWithExactly() + }) + + it('should call setTag with proper code when result is a string', () => { + sinon.stub(storage, 'getStore').returns({ span: spanStub }) + const shellPlugin = new ChildProcessPlugin(tracerStub, configStub) + + shellPlugin.end({ result: 'test' }) + + expect(spanStub.setTag).to.have.been.calledOnceWithExactly('cmd.exit_code', '0') + expect(spanStub.finish).to.have.been.calledOnceWithExactly() + }) + + it('should call setTag with proper code when an error is thrown', () => { + sinon.stub(storage, 'getStore').returns({ span: spanStub }) + const shellPlugin = new ChildProcessPlugin(tracerStub, configStub) + + shellPlugin.end({ error: { status: -1 } }) + + expect(spanStub.setTag).to.have.been.calledOnceWithExactly('cmd.exit_code', '-1') + expect(spanStub.finish).to.have.been.calledOnceWithExactly() + }) + }) + + describe('asyncEnd', () => { + it('should call setTag with undefined code if neither error nor result is passed', () => { + sinon.stub(storage, 'getStore').returns({ span: spanStub }) + const shellPlugin = new ChildProcessPlugin(tracerStub, configStub) + + shellPlugin.asyncEnd({}) + + expect(spanStub.setTag).to.have.been.calledOnceWithExactly('cmd.exit_code', 'undefined') + expect(spanStub.finish).to.have.been.calledOnce + }) + + it('should call setTag with proper code when a proper code is returned', () => { + sinon.stub(storage, 'getStore').returns({ span: spanStub }) + const shellPlugin = new ChildProcessPlugin(tracerStub, configStub) + + shellPlugin.asyncEnd({ result: 0 }) + + expect(spanStub.setTag).to.have.been.calledOnceWithExactly('cmd.exit_code', '0') + expect(spanStub.finish).to.have.been.calledOnceWithExactly() + }) + }) + + describe('channel', () => { + it('should return proper prefix', () => { + expect(ChildProcessPlugin.prefix).to.be.equal('tracing:datadog:child_process:execution') + }) + + it('should return proper id', () => { + expect(ChildProcessPlugin.id).to.be.equal('child_process') + }) + }) + }) + + describe('context maintenance', () => { + let parent + let childProcess + let tracer + + before(() => { + return agent.load(['child_process']) + .then(() => { + childProcess = require('child_process') + tracer = require('../../dd-trace') + tracer.init() + parent = tracer.startSpan('parent') + parent.finish() + }).then(_port => { + return new Promise(resolve => setImmediate(resolve)) + }) + }) + + after(() => { + return agent.close() + }) + + it('should preserve context around execSync calls', () => { + tracer.scope().activate(parent, () => { + expect(tracer.scope().active()).to.equal(parent) + childProcess.execSync('ls') + expect(tracer.scope().active()).to.equal(parent) + }) + }) + + it('should preserve context around exec calls', (done) => { + tracer.scope().activate(parent, () => { + expect(tracer.scope().active()).to.equal(parent) + childProcess.exec('ls', () => { + expect(tracer.scope().active()).to.equal(parent) + done() + }) + }) + }) + + it('should preserve context around execFileSync calls', () => { + tracer.scope().activate(parent, () => { + expect(tracer.scope().active()).to.equal(parent) + childProcess.execFileSync('ls') + expect(tracer.scope().active()).to.equal(parent) + }) + }) + + it('should preserve context around execFile calls', (done) => { + tracer.scope().activate(parent, () => { + expect(tracer.scope().active()).to.equal(parent) + childProcess.execFile('ls', () => { + expect(tracer.scope().active()).to.equal(parent) + done() + }) + }) + }) + + it('should preserve context around spawnSync calls', () => { + tracer.scope().activate(parent, () => { + expect(tracer.scope().active()).to.equal(parent) + childProcess.spawnSync('ls') + expect(tracer.scope().active()).to.equal(parent) + }) + }) + + it('should preserve context around spawn calls', (done) => { + tracer.scope().activate(parent, () => { + expect(tracer.scope().active()).to.equal(parent) + childProcess.spawn('ls') + expect(tracer.scope().active()).to.equal(parent) + done() + }) + }) + }) + + describe('Integration', () => { + describe('Methods which spawn a shell by default', () => { + const execAsyncMethods = ['exec'] + const execSyncMethods = ['execSync'] + let childProcess, tracer + + beforeEach(() => { + return agent.load('child_process', undefined, { flushInterval: 1 }).then(() => { + tracer = require('../../dd-trace') + childProcess = require('child_process') + tracer.use('child_process', { enabled: true }) + }) + }) + + afterEach(() => agent.close({ ritmReset: false })) + const parentSpanList = [true, false] + parentSpanList.forEach(hasParentSpan => { + let parentSpan + + describe(`${hasParentSpan ? 'with' : 'without'} parent span`, () => { + const methods = [ + ...execAsyncMethods.map(methodName => ({ methodName, async: true })), + ...execSyncMethods.map(methodName => ({ methodName, async: false })) + ] + + beforeEach((done) => { + if (hasParentSpan) { + parentSpan = tracer.startSpan('parent') + parentSpan.finish() + tracer.scope().activate(parentSpan, done) + } else { + storage.enterWith({}) + done() + } + }) + + methods.forEach(({ methodName, async }) => { + describe(methodName, () => { + it('should be instrumented', (done) => { + const expected = { + type: 'system', + name: 'command_execution', + error: 0, + meta: { + component: 'subprocess', + 'cmd.shell': 'ls', + 'cmd.exit_code': '0' + } + } + + expectSomeSpan(agent, expected).then(done, done) + + const res = childProcess[methodName]('ls') + if (async) { + res.on('close', noop) + } + }) + + it('should maintain previous span after the execution', (done) => { + const res = childProcess[methodName]('ls') + const span = storage.getStore()?.span + expect(span).to.be.equals(parentSpan) + if (async) { + res.on('close', () => { + expect(span).to.be.equals(parentSpan) + done() + }) + } else { + done() + } + }) + + if (async) { + it('should maintain previous span in the callback', (done) => { + childProcess[methodName]('ls', () => { + const span = storage.getStore()?.span + expect(span).to.be.equals(parentSpan) + done() + }) + }) + } + + it('command should be scrubbed', (done) => { + const expected = { + type: 'system', + name: 'command_execution', + error: 0, + meta: { + component: 'subprocess', + 'cmd.shell': 'echo password ?', + 'cmd.exit_code': '0' + } + } + expectSomeSpan(agent, expected).then(done, done) + + const args = [] + if (methodName === 'exec' || methodName === 'execSync') { + args.push('echo password 123') + } else { + args.push('echo') + args.push(['password', '123']) + } + + const res = childProcess[methodName](...args) + if (async) { + res.on('close', noop) + } + }) + + it('should be instrumented with error code', (done) => { + const command = ['node', '-badOption'] + const options = { + stdio: 'pipe' + } + const expected = { + type: 'system', + name: 'command_execution', + error: 1, + meta: { + component: 'subprocess', + 'cmd.shell': 'node -badOption', + 'cmd.exit_code': '9' + } + } + + expectSomeSpan(agent, expected).then(done, done) + + const args = normalizeArgs(methodName, command, options) + + if (async) { + const res = childProcess[methodName].apply(null, args) + res.on('close', noop) + } else { + try { + childProcess[methodName].apply(null, args) + } catch { + // process exit with code 1, exceptions are expected + } + } + }) + }) + }) + }) + }) + }) + + describe('Methods which do not spawn a shell by default', () => { + const execAsyncMethods = ['execFile', 'spawn'] + const execSyncMethods = ['execFileSync', 'spawnSync'] + let childProcess, tracer + + beforeEach(() => { + return agent.load('child_process', undefined, { flushInterval: 1 }).then(() => { + tracer = require('../../dd-trace') + childProcess = require('child_process') + tracer.use('child_process', { enabled: true }) + }) + }) + + afterEach(() => agent.close({ ritmReset: false })) + const parentSpanList = [true, false] + parentSpanList.forEach(parentSpan => { + describe(`${parentSpan ? 'with' : 'without'} parent span`, () => { + const methods = [ + ...execAsyncMethods.map(methodName => ({ methodName, async: true })), + ...execSyncMethods.map(methodName => ({ methodName, async: false })) + ] + if (parentSpan) { + beforeEach((done) => { + const parentSpan = tracer.startSpan('parent') + parentSpan.finish() + tracer.scope().activate(parentSpan, done) + }) + } + + methods.forEach(({ methodName, async }) => { + describe(methodName, () => { + it('should be instrumented', (done) => { + const expected = { + type: 'system', + name: 'command_execution', + error: 0, + meta: { + component: 'subprocess', + 'cmd.exec': '["ls"]', + 'cmd.exit_code': '0' + } + } + expectSomeSpan(agent, expected).then(done, done) + + const res = childProcess[methodName]('ls') + if (async) { + res.on('close', noop) + } + }) + + it('command should be scrubbed', (done) => { + const expected = { + type: 'system', + name: 'command_execution', + error: 0, + meta: { + component: 'subprocess', + 'cmd.exec': '["echo","password","?"]', + 'cmd.exit_code': '0' + } + } + expectSomeSpan(agent, expected).then(done, done) + + const args = [] + if (methodName === 'exec' || methodName === 'execSync') { + args.push('echo password 123') + } else { + args.push('echo') + args.push(['password', '123']) + } + + const res = childProcess[methodName](...args) + if (async) { + res.on('close', noop) + } + }) + + it('should be instrumented with error code', (done) => { + const command = ['node', '-badOption'] + const options = { + stdio: 'pipe' + } + + const errorExpected = { + type: 'system', + name: 'command_execution', + error: 1, + meta: { + component: 'subprocess', + 'cmd.exec': '["node","-badOption"]', + 'cmd.exit_code': '9' + } + } + + const noErrorExpected = { + type: 'system', + name: 'command_execution', + error: 0, + meta: { + component: 'subprocess', + 'cmd.exec': '["node","-badOption"]', + 'cmd.exit_code': '9' + } + } + + const args = normalizeArgs(methodName, command, options) + + if (async) { + expectSomeSpan(agent, errorExpected).then(done, done) + const res = childProcess[methodName].apply(null, args) + res.on('close', noop) + } else { + try { + if (methodName === 'spawnSync') { + expectSomeSpan(agent, noErrorExpected).then(done, done) + } else { + expectSomeSpan(agent, errorExpected).then(done, done) + } + childProcess[methodName].apply(null, args) + } catch { + // process exit with code 1, exceptions are expected + } + } + }) + + if (methodName !== 'execFileSync' || NODE_MAJOR > 16) { + // when a process return an invalid code, in node <=16, in execFileSync with shell:true + // an exception is not thrown + it('should be instrumented with error code (override shell default behavior)', (done) => { + const command = ['node', '-badOption'] + const options = { + stdio: 'pipe', + shell: true + } + + const errorExpected = { + type: 'system', + name: 'command_execution', + error: 1, + meta: { + component: 'subprocess', + 'cmd.shell': 'node -badOption', + 'cmd.exit_code': '9' + } + } + + const noErrorExpected = { + type: 'system', + name: 'command_execution', + error: 0, + meta: { + component: 'subprocess', + 'cmd.shell': 'node -badOption', + 'cmd.exit_code': '9' + } + } + + const args = normalizeArgs(methodName, command, options) + + if (async) { + expectSomeSpan(agent, errorExpected).then(done, done) + const res = childProcess[methodName].apply(null, args) + res.on('close', noop) + } else { + try { + if (methodName === 'spawnSync') { + expectSomeSpan(agent, noErrorExpected).then(done, done) + } else { + expectSomeSpan(agent, errorExpected).then(done, done) + } + childProcess[methodName].apply(null, args) + } catch { + // process exit with code 1, exceptions are expected + } + } + }) + } + }) + }) + }) + }) + }) + }) +}) diff --git a/packages/datadog-plugin-child_process/test/scrub-cmd-params.spec.js b/packages/datadog-plugin-child_process/test/scrub-cmd-params.spec.js new file mode 100644 index 00000000000..1032d3e54c1 --- /dev/null +++ b/packages/datadog-plugin-child_process/test/scrub-cmd-params.spec.js @@ -0,0 +1,82 @@ +'use strict' + +const scrubCmdParams = require('../src/scrub-cmd-params') + +describe('scrub cmds', () => { + it('Should not scrub single command', () => { + expect(scrubCmdParams('ls -la')).to.be.deep.equal(['ls', '-la']) + }) + + it('Should split correctly comments', () => { + expect(scrubCmdParams('ls #comment')).to.be.deep.equal(['ls', '#comment']) + expect(scrubCmdParams('ls #comment with spaces')).to.be.deep.equal(['ls', '#comment with spaces']) + }) + + it('Should split globs', () => { + expect(scrubCmdParams('ls node_modules/*')).to.be.deep.equal(['ls', 'node_modules/*']) + expect(scrubCmdParams('ls *')).to.be.deep.equal(['ls', '*']) + }) + + it('Should split correctly texts', () => { + expect(scrubCmdParams('echo "Hello\\ text"')).to.be.deep.equal(['echo', 'Hello\\ text']) + expect(scrubCmdParams('node -e "process.exit(1)"')).to.be.deep.equal(['node', '-e', 'process.exit(1)']) + }) + + it('Should not scrub chained command', () => { + expect(scrubCmdParams('ls -la|grep something')).to.be.deep.equal(['ls', '-la', '|', 'grep', 'something']) + }) + + it('Should scrub environment variables', () => { + expect(scrubCmdParams('ENV=XXX LD_PRELOAD=YYY ls')).to.be.deep.equal(['ENV=?', 'LD_PRELOAD=YYY', 'ls']) + expect(scrubCmdParams('DD_TEST=info SHELL=zsh ls -l')).to.be.deep.equal(['DD_TEST=?', 'SHELL=?', 'ls', '-l']) + }) + + it('Should scrub secret values', () => { + expect(scrubCmdParams('cmd --pass abc --token=def')).to.be.deep.equal(['cmd', '--pass', '?', '--token=?']) + + expect(scrubCmdParams('mysqladmin -u root password very_secret')) + .to.be.deep.equal(['mysqladmin', '-u', 'root', 'password', '?']) + + expect(scrubCmdParams('test -password very_secret -api_key 1234')) + .to.be.deep.equal(['test', '-password', '?', '-api_key', '?']) + + expect(scrubCmdParams('test --address https://some.address.com --email testing@to.es --api-key 1234')) + .to.be.deep.equal(['test', '--address', '?', '--email', '?', '--api-key', '?']) + }) + + it('Should scrub md5 commands', () => { + expect(scrubCmdParams('md5 -s pony')).to.be.deep.equal(['md5', '?', '?']) + + expect(scrubCmdParams('cat passwords.txt | while read line; do; md5 -s $line; done')).to.be.deep + .equal([ + 'cat', + 'passwords.txt', + '|', + 'while', + 'read', + 'line', + ';', + 'do', + ';', + 'md5', + '?', + '?', + ';', + 'done' + ]) + }) + + it('should scrub shell expressions', () => { + expect(scrubCmdParams('md5 -s secret ; mysqladmin -u root password 1234 | test api_key 4321')).to.be.deep.equal([ + 'md5', '?', '?', ';', 'mysqladmin', '-u', 'root', 'password', '?', '|', 'test', 'api_key', '?' + ]) + }) + + it('Should not scrub md5sum commands', () => { + expect(scrubCmdParams('md5sum file')).to.be.deep.equal(['md5sum', 'file']) + }) + + it('Should maintain var names', () => { + expect(scrubCmdParams('echo $something')).to.be.deep.equal(['echo', '$something']) + }) +}) diff --git a/packages/datadog-plugin-connect/test/index.spec.js b/packages/datadog-plugin-connect/test/index.spec.js index 239ab4c09c4..62b64bcc8a7 100644 --- a/packages/datadog-plugin-connect/test/index.spec.js +++ b/packages/datadog-plugin-connect/test/index.spec.js @@ -2,7 +2,6 @@ const axios = require('axios') const http = require('http') -const getPort = require('get-port') const agent = require('../../dd-trace/test/plugins/agent') const { AsyncLocalStorage } = require('async_hooks') const { ERROR_MESSAGE, ERROR_STACK, ERROR_TYPE } = require('../../dd-trace/src/constants') @@ -45,7 +44,9 @@ describe('Plugin', () => { res.end() }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -62,11 +63,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -76,7 +75,9 @@ describe('Plugin', () => { app.use(function named (req, res, next) { next() }) app.use('/app/user', (req, res) => res.end()) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -94,11 +95,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/1`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app/user/1`) + .catch(done) }) }) @@ -109,7 +108,9 @@ describe('Plugin', () => { app.use('/foo/bar', (req, res, next) => next()) app.use('/foo', (req, res) => res.end()) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -119,11 +120,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/foo/bar`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/foo/bar`) + .catch(done) }) }) @@ -137,7 +136,9 @@ describe('Plugin', () => { app.use('/parent', childApp) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -148,11 +149,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/parent/child`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/parent/child`) + .catch(done) }) }) @@ -175,12 +174,12 @@ describe('Plugin', () => { done() }) - getPort().then(port => { - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/1`) - .catch(done) - }) + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/app/user/1`) + .catch(done) }) }) @@ -196,7 +195,9 @@ describe('Plugin', () => { res.end() }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -206,11 +207,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app`) + .catch(done) }) }) @@ -239,12 +238,12 @@ describe('Plugin', () => { res.end() }) - getPort().then(port => { - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app`) - .catch(done) - }) + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/app`) + .catch(done) }) }) @@ -253,7 +252,9 @@ describe('Plugin', () => { app.use((req, res, next) => res.end()) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -263,11 +264,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app`) + .catch(done) }) }) @@ -293,11 +292,11 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/user`) - .catch(done) - }) + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + + axios.get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -308,7 +307,9 @@ describe('Plugin', () => { app.use('/app', (req, res) => res.end()) app.use('/bar', (req, res, next) => next()) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -318,10 +319,8 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/app/user/123`) - .catch(done) - }) + axios.get(`http://localhost:${port}/app/user/123`) + .catch(done) }) }) @@ -332,7 +331,9 @@ describe('Plugin', () => { res.end() }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent.use(traces => { const spans = sort(traces[0]) @@ -342,17 +343,15 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - headers: { - 'x-datadog-trace-id': '1234', - 'x-datadog-parent-id': '5678', - 'ot-baggage-foo': 'bar' - } - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + headers: { + 'x-datadog-trace-id': '1234', + 'x-datadog-parent-id': '5678', + 'ot-baggage-foo': 'bar' + } + }) + .catch(done) }) }) @@ -368,7 +367,9 @@ describe('Plugin', () => { res.end() }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -381,13 +382,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) @@ -403,7 +402,9 @@ describe('Plugin', () => { throw new Error('boom') }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -416,13 +417,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 400 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 400 + }) + .catch(done) }) }) @@ -432,7 +431,9 @@ describe('Plugin', () => { app.use(() => { throw error }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -447,13 +448,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) @@ -477,12 +476,12 @@ describe('Plugin', () => { res.end() }) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -491,12 +490,15 @@ describe('Plugin', () => { const error = new Error('boom') app.use((req, res) => { throw error }) + // eslint-disable-next-line n/handle-callback-err app.use((error, req, res, next) => { res.statusCode = 500 res.end() }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -515,13 +517,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) }) @@ -550,7 +550,9 @@ describe('Plugin', () => { res.end() }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -560,11 +562,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -576,7 +576,9 @@ describe('Plugin', () => { res.end() }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -586,13 +588,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 400 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 400 + }) + .catch(done) }) }) @@ -603,7 +603,9 @@ describe('Plugin', () => { res.end() }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -613,13 +615,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - headers: { 'User-Agent': 'test' } - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + headers: { 'User-Agent': 'test' } + }) + .catch(done) }) }) @@ -631,7 +631,9 @@ describe('Plugin', () => { res.end() }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -648,11 +650,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -661,12 +661,15 @@ describe('Plugin', () => { const error = new Error('boom') app.use((req, res) => { throw error }) + // eslint-disable-next-line n/handle-callback-err app.use((error, req, res, next) => { res.statusCode = 500 res.end() }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -685,13 +688,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) @@ -701,7 +702,9 @@ describe('Plugin', () => { app.use(() => { throw error }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -716,13 +719,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) }) @@ -748,7 +749,9 @@ describe('Plugin', () => { app.use(function named (req, res, next) { next() }) app.use('/app/user', (req, res) => res.end()) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -760,11 +763,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/1`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app/user/1`) + .catch(done) }) }) @@ -789,11 +790,11 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/user`) - .catch(done) - }) + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + + axios.get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -802,12 +803,15 @@ describe('Plugin', () => { const error = new Error('boom') app.use((req, res) => { throw error }) + // eslint-disable-next-line n/handle-callback-err app.use((error, req, res, next) => { res.statusCode = 500 res.end() }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -822,13 +826,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) @@ -838,7 +840,9 @@ describe('Plugin', () => { app.use(() => { throw error }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -853,13 +857,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) }) diff --git a/packages/datadog-plugin-connect/test/integration-test/client.spec.js b/packages/datadog-plugin-connect/test/integration-test/client.spec.js index 78fe5cbcec7..a045ba8bf09 100644 --- a/packages/datadog-plugin-connect/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-connect/test/integration-test/client.spec.js @@ -19,7 +19,7 @@ describe('esm', () => { before(async function () { this.timeout(20000) sandbox = await createSandbox([`'connect@${version}'`], false, [ - `./packages/datadog-plugin-connect/test/integration-test/*`]) + './packages/datadog-plugin-connect/test/integration-test/*']) }) after(async () => { diff --git a/packages/datadog-plugin-couchbase/src/index.js b/packages/datadog-plugin-couchbase/src/index.js index da6ec18ba4b..cb764875de7 100644 --- a/packages/datadog-plugin-couchbase/src/index.js +++ b/packages/datadog-plugin-couchbase/src/index.js @@ -16,7 +16,7 @@ class CouchBasePlugin extends StoragePlugin { startSpan (operation, customTags, store, { bucket, collection, seedNodes }) { const tags = { 'db.type': 'couchbase', - 'component': 'couchbase', + component: 'couchbase', 'resource.name': `couchbase.${operation}`, 'span.kind': this.constructor.kind, 'db.couchbase.seed.nodes': seedNodes @@ -61,6 +61,7 @@ class CouchBasePlugin extends StoragePlugin { this._addCommandSubs('append') this._addCommandSubs('prepend') } + _addCommandSubs (name) { this.addSubs(name, ({ bucket, collection, seedNodes }) => { const store = storage.getStore() diff --git a/packages/datadog-plugin-couchbase/test/index.spec.js b/packages/datadog-plugin-couchbase/test/index.spec.js index 1869afb8483..6532360feb3 100644 --- a/packages/datadog-plugin-couchbase/test/index.spec.js +++ b/packages/datadog-plugin-couchbase/test/index.spec.js @@ -289,6 +289,7 @@ describe('Plugin', () => { if (err) done(err) }) }) + describe('errors are handled correctly in callbacks', () => { it('should catch error in callback for non-traced functions', done => { const invalidIndex = '-1' diff --git a/packages/datadog-plugin-couchbase/test/integration-test/client.spec.js b/packages/datadog-plugin-couchbase/test/integration-test/client.spec.js index d1d645f5e4b..0359c33afba 100644 --- a/packages/datadog-plugin-couchbase/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-couchbase/test/integration-test/client.spec.js @@ -22,7 +22,7 @@ describe('esm', () => { before(async function () { this.timeout(20000) sandbox = await createSandbox([`'couchbase@${version}'`], false, [ - `./packages/datadog-plugin-couchbase/test/integration-test/*`]) + './packages/datadog-plugin-couchbase/test/integration-test/*']) }) after(async () => { diff --git a/packages/datadog-plugin-cucumber/src/index.js b/packages/datadog-plugin-cucumber/src/index.js index 98fa1b4037c..d24f97c33e6 100644 --- a/packages/datadog-plugin-cucumber/src/index.js +++ b/packages/datadog-plugin-cucumber/src/index.js @@ -12,10 +12,53 @@ const { getTestSuiteCommonTags, addIntelligentTestRunnerSpanTags, TEST_ITR_UNSKIPPABLE, - TEST_ITR_FORCED_RUN + TEST_ITR_FORCED_RUN, + TEST_CODE_OWNERS, + ITR_CORRELATION_ID, + TEST_SOURCE_FILE, + TEST_EARLY_FLAKE_ENABLED, + TEST_EARLY_FLAKE_ABORT_REASON, + TEST_IS_NEW, + TEST_IS_RETRY, + TEST_SUITE_ID, + TEST_SESSION_ID, + TEST_COMMAND, + TEST_MODULE, + TEST_MODULE_ID, + TEST_SUITE, + CUCUMBER_IS_PARALLEL } = require('../../dd-trace/src/plugins/util/test') const { RESOURCE_NAME } = require('../../../ext/tags') const { COMPONENT, ERROR_MESSAGE } = require('../../dd-trace/src/constants') +const { + TELEMETRY_EVENT_CREATED, + TELEMETRY_EVENT_FINISHED, + TELEMETRY_CODE_COVERAGE_STARTED, + TELEMETRY_CODE_COVERAGE_FINISHED, + TELEMETRY_ITR_FORCED_TO_RUN, + TELEMETRY_CODE_COVERAGE_EMPTY, + TELEMETRY_ITR_UNSKIPPABLE, + TELEMETRY_CODE_COVERAGE_NUM_FILES, + TEST_IS_RUM_ACTIVE, + TEST_BROWSER_DRIVER, + TELEMETRY_TEST_SESSION +} = require('../../dd-trace/src/ci-visibility/telemetry') +const id = require('../../dd-trace/src/id') + +const isCucumberWorker = !!process.env.CUCUMBER_WORKER_ID + +function getTestSuiteTags (testSuiteSpan) { + const suiteTags = { + [TEST_SUITE_ID]: testSuiteSpan.context().toSpanId(), + [TEST_SESSION_ID]: testSuiteSpan.context().toTraceId(), + [TEST_COMMAND]: testSuiteSpan.context()._tags[TEST_COMMAND], + [TEST_MODULE]: 'cucumber' + } + if (testSuiteSpan.context()._parentId) { + suiteTags[TEST_MODULE_ID] = testSuiteSpan.context()._parentId.toString(10) + } + return suiteTags +} class CucumberPlugin extends CiPlugin { static get id () { @@ -27,15 +70,20 @@ class CucumberPlugin extends CiPlugin { this.sourceRoot = process.cwd() + this.testSuiteSpanByPath = {} + this.addSub('ci:cucumber:session:finish', ({ status, isSuitesSkipped, numSkippedSuites, testCodeCoverageLinesTotal, hasUnskippableSuites, - hasForcedToRunSuites + hasForcedToRunSuites, + isEarlyFlakeDetectionEnabled, + isEarlyFlakeDetectionFaulty, + isParallel }) => { - const { isSuitesSkippingEnabled, isCodeCoverageEnabled } = this.itrConfig || {} + const { isSuitesSkippingEnabled, isCodeCoverageEnabled } = this.libraryConfig || {} addIntelligentTestRunnerSpanTags( this.testSessionSpan, this.testModuleSpan, @@ -50,18 +98,38 @@ class CucumberPlugin extends CiPlugin { hasForcedToRunSuites } ) + if (isEarlyFlakeDetectionEnabled) { + this.testSessionSpan.setTag(TEST_EARLY_FLAKE_ENABLED, 'true') + } + if (isEarlyFlakeDetectionFaulty) { + this.testSessionSpan.setTag(TEST_EARLY_FLAKE_ABORT_REASON, 'faulty') + } + if (isParallel) { + this.testSessionSpan.setTag(CUCUMBER_IS_PARALLEL, 'true') + } this.testSessionSpan.setTag(TEST_STATUS, status) this.testModuleSpan.setTag(TEST_STATUS, status) this.testModuleSpan.finish() + this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'module') this.testSessionSpan.finish() + this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'session') finishAllTraceSpans(this.testSessionSpan) + this.telemetry.count(TELEMETRY_TEST_SESSION, { provider: this.ciProviderName }) - this.itrConfig = null + this.libraryConfig = null this.tracer._exporter.flush() }) - this.addSub('ci:cucumber:test-suite:start', ({ testSuitePath, isUnskippable, isForcedToRun }) => { + this.addSub('ci:cucumber:test-suite:start', ({ + testFileAbsolutePath, + isUnskippable, + isForcedToRun, + itrCorrelationId + }) => { + const testSuitePath = getTestSuitePath(testFileAbsolutePath, process.cwd()) + const testSourceFile = getTestSuitePath(testFileAbsolutePath, this.repositoryRoot) + const testSuiteMetadata = getTestSuiteCommonTags( this.command, this.frameworkVersion, @@ -69,12 +137,27 @@ class CucumberPlugin extends CiPlugin { 'cucumber' ) if (isUnskippable) { + this.telemetry.count(TELEMETRY_ITR_UNSKIPPABLE, { testLevel: 'suite' }) testSuiteMetadata[TEST_ITR_UNSKIPPABLE] = 'true' } if (isForcedToRun) { + this.telemetry.count(TELEMETRY_ITR_FORCED_TO_RUN, { testLevel: 'suite' }) testSuiteMetadata[TEST_ITR_FORCED_RUN] = 'true' } - this.testSuiteSpan = this.tracer.startSpan('cucumber.test_suite', { + if (itrCorrelationId) { + testSuiteMetadata[ITR_CORRELATION_ID] = itrCorrelationId + } + if (testSourceFile) { + testSuiteMetadata[TEST_SOURCE_FILE] = testSourceFile + testSuiteMetadata[TEST_SOURCE_START] = 1 + } + + const codeOwners = this.getCodeOwners(testSuiteMetadata) + if (codeOwners) { + testSuiteMetadata[TEST_CODE_OWNERS] = codeOwners + } + + const testSuiteSpan = this.tracer.startSpan('cucumber.test_suite', { childOf: this.testModuleSpan, tags: { [COMPONENT]: this.constructor.id, @@ -82,37 +165,74 @@ class CucumberPlugin extends CiPlugin { ...testSuiteMetadata } }) + this.testSuiteSpanByPath[testSuitePath] = testSuiteSpan + + this.telemetry.ciVisEvent(TELEMETRY_EVENT_CREATED, 'suite') + if (this.libraryConfig?.isCodeCoverageEnabled) { + this.telemetry.ciVisEvent(TELEMETRY_CODE_COVERAGE_STARTED, 'suite', { library: 'istanbul' }) + } }) - this.addSub('ci:cucumber:test-suite:finish', status => { - this.testSuiteSpan.setTag(TEST_STATUS, status) - this.testSuiteSpan.finish() + this.addSub('ci:cucumber:test-suite:finish', ({ status, testSuitePath }) => { + const testSuiteSpan = this.testSuiteSpanByPath[testSuitePath] + testSuiteSpan.setTag(TEST_STATUS, status) + testSuiteSpan.finish() + this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'suite') }) - this.addSub('ci:cucumber:test-suite:code-coverage', ({ coverageFiles, suiteFile }) => { - if (!this.itrConfig || !this.itrConfig.isCodeCoverageEnabled) { + this.addSub('ci:cucumber:test-suite:code-coverage', ({ coverageFiles, suiteFile, testSuitePath }) => { + if (!this.libraryConfig?.isCodeCoverageEnabled) { return } + if (!coverageFiles.length) { + this.telemetry.count(TELEMETRY_CODE_COVERAGE_EMPTY) + } + const testSuiteSpan = this.testSuiteSpanByPath[testSuitePath] + const relativeCoverageFiles = [...coverageFiles, suiteFile] - .map(filename => getTestSuitePath(filename, this.sourceRoot)) + .map(filename => getTestSuitePath(filename, this.repositoryRoot)) + + this.telemetry.distribution(TELEMETRY_CODE_COVERAGE_NUM_FILES, {}, relativeCoverageFiles.length) const formattedCoverage = { - sessionId: this.testSuiteSpan.context()._traceId, - suiteId: this.testSuiteSpan.context()._spanId, + sessionId: testSuiteSpan.context()._traceId, + suiteId: testSuiteSpan.context()._spanId, files: relativeCoverageFiles } this.tracer._exporter.exportCoverage(formattedCoverage) + this.telemetry.ciVisEvent(TELEMETRY_CODE_COVERAGE_FINISHED, 'suite', { library: 'istanbul' }) }) - this.addSub('ci:cucumber:test:start', ({ testName, fullTestSuite, testSourceLine }) => { + this.addSub('ci:cucumber:test:start', ({ testName, testFileAbsolutePath, testSourceLine, isParallel }) => { const store = storage.getStore() - const testSuite = getTestSuitePath(fullTestSuite, this.sourceRoot) - const testSpan = this.startTestSpan(testName, testSuite, testSourceLine) + const testSuite = getTestSuitePath(testFileAbsolutePath, this.sourceRoot) + const testSourceFile = getTestSuitePath(testFileAbsolutePath, this.repositoryRoot) + + const extraTags = { + [TEST_SOURCE_START]: testSourceLine, + [TEST_SOURCE_FILE]: testSourceFile + } + if (isParallel) { + extraTags[CUCUMBER_IS_PARALLEL] = 'true' + } + + const testSpan = this.startTestSpan(testName, testSuite, extraTags) this.enter(testSpan, store) }) + this.addSub('ci:cucumber:test:retry', (isFlakyRetry) => { + const store = storage.getStore() + const span = store.span + if (isFlakyRetry) { + span.setTag(TEST_IS_RETRY, 'true') + } + span.setTag(TEST_STATUS, 'fail') + span.finish() + finishAllTraceSpans(span) + }) + this.addSub('ci:cucumber:test-step:start', ({ resource }) => { const store = storage.getStore() const childOf = store ? store.span : store @@ -127,12 +247,57 @@ class CucumberPlugin extends CiPlugin { this.enter(span, store) }) - this.addSub('ci:cucumber:test:finish', ({ isStep, status, skipReason, errorMessage }) => { + this.addSub('ci:cucumber:worker-report:trace', (traces) => { + const formattedTraces = JSON.parse(traces).map(trace => + trace.map(span => ({ + ...span, + span_id: id(span.span_id), + trace_id: id(span.trace_id), + parent_id: id(span.parent_id) + })) + ) + + // We have to update the test session, test module and test suite ids + // before we export them in the main process + formattedTraces.forEach(trace => { + trace.forEach(span => { + if (span.name === 'cucumber.test') { + const testSuite = span.meta[TEST_SUITE] + const testSuiteSpan = this.testSuiteSpanByPath[testSuite] + + const testSuiteTags = getTestSuiteTags(testSuiteSpan) + span.meta = { + ...span.meta, + ...testSuiteTags + } + } + }) + + this.tracer._exporter.export(trace) + }) + }) + + this.addSub('ci:cucumber:test:finish', ({ + isStep, + status, + skipReason, + errorMessage, + isNew, + isEfdRetry, + isFlakyRetry + }) => { const span = storage.getStore().span const statusTag = isStep ? 'step.status' : TEST_STATUS span.setTag(statusTag, status) + if (isNew) { + span.setTag(TEST_IS_NEW, 'true') + if (isEfdRetry) { + span.setTag(TEST_IS_RETRY, 'true') + } + } + if (skipReason) { span.setTag(TEST_SKIP_REASON, skipReason) } @@ -141,9 +306,28 @@ class CucumberPlugin extends CiPlugin { span.setTag(ERROR_MESSAGE, errorMessage) } + if (isFlakyRetry > 0) { + span.setTag(TEST_IS_RETRY, 'true') + } + span.finish() if (!isStep) { + const spanTags = span.context()._tags + this.telemetry.ciVisEvent( + TELEMETRY_EVENT_FINISHED, + 'test', + { + hasCodeOwners: !!spanTags[TEST_CODE_OWNERS], + isNew, + isRum: spanTags[TEST_IS_RUM_ACTIVE] === 'true', + browserDriver: spanTags[TEST_BROWSER_DRIVER] + } + ) finishAllTraceSpans(span) + // If it's a worker, flushing is cheap, as it's just sending data to the main process + if (isCucumberWorker) { + this.tracer._exporter.flush() + } } }) @@ -155,12 +339,13 @@ class CucumberPlugin extends CiPlugin { }) } - startTestSpan (testName, testSuite, testSourceLine) { + startTestSpan (testName, testSuite, extraTags) { + const testSuiteSpan = this.testSuiteSpanByPath[testSuite] return super.startTestSpan( testName, testSuite, - this.testSuiteSpan, - { [TEST_SOURCE_START]: testSourceLine } + testSuiteSpan, + extraTags ) } } diff --git a/packages/datadog-plugin-cucumber/test/index.spec.js b/packages/datadog-plugin-cucumber/test/index.spec.js index 71023f58d0f..a43a2a53509 100644 --- a/packages/datadog-plugin-cucumber/test/index.spec.js +++ b/packages/datadog-plugin-cucumber/test/index.spec.js @@ -1,6 +1,7 @@ 'use strict' const path = require('path') const { PassThrough } = require('stream') +const semver = require('semver') const proxyquire = require('proxyquire').noPreserveCache() const nock = require('nock') @@ -23,6 +24,7 @@ const { TEST_SOURCE_START } = require('../../dd-trace/src/plugins/util/test') +const { NODE_MAJOR } = require('../../../version') const { version: ddTraceVersion } = require('../../../package.json') const runCucumber = (version, Cucumber, requireName, featureName, testName) => { @@ -53,7 +55,9 @@ const runCucumber = (version, Cucumber, requireName, featureName, testName) => { describe('Plugin', function () { let Cucumber this.timeout(10000) - withVersions('cucumber', '@cucumber/cucumber', version => { + withVersions('cucumber', '@cucumber/cucumber', (version, _, specificVersion) => { + if (NODE_MAJOR <= 16 && semver.satisfies(specificVersion, '>=10')) return + afterEach(() => { // > If you want to run tests multiple times, you may need to clear Node's require cache // before subsequent calls in whichever manner best suits your needs. @@ -108,6 +112,7 @@ describe('Plugin', function () { expect(result.success).to.equal(true) await checkTraces }) + it('should create spans for each cucumber step', async () => { const steps = [ { name: 'datadog', stepStatus: 'pass' }, @@ -136,6 +141,7 @@ describe('Plugin', function () { await checkTraces }) }) + describe('failing test', () => { it('should create a test span', async function () { const checkTraces = agent.use(traces => { @@ -167,6 +173,7 @@ describe('Plugin', function () { expect(result.success).to.equal(false) await checkTraces }) + it('should create spans for each cucumber step', async () => { const steps = [ { name: 'datadog', stepStatus: 'pass' }, @@ -202,6 +209,7 @@ describe('Plugin', function () { await checkTraces }) }) + describe('skipped test', () => { it('should create a test span', async function () { const checkTraces = agent.use(traces => { @@ -233,6 +241,7 @@ describe('Plugin', function () { expect(result.success).to.equal(true) await checkTraces }) + it('should create spans for each cucumber step', async () => { const steps = [ { name: 'datadog', stepStatus: 'pass' }, @@ -261,6 +270,7 @@ describe('Plugin', function () { await checkTraces }) }) + describe('skipped test based on tag', () => { it('should create a test span', async function () { const checkTraces = agent.use(traces => { @@ -298,6 +308,7 @@ describe('Plugin', function () { expect(result.success).to.equal(true) await checkTraces }) + it('should create spans for each cucumber step', async () => { const steps = [ { name: 'datadog', stepStatus: 'skip' } @@ -330,6 +341,7 @@ describe('Plugin', function () { await checkTraces }) }) + describe('not implemented step', () => { it('should create a test span with a skip reason', async () => { const checkTraces = agent.use(traces => { @@ -360,6 +372,7 @@ describe('Plugin', function () { await checkTraces }) }) + describe('integration test', () => { it('should create a test span and a span for the integration', async function () { const checkTraces = agent.use(traces => { @@ -396,6 +409,7 @@ describe('Plugin', function () { expect(result.success).to.equal(true) await checkTraces }) + it('should create spans for each cucumber step', async () => { const steps = [ { name: 'datadog', stepStatus: 'pass' }, @@ -424,6 +438,7 @@ describe('Plugin', function () { await checkTraces }) }) + describe('hook fail', () => { it('should create a test span', async function () { const checkTraces = agent.use(traces => { @@ -451,14 +466,17 @@ describe('Plugin', function () { expect(testSpan.name).to.equal('cucumber.test') expect(testSpan.resource.endsWith('simple.feature.hooks fail')).to.equal(true) expect( - testSpan.meta[ERROR_MESSAGE].startsWith(`TypeError: Cannot set property 'boom' of undefined`) || - testSpan.meta[ERROR_MESSAGE].startsWith(`TypeError: Cannot set properties of undefined (setting 'boom')`) + testSpan.meta[ERROR_MESSAGE].startsWith( + 'TypeError: Cannot set property \'boom\' of undefined') || + testSpan.meta[ERROR_MESSAGE].startsWith( + 'TypeError: Cannot set properties of undefined (setting \'boom\')') ).to.equal(true) }) const result = await runCucumber(version, Cucumber, 'simple.js', 'simple.feature', 'hooks fail') expect(result.success).to.equal(false) await checkTraces }) + it('should create spans for each cucumber step', async () => { const steps = [ { name: 'datadog', stepStatus: 'skip' }, diff --git a/packages/datadog-plugin-cypress/src/after-run.js b/packages/datadog-plugin-cypress/src/after-run.js new file mode 100644 index 00000000000..288218850d8 --- /dev/null +++ b/packages/datadog-plugin-cypress/src/after-run.js @@ -0,0 +1,3 @@ +const cypressPlugin = require('./cypress-plugin') + +module.exports = cypressPlugin.afterRun.bind(cypressPlugin) diff --git a/packages/datadog-plugin-cypress/src/after-spec.js b/packages/datadog-plugin-cypress/src/after-spec.js new file mode 100644 index 00000000000..4fdf98ad582 --- /dev/null +++ b/packages/datadog-plugin-cypress/src/after-spec.js @@ -0,0 +1,3 @@ +const cypressPlugin = require('./cypress-plugin') + +module.exports = cypressPlugin.afterSpec.bind(cypressPlugin) diff --git a/packages/datadog-plugin-cypress/src/cypress-plugin.js b/packages/datadog-plugin-cypress/src/cypress-plugin.js new file mode 100644 index 00000000000..630d613f772 --- /dev/null +++ b/packages/datadog-plugin-cypress/src/cypress-plugin.js @@ -0,0 +1,733 @@ +const { + TEST_STATUS, + TEST_IS_RUM_ACTIVE, + TEST_CODE_OWNERS, + getTestEnvironmentMetadata, + CI_APP_ORIGIN, + getTestParentSpan, + getCodeOwnersFileEntries, + getCodeOwnersForFilename, + getTestCommonTags, + getTestSessionCommonTags, + getTestModuleCommonTags, + getTestSuiteCommonTags, + TEST_SUITE_ID, + TEST_MODULE_ID, + TEST_SESSION_ID, + TEST_COMMAND, + TEST_MODULE, + TEST_SOURCE_START, + finishAllTraceSpans, + getCoveredFilenamesFromCoverage, + getTestSuitePath, + addIntelligentTestRunnerSpanTags, + TEST_SKIPPED_BY_ITR, + TEST_ITR_UNSKIPPABLE, + TEST_ITR_FORCED_RUN, + ITR_CORRELATION_ID, + TEST_SOURCE_FILE, + TEST_IS_NEW, + TEST_IS_RETRY, + TEST_EARLY_FLAKE_ENABLED, + getTestSessionName, + TEST_SESSION_NAME, + TEST_LEVEL_EVENT_TYPES +} = require('../../dd-trace/src/plugins/util/test') +const { isMarkedAsUnskippable } = require('../../datadog-plugin-jest/src/util') +const { ORIGIN_KEY, COMPONENT } = require('../../dd-trace/src/constants') +const { appClosing: appClosingTelemetry } = require('../../dd-trace/src/telemetry') +const log = require('../../dd-trace/src/log') + +const { + TELEMETRY_EVENT_CREATED, + TELEMETRY_EVENT_FINISHED, + TELEMETRY_ITR_FORCED_TO_RUN, + TELEMETRY_CODE_COVERAGE_EMPTY, + TELEMETRY_ITR_UNSKIPPABLE, + TELEMETRY_CODE_COVERAGE_NUM_FILES, + incrementCountMetric, + distributionMetric, + TELEMETRY_ITR_SKIPPED, + TELEMETRY_TEST_SESSION +} = require('../../dd-trace/src/ci-visibility/telemetry') + +const { + GIT_REPOSITORY_URL, + GIT_COMMIT_SHA, + GIT_BRANCH, + CI_PROVIDER_NAME, + CI_WORKSPACE_PATH +} = require('../../dd-trace/src/plugins/util/tags') +const { + OS_VERSION, + OS_PLATFORM, + OS_ARCHITECTURE, + RUNTIME_NAME, + RUNTIME_VERSION +} = require('../../dd-trace/src/plugins/util/env') + +const TEST_FRAMEWORK_NAME = 'cypress' + +const CYPRESS_STATUS_TO_TEST_STATUS = { + passed: 'pass', + failed: 'fail', + pending: 'skip', + skipped: 'skip' +} + +function getSessionStatus (summary) { + if (summary.totalFailed !== undefined && summary.totalFailed > 0) { + return 'fail' + } + if (summary.totalSkipped !== undefined && summary.totalSkipped === summary.totalTests) { + return 'skip' + } + return 'pass' +} + +function getCypressVersion (details) { + if (details?.cypressVersion) { + return details.cypressVersion + } + if (details?.config?.version) { + return details.config.version + } + return '' +} + +function getRootDir (details) { + if (details?.config) { + return details.config.projectRoot || details.config.repoRoot || process.cwd() + } + return process.cwd() +} + +function getCypressCommand (details) { + if (!details) { + return TEST_FRAMEWORK_NAME + } + return `${TEST_FRAMEWORK_NAME} ${details.specPattern || ''}` +} + +function getLibraryConfiguration (tracer, testConfiguration) { + return new Promise(resolve => { + if (!tracer._tracer._exporter?.getLibraryConfiguration) { + return resolve({ err: new Error('CI Visibility was not initialized correctly') }) + } + + tracer._tracer._exporter.getLibraryConfiguration(testConfiguration, (err, libraryConfig) => { + resolve({ err, libraryConfig }) + }) + }) +} + +function getSkippableTests (tracer, testConfiguration) { + return new Promise(resolve => { + if (!tracer._tracer._exporter?.getSkippableSuites) { + return resolve({ err: new Error('CI Visibility was not initialized correctly') }) + } + tracer._tracer._exporter.getSkippableSuites(testConfiguration, (err, skippableTests, correlationId) => { + resolve({ + err, + skippableTests, + correlationId + }) + }) + }) +} + +function getKnownTests (tracer, testConfiguration) { + return new Promise(resolve => { + if (!tracer._tracer._exporter?.getKnownTests) { + return resolve({ err: new Error('CI Visibility was not initialized correctly') }) + } + tracer._tracer._exporter.getKnownTests(testConfiguration, (err, knownTests) => { + resolve({ + err, + knownTests + }) + }) + }) +} + +function getSuiteStatus (suiteStats) { + if (!suiteStats) { + return 'skip' + } + if (suiteStats.failures !== undefined && suiteStats.failures > 0) { + return 'fail' + } + if (suiteStats.tests !== undefined && + (suiteStats.tests === suiteStats.pending || suiteStats.tests === suiteStats.skipped)) { + return 'skip' + } + return 'pass' +} + +class CypressPlugin { + constructor () { + this._isInit = false + this.testEnvironmentMetadata = getTestEnvironmentMetadata(TEST_FRAMEWORK_NAME) + + const { + [GIT_REPOSITORY_URL]: repositoryUrl, + [GIT_COMMIT_SHA]: sha, + [OS_VERSION]: osVersion, + [OS_PLATFORM]: osPlatform, + [OS_ARCHITECTURE]: osArchitecture, + [RUNTIME_NAME]: runtimeName, + [RUNTIME_VERSION]: runtimeVersion, + [GIT_BRANCH]: branch, + [CI_PROVIDER_NAME]: ciProviderName, + [CI_WORKSPACE_PATH]: repositoryRoot + } = this.testEnvironmentMetadata + + this.repositoryRoot = repositoryRoot + this.ciProviderName = ciProviderName + this.codeOwnersEntries = getCodeOwnersFileEntries(repositoryRoot) + + this.testConfiguration = { + repositoryUrl, + sha, + osVersion, + osPlatform, + osArchitecture, + runtimeName, + runtimeVersion, + branch, + testLevel: 'test' + } + this.finishedTestsByFile = {} + + this.isTestsSkipped = false + this.isSuitesSkippingEnabled = false + this.isCodeCoverageEnabled = false + this.isEarlyFlakeDetectionEnabled = false + this.earlyFlakeDetectionNumRetries = 0 + this.testsToSkip = [] + this.skippedTests = [] + this.hasForcedToRunSuites = false + this.hasUnskippableSuites = false + this.unskippableSuites = [] + this.knownTests = [] + } + + // Init function returns a promise that resolves with the Cypress configuration + // Depending on the received configuration, the Cypress configuration can be modified: + // for example, to enable retries for failed tests. + init (tracer, cypressConfig) { + this._isInit = true + this.tracer = tracer + this.cypressConfig = cypressConfig + + this.libraryConfigurationPromise = getLibraryConfiguration(this.tracer, this.testConfiguration) + .then((libraryConfigurationResponse) => { + if (libraryConfigurationResponse.err) { + log.error(libraryConfigurationResponse.err) + } else { + const { + libraryConfig: { + isSuitesSkippingEnabled, + isCodeCoverageEnabled, + isEarlyFlakeDetectionEnabled, + earlyFlakeDetectionNumRetries, + isFlakyTestRetriesEnabled, + flakyTestRetriesCount + } + } = libraryConfigurationResponse + this.isSuitesSkippingEnabled = isSuitesSkippingEnabled + this.isCodeCoverageEnabled = isCodeCoverageEnabled + this.isEarlyFlakeDetectionEnabled = isEarlyFlakeDetectionEnabled + this.earlyFlakeDetectionNumRetries = earlyFlakeDetectionNumRetries + if (isFlakyTestRetriesEnabled) { + this.cypressConfig.retries.runMode = flakyTestRetriesCount + } + } + return this.cypressConfig + }) + return this.libraryConfigurationPromise + } + + getTestSuiteSpan ({ testSuite, testSuiteAbsolutePath }) { + const testSuiteSpanMetadata = + getTestSuiteCommonTags(this.command, this.frameworkVersion, testSuite, TEST_FRAMEWORK_NAME) + + this.ciVisEvent(TELEMETRY_EVENT_CREATED, 'suite') + + if (testSuiteAbsolutePath) { + const testSourceFile = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) + testSuiteSpanMetadata[TEST_SOURCE_FILE] = testSourceFile + testSuiteSpanMetadata[TEST_SOURCE_START] = 1 + const codeOwners = this.getTestCodeOwners({ testSuite, testSourceFile }) + if (codeOwners) { + testSuiteSpanMetadata[TEST_CODE_OWNERS] = codeOwners + } + } + + return this.tracer.startSpan(`${TEST_FRAMEWORK_NAME}.test_suite`, { + childOf: this.testModuleSpan, + tags: { + [COMPONENT]: TEST_FRAMEWORK_NAME, + ...this.testEnvironmentMetadata, + ...testSuiteSpanMetadata + } + }) + } + + getTestSpan ({ testName, testSuite, isUnskippable, isForcedToRun, testSourceFile }) { + const testSuiteTags = { + [TEST_COMMAND]: this.command, + [TEST_COMMAND]: this.command, + [TEST_MODULE]: TEST_FRAMEWORK_NAME + } + if (this.testSuiteSpan) { + testSuiteTags[TEST_SUITE_ID] = this.testSuiteSpan.context().toSpanId() + } + if (this.testSessionSpan && this.testModuleSpan) { + testSuiteTags[TEST_SESSION_ID] = this.testSessionSpan.context().toTraceId() + testSuiteTags[TEST_MODULE_ID] = this.testModuleSpan.context().toSpanId() + // If testSuiteSpan couldn't be created, we'll use the testModuleSpan as the parent + if (!this.testSuiteSpan) { + testSuiteTags[TEST_SUITE_ID] = this.testModuleSpan.context().toSpanId() + } + } + + const childOf = getTestParentSpan(this.tracer) + const { + resource, + ...testSpanMetadata + } = getTestCommonTags(testName, testSuite, this.cypressConfig.version, TEST_FRAMEWORK_NAME) + + if (testSourceFile) { + testSpanMetadata[TEST_SOURCE_FILE] = testSourceFile + } + + const codeOwners = this.getTestCodeOwners({ testSuite, testSourceFile }) + if (codeOwners) { + testSpanMetadata[TEST_CODE_OWNERS] = codeOwners + } + + if (isUnskippable) { + this.hasUnskippableSuites = true + incrementCountMetric(TELEMETRY_ITR_UNSKIPPABLE, { testLevel: 'suite' }) + testSpanMetadata[TEST_ITR_UNSKIPPABLE] = 'true' + } + + if (isForcedToRun) { + this.hasForcedToRunSuites = true + incrementCountMetric(TELEMETRY_ITR_FORCED_TO_RUN, { testLevel: 'suite' }) + testSpanMetadata[TEST_ITR_FORCED_RUN] = 'true' + } + + this.ciVisEvent(TELEMETRY_EVENT_CREATED, 'test', { hasCodeOwners: !!codeOwners }) + + return this.tracer.startSpan(`${TEST_FRAMEWORK_NAME}.test`, { + childOf, + tags: { + [COMPONENT]: TEST_FRAMEWORK_NAME, + [ORIGIN_KEY]: CI_APP_ORIGIN, + ...testSpanMetadata, + ...this.testEnvironmentMetadata, + ...testSuiteTags + } + }) + } + + ciVisEvent (name, testLevel, tags = {}) { + incrementCountMetric(name, { + testLevel, + testFramework: 'cypress', + isUnsupportedCIProvider: !this.ciProviderName, + ...tags + }) + } + + isNewTest (testName, testSuite) { + return !this.knownTestsByTestSuite?.[testSuite]?.includes(testName) + } + + async beforeRun (details) { + // We need to make sure that the plugin is initialized before running the tests + // This is for the case where the user has not returned the promise from the init function + await this.libraryConfigurationPromise + this.command = getCypressCommand(details) + this.frameworkVersion = getCypressVersion(details) + this.rootDir = getRootDir(details) + + if (this.isEarlyFlakeDetectionEnabled) { + const knownTestsResponse = await getKnownTests( + this.tracer, + this.testConfiguration + ) + if (knownTestsResponse.err) { + log.error(knownTestsResponse.err) + this.isEarlyFlakeDetectionEnabled = false + } else { + // We use TEST_FRAMEWORK_NAME for the name of the module + this.knownTestsByTestSuite = knownTestsResponse.knownTests[TEST_FRAMEWORK_NAME] + } + } + + if (this.isSuitesSkippingEnabled) { + const skippableTestsResponse = await getSkippableTests( + this.tracer, + this.testConfiguration + ) + if (skippableTestsResponse.err) { + log.error(skippableTestsResponse.err) + } else { + const { skippableTests, correlationId } = skippableTestsResponse + this.testsToSkip = skippableTests || [] + this.itrCorrelationId = correlationId + incrementCountMetric(TELEMETRY_ITR_SKIPPED, { testLevel: 'test' }, this.testsToSkip.length) + } + } + + // `details.specs` are test files + details.specs?.forEach(({ absolute, relative }) => { + const isUnskippableSuite = isMarkedAsUnskippable({ path: absolute }) + if (isUnskippableSuite) { + this.unskippableSuites.push(relative) + } + }) + + const childOf = getTestParentSpan(this.tracer) + + const testSessionSpanMetadata = + getTestSessionCommonTags(this.command, this.frameworkVersion, TEST_FRAMEWORK_NAME) + const testModuleSpanMetadata = + getTestModuleCommonTags(this.command, this.frameworkVersion, TEST_FRAMEWORK_NAME) + + if (this.isEarlyFlakeDetectionEnabled) { + testSessionSpanMetadata[TEST_EARLY_FLAKE_ENABLED] = 'true' + } + + const testSessionName = getTestSessionName(this.tracer._tracer._config, this.command, this.testEnvironmentMetadata) + + if (this.tracer._tracer._exporter?.setMetadataTags) { + const metadataTags = {} + for (const testLevel of TEST_LEVEL_EVENT_TYPES) { + metadataTags[testLevel] = { + [TEST_SESSION_NAME]: testSessionName + } + } + this.tracer._tracer._exporter.setMetadataTags(metadataTags) + } + + this.testSessionSpan = this.tracer.startSpan(`${TEST_FRAMEWORK_NAME}.test_session`, { + childOf, + tags: { + [COMPONENT]: TEST_FRAMEWORK_NAME, + ...this.testEnvironmentMetadata, + ...testSessionSpanMetadata + } + }) + this.ciVisEvent(TELEMETRY_EVENT_CREATED, 'session') + + this.testModuleSpan = this.tracer.startSpan(`${TEST_FRAMEWORK_NAME}.test_module`, { + childOf: this.testSessionSpan, + tags: { + [COMPONENT]: TEST_FRAMEWORK_NAME, + ...this.testEnvironmentMetadata, + ...testModuleSpanMetadata + } + }) + this.ciVisEvent(TELEMETRY_EVENT_CREATED, 'module') + + return details + } + + afterRun (suiteStats) { + if (!this._isInit) { + log.warn('Attemping to call afterRun without initializating the plugin first') + return + } + if (this.testSessionSpan && this.testModuleSpan) { + const testStatus = getSessionStatus(suiteStats) + this.testModuleSpan.setTag(TEST_STATUS, testStatus) + this.testSessionSpan.setTag(TEST_STATUS, testStatus) + + addIntelligentTestRunnerSpanTags( + this.testSessionSpan, + this.testModuleSpan, + { + isSuitesSkipped: this.isTestsSkipped, + isSuitesSkippingEnabled: this.isSuitesSkippingEnabled, + isCodeCoverageEnabled: this.isCodeCoverageEnabled, + skippingType: 'test', + skippingCount: this.skippedTests.length, + hasForcedToRunSuites: this.hasForcedToRunSuites, + hasUnskippableSuites: this.hasUnskippableSuites + } + ) + + this.testModuleSpan.finish() + this.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'module') + this.testSessionSpan.finish() + this.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'session') + incrementCountMetric(TELEMETRY_TEST_SESSION, { + provider: this.ciProviderName + }) + + finishAllTraceSpans(this.testSessionSpan) + } + + return new Promise(resolve => { + const exporter = this.tracer._tracer._exporter + if (!exporter) { + return resolve(null) + } + if (exporter.flush) { + exporter.flush(() => { + appClosingTelemetry() + resolve(null) + }) + } else if (exporter._writer) { + exporter._writer.flush(() => { + appClosingTelemetry() + resolve(null) + }) + } + }) + } + + afterSpec (spec, results) { + const { tests, stats } = results || {} + const cypressTests = tests || [] + const finishedTests = this.finishedTestsByFile[spec.relative] || [] + + if (!this.testSuiteSpan) { + // dd:testSuiteStart hasn't been triggered for whatever reason + // We will create the test suite span on the spot if that's the case + log.warn('There was an error creating the test suite event.') + this.testSuiteSpan = this.getTestSuiteSpan({ + testSuite: spec.relative, + testSuiteAbsolutePath: spec.absolute + }) + } + + // Get tests that didn't go through `dd:afterEach` + // and create a skipped test span for each of them + cypressTests.filter(({ title }) => { + const cypressTestName = title.join(' ') + const isTestFinished = finishedTests.find(({ testName }) => cypressTestName === testName) + + return !isTestFinished + }).forEach(({ title }) => { + const cypressTestName = title.join(' ') + const isSkippedByItr = this.testsToSkip.find(test => + cypressTestName === test.name && spec.relative === test.suite + ) + let testSourceFile + + if (spec.absolute && this.repositoryRoot) { + testSourceFile = getTestSuitePath(spec.absolute, this.repositoryRoot) + } else { + testSourceFile = spec.relative + } + + const skippedTestSpan = this.getTestSpan({ testName: cypressTestName, testSuite: spec.relative, testSourceFile }) + + skippedTestSpan.setTag(TEST_STATUS, 'skip') + if (isSkippedByItr) { + skippedTestSpan.setTag(TEST_SKIPPED_BY_ITR, 'true') + } + if (this.itrCorrelationId) { + skippedTestSpan.setTag(ITR_CORRELATION_ID, this.itrCorrelationId) + } + skippedTestSpan.finish() + }) + + // Make sure that reported test statuses are the same as Cypress reports. + // This is not always the case, such as when an `after` hook fails: + // Cypress will report the last run test as failed, but we don't know that yet at `dd:afterEach` + let latestError + + const finishedTestsByTestName = finishedTests.reduce((acc, finishedTest) => { + if (!acc[finishedTest.testName]) { + acc[finishedTest.testName] = [] + } + acc[finishedTest.testName].push(finishedTest) + return acc + }, {}) + + Object.entries(finishedTestsByTestName).forEach(([testName, finishedTestAttempts]) => { + finishedTestAttempts.forEach((finishedTest, attemptIndex) => { + // TODO: there could be multiple if there have been retries! + // potentially we need to match the test status! + const cypressTest = cypressTests.find(test => test.title.join(' ') === testName) + if (!cypressTest) { + return + } + // finishedTests can include multiple tests with the same name if they have been retried + // by early flake detection. Cypress is unaware of this so .attempts does not necessarily have + // the same length as `finishedTestAttempts` + let cypressTestStatus = CYPRESS_STATUS_TO_TEST_STATUS[cypressTest.state] + if (cypressTest.attempts && cypressTest.attempts[attemptIndex]) { + cypressTestStatus = CYPRESS_STATUS_TO_TEST_STATUS[cypressTest.attempts[attemptIndex].state] + if (attemptIndex > 0) { + finishedTest.testSpan.setTag(TEST_IS_RETRY, 'true') + } + } + if (cypressTest.displayError) { + latestError = new Error(cypressTest.displayError) + } + // Update test status + if (cypressTestStatus !== finishedTest.testStatus) { + finishedTest.testSpan.setTag(TEST_STATUS, cypressTestStatus) + finishedTest.testSpan.setTag('error', latestError) + } + if (this.itrCorrelationId) { + finishedTest.testSpan.setTag(ITR_CORRELATION_ID, this.itrCorrelationId) + } + let testSourceFile + if (spec.absolute && this.repositoryRoot) { + testSourceFile = getTestSuitePath(spec.absolute, this.repositoryRoot) + } else { + testSourceFile = spec.relative + } + if (testSourceFile) { + finishedTest.testSpan.setTag(TEST_SOURCE_FILE, testSourceFile) + } + const codeOwners = this.getTestCodeOwners({ testSuite: spec.relative, testSourceFile }) + + if (codeOwners) { + finishedTest.testSpan.setTag(TEST_CODE_OWNERS, codeOwners) + } + + finishedTest.testSpan.finish(finishedTest.finishTime) + }) + }) + + if (this.testSuiteSpan) { + const status = getSuiteStatus(stats) + this.testSuiteSpan.setTag(TEST_STATUS, status) + + if (latestError) { + this.testSuiteSpan.setTag('error', latestError) + } + this.testSuiteSpan.finish() + this.testSuiteSpan = null + this.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'suite') + } + } + + getTasks () { + return { + 'dd:testSuiteStart': ({ testSuite, testSuiteAbsolutePath }) => { + const suitePayload = { + isEarlyFlakeDetectionEnabled: this.isEarlyFlakeDetectionEnabled, + knownTestsForSuite: this.knownTestsByTestSuite?.[testSuite] || [], + earlyFlakeDetectionNumRetries: this.earlyFlakeDetectionNumRetries + } + + if (this.testSuiteSpan) { + return suitePayload + } + this.testSuiteSpan = this.getTestSuiteSpan({ testSuite, testSuiteAbsolutePath }) + return suitePayload + }, + 'dd:beforeEach': (test) => { + const { testName, testSuite } = test + const shouldSkip = !!this.testsToSkip.find(test => { + return testName === test.name && testSuite === test.suite + }) + const isUnskippable = this.unskippableSuites.includes(testSuite) + const isForcedToRun = shouldSkip && isUnskippable + + // skip test + if (shouldSkip && !isUnskippable) { + this.skippedTests.push(test) + this.isTestsSkipped = true + return { shouldSkip: true } + } + + if (!this.activeTestSpan) { + this.activeTestSpan = this.getTestSpan({ + testName, + testSuite, + isUnskippable, + isForcedToRun + }) + } + + return this.activeTestSpan ? { traceId: this.activeTestSpan.context().toTraceId() } : {} + }, + 'dd:afterEach': ({ test, coverage }) => { + const { state, error, isRUMActive, testSourceLine, testSuite, testName, isNew, isEfdRetry } = test + if (this.activeTestSpan) { + if (coverage && this.isCodeCoverageEnabled && this.tracer._tracer._exporter?.exportCoverage) { + const coverageFiles = getCoveredFilenamesFromCoverage(coverage) + const relativeCoverageFiles = coverageFiles.map(file => getTestSuitePath(file, this.rootDir)) + if (!relativeCoverageFiles.length) { + incrementCountMetric(TELEMETRY_CODE_COVERAGE_EMPTY) + } + distributionMetric(TELEMETRY_CODE_COVERAGE_NUM_FILES, {}, relativeCoverageFiles.length) + const { _traceId, _spanId } = this.testSuiteSpan.context() + const formattedCoverage = { + sessionId: _traceId, + suiteId: _spanId, + testId: this.activeTestSpan.context()._spanId, + files: relativeCoverageFiles + } + this.tracer._tracer._exporter.exportCoverage(formattedCoverage) + } + const testStatus = CYPRESS_STATUS_TO_TEST_STATUS[state] + this.activeTestSpan.setTag(TEST_STATUS, testStatus) + + if (error) { + this.activeTestSpan.setTag('error', error) + } + if (isRUMActive) { + this.activeTestSpan.setTag(TEST_IS_RUM_ACTIVE, 'true') + } + if (testSourceLine) { + this.activeTestSpan.setTag(TEST_SOURCE_START, testSourceLine) + } + if (isNew) { + this.activeTestSpan.setTag(TEST_IS_NEW, 'true') + if (isEfdRetry) { + this.activeTestSpan.setTag(TEST_IS_RETRY, 'true') + } + } + const finishedTest = { + testName, + testStatus, + finishTime: this.activeTestSpan._getTime(), // we store the finish time here + testSpan: this.activeTestSpan + } + if (this.finishedTestsByFile[testSuite]) { + this.finishedTestsByFile[testSuite].push(finishedTest) + } else { + this.finishedTestsByFile[testSuite] = [finishedTest] + } + // test spans are finished at after:spec + } + this.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'test', { + hasCodeOwners: !!this.activeTestSpan.context()._tags[TEST_CODE_OWNERS], + isNew, + isRum: isRUMActive, + browserDriver: 'cypress' + }) + this.activeTestSpan = null + + return null + }, + 'dd:addTags': (tags) => { + if (this.activeTestSpan) { + this.activeTestSpan.addTags(tags) + } + return null + } + } + } + + getTestCodeOwners ({ testSuite, testSourceFile }) { + if (testSourceFile) { + return getCodeOwnersForFilename(testSourceFile, this.codeOwnersEntries) + } + return getCodeOwnersForFilename(testSuite, this.codeOwnersEntries) + } +} + +module.exports = new CypressPlugin() diff --git a/packages/datadog-plugin-cypress/src/plugin.js b/packages/datadog-plugin-cypress/src/plugin.js index 6b767501362..6655c54b747 100644 --- a/packages/datadog-plugin-cypress/src/plugin.js +++ b/packages/datadog-plugin-cypress/src/plugin.js @@ -1,127 +1,5 @@ -const { - TEST_STATUS, - TEST_IS_RUM_ACTIVE, - TEST_CODE_OWNERS, - getTestEnvironmentMetadata, - CI_APP_ORIGIN, - getTestParentSpan, - getCodeOwnersFileEntries, - getCodeOwnersForFilename, - getTestCommonTags, - getTestSessionCommonTags, - getTestModuleCommonTags, - getTestSuiteCommonTags, - TEST_SUITE_ID, - TEST_MODULE_ID, - TEST_SESSION_ID, - TEST_COMMAND, - TEST_MODULE, - TEST_SOURCE_START, - finishAllTraceSpans, - getCoveredFilenamesFromCoverage, - getTestSuitePath, - addIntelligentTestRunnerSpanTags, - TEST_SKIPPED_BY_ITR, - TEST_ITR_UNSKIPPABLE, - TEST_ITR_FORCED_RUN -} = require('../../dd-trace/src/plugins/util/test') -const { ORIGIN_KEY, COMPONENT } = require('../../dd-trace/src/constants') -const log = require('../../dd-trace/src/log') const NoopTracer = require('../../dd-trace/src/noop/tracer') -const { isMarkedAsUnskippable } = require('../../datadog-plugin-jest/src/util') - -const TEST_FRAMEWORK_NAME = 'cypress' - -const CYPRESS_STATUS_TO_TEST_STATUS = { - passed: 'pass', - failed: 'fail', - pending: 'skip', - skipped: 'skip' -} - -function getTestSpanMetadata (tracer, testName, testSuite, cypressConfig) { - const childOf = getTestParentSpan(tracer) - - const commonTags = getTestCommonTags(testName, testSuite, cypressConfig.version, TEST_FRAMEWORK_NAME) - - return { - childOf, - ...commonTags - } -} - -function getCypressVersion (details) { - if (details && details.cypressVersion) { - return details.cypressVersion - } - if (details && details.config && details.config.version) { - return details.config.version - } - return '' -} - -function getRootDir (details) { - if (details && details.config) { - return details.config.projectRoot || details.config.repoRoot || process.cwd() - } - return process.cwd() -} - -function getCypressCommand (details) { - if (!details) { - return TEST_FRAMEWORK_NAME - } - return `${TEST_FRAMEWORK_NAME} ${details.specPattern || ''}` -} - -function getSessionStatus (summary) { - if (summary.totalFailed !== undefined && summary.totalFailed > 0) { - return 'fail' - } - if (summary.totalSkipped !== undefined && summary.totalSkipped === summary.totalTests) { - return 'skip' - } - return 'pass' -} - -function getSuiteStatus (suiteStats) { - if (suiteStats.failures !== undefined && suiteStats.failures > 0) { - return 'fail' - } - if (suiteStats.tests !== undefined && suiteStats.tests === suiteStats.pending) { - return 'skip' - } - return 'pass' -} - -function getItrConfig (tracer, testConfiguration) { - return new Promise(resolve => { - if (!tracer._tracer._exporter || !tracer._tracer._exporter.getItrConfiguration) { - return resolve({ err: new Error('CI Visibility was not initialized correctly') }) - } - - tracer._tracer._exporter.getItrConfiguration(testConfiguration, (err, itrConfig) => { - resolve({ err, itrConfig }) - }) - }) -} - -function getSkippableTests (isSuitesSkippingEnabled, tracer, testConfiguration) { - if (!isSuitesSkippingEnabled) { - return Promise.resolve({ skippableTests: [] }) - } - return new Promise(resolve => { - if (!tracer._tracer._exporter || !tracer._tracer._exporter.getItrConfiguration) { - return resolve({ err: new Error('CI Visibility was not initialized correctly') }) - } - tracer._tracer._exporter.getSkippableSuites(testConfiguration, (err, skippableTests) => { - resolve({ - err, - skippableTests - }) - }) - }) -} +const cypressPlugin = require('./cypress-plugin') const noopTask = { 'dd:testSuiteStart': () => { @@ -139,344 +17,19 @@ const noopTask = { } module.exports = (on, config) => { - let isTestsSkipped = false - const skippedTests = [] const tracer = require('../../dd-trace') // The tracer was not init correctly for whatever reason (such as invalid DD_SITE) if (tracer._tracer instanceof NoopTracer) { // We still need to register these tasks or the support file will fail - return on('task', noopTask) - } - - const testEnvironmentMetadata = getTestEnvironmentMetadata(TEST_FRAMEWORK_NAME) - - const { - 'git.repository_url': repositoryUrl, - 'git.commit.sha': sha, - 'os.version': osVersion, - 'os.platform': osPlatform, - 'os.architecture': osArchitecture, - 'runtime.name': runtimeName, - 'runtime.version': runtimeVersion, - 'git.branch': branch - } = testEnvironmentMetadata - - const finishedTestsByFile = {} - - const testConfiguration = { - repositoryUrl, - sha, - osVersion, - osPlatform, - osArchitecture, - runtimeName, - runtimeVersion, - branch, - testLevel: 'test' + on('task', noopTask) + return config } - const codeOwnersEntries = getCodeOwnersFileEntries() - - let activeSpan = null - let testSessionSpan = null - let testModuleSpan = null - let testSuiteSpan = null - let command = null - let frameworkVersion - let rootDir - let isSuitesSkippingEnabled = false - let isCodeCoverageEnabled = false - let testsToSkip = [] - const unskippableSuites = [] - let hasForcedToRunSuites = false - let hasUnskippableSuites = false - - function getTestSpan (testName, testSuite, isUnskippable, isForcedToRun) { - const testSuiteTags = { - [TEST_COMMAND]: command, - [TEST_COMMAND]: command, - [TEST_MODULE]: TEST_FRAMEWORK_NAME - } - if (testSuiteSpan) { - testSuiteTags[TEST_SUITE_ID] = testSuiteSpan.context().toSpanId() - } - if (testSessionSpan && testModuleSpan) { - testSuiteTags[TEST_SESSION_ID] = testSessionSpan.context().toTraceId() - testSuiteTags[TEST_MODULE_ID] = testModuleSpan.context().toSpanId() - } - - const { - childOf, - resource, - ...testSpanMetadata - } = getTestSpanMetadata(tracer, testName, testSuite, config) - - const codeOwners = getCodeOwnersForFilename(testSuite, codeOwnersEntries) - - if (codeOwners) { - testSpanMetadata[TEST_CODE_OWNERS] = codeOwners - } - - if (isUnskippable) { - hasUnskippableSuites = true - testSpanMetadata[TEST_ITR_UNSKIPPABLE] = 'true' - } - - if (isForcedToRun) { - hasForcedToRunSuites = true - testSpanMetadata[TEST_ITR_FORCED_RUN] = 'true' - } - - return tracer.startSpan(`${TEST_FRAMEWORK_NAME}.test`, { - childOf, - tags: { - [COMPONENT]: TEST_FRAMEWORK_NAME, - [ORIGIN_KEY]: CI_APP_ORIGIN, - ...testSpanMetadata, - ...testEnvironmentMetadata, - ...testSuiteTags - } - }) - } - - on('before:run', (details) => { - return getItrConfig(tracer, testConfiguration).then(({ err, itrConfig }) => { - if (err) { - log.error(err) - } else { - isSuitesSkippingEnabled = itrConfig.isSuitesSkippingEnabled - isCodeCoverageEnabled = itrConfig.isCodeCoverageEnabled - } - - return getSkippableTests(isSuitesSkippingEnabled, tracer, testConfiguration).then(({ err, skippableTests }) => { - if (err) { - log.error(err) - } else { - testsToSkip = skippableTests || [] - } - - // `details.specs` are test files - details.specs.forEach(({ absolute, relative }) => { - const isUnskippableSuite = isMarkedAsUnskippable({ path: absolute }) - if (isUnskippableSuite) { - unskippableSuites.push(relative) - } - }) - - const childOf = getTestParentSpan(tracer) - rootDir = getRootDir(details) - - command = getCypressCommand(details) - frameworkVersion = getCypressVersion(details) - - const testSessionSpanMetadata = getTestSessionCommonTags(command, frameworkVersion, TEST_FRAMEWORK_NAME) - const testModuleSpanMetadata = getTestModuleCommonTags(command, frameworkVersion, TEST_FRAMEWORK_NAME) - - testSessionSpan = tracer.startSpan(`${TEST_FRAMEWORK_NAME}.test_session`, { - childOf, - tags: { - [COMPONENT]: TEST_FRAMEWORK_NAME, - ...testEnvironmentMetadata, - ...testSessionSpanMetadata - } - }) - testModuleSpan = tracer.startSpan(`${TEST_FRAMEWORK_NAME}.test_module`, { - childOf: testSessionSpan, - tags: { - [COMPONENT]: TEST_FRAMEWORK_NAME, - ...testEnvironmentMetadata, - ...testModuleSpanMetadata - } - }) - return details - }) - }) - }) - on('after:spec', (spec, { tests, stats }) => { - const cypressTests = tests || [] - const finishedTests = finishedTestsByFile[spec.relative] || [] - - // Get tests that didn't go through `dd:afterEach` - // and create a skipped test span for each of them - cypressTests.filter(({ title }) => { - const cypressTestName = title.join(' ') - const isTestFinished = finishedTests.find(({ testName }) => cypressTestName === testName) - - return !isTestFinished - }).forEach(({ title }) => { - const cypressTestName = title.join(' ') - const isSkippedByItr = testsToSkip.find(test => - cypressTestName === test.name && spec.relative === test.suite - ) - const skippedTestSpan = getTestSpan(cypressTestName, spec.relative) - skippedTestSpan.setTag(TEST_STATUS, 'skip') - if (isSkippedByItr) { - skippedTestSpan.setTag(TEST_SKIPPED_BY_ITR, 'true') - } - skippedTestSpan.finish() - }) - - // Make sure that reported test statuses are the same as Cypress reports. - // This is not always the case, such as when an `after` hook fails: - // Cypress will report the last run test as failed, but we don't know that yet at `dd:afterEach` - let latestError - finishedTests.forEach((finishedTest) => { - const cypressTest = cypressTests.find(test => test.title.join(' ') === finishedTest.testName) - if (!cypressTest) { - return - } - if (cypressTest.displayError) { - latestError = new Error(cypressTest.displayError) - } - const cypressTestStatus = CYPRESS_STATUS_TO_TEST_STATUS[cypressTest.state] - // update test status - if (cypressTestStatus !== finishedTest.testStatus) { - finishedTest.testSpan.setTag(TEST_STATUS, cypressTestStatus) - finishedTest.testSpan.setTag('error', latestError) - } - finishedTest.testSpan.finish(finishedTest.finishTime) - }) - - if (testSuiteSpan) { - const status = getSuiteStatus(stats) - testSuiteSpan.setTag(TEST_STATUS, status) - - if (latestError) { - testSuiteSpan.setTag('error', latestError) - } - testSuiteSpan.finish() - testSuiteSpan = null - } - }) - - on('after:run', (suiteStats) => { - if (testSessionSpan && testModuleSpan) { - const testStatus = getSessionStatus(suiteStats) - testModuleSpan.setTag(TEST_STATUS, testStatus) - testSessionSpan.setTag(TEST_STATUS, testStatus) - - addIntelligentTestRunnerSpanTags( - testSessionSpan, - testModuleSpan, - { - isSuitesSkipped: isTestsSkipped, - isSuitesSkippingEnabled, - isCodeCoverageEnabled, - skippingType: 'test', - skippingCount: skippedTests.length, - hasForcedToRunSuites, - hasUnskippableSuites - } - ) - - testModuleSpan.finish() - testSessionSpan.finish() - - finishAllTraceSpans(testSessionSpan) - } - - return new Promise(resolve => { - const exporter = tracer._tracer._exporter - if (!exporter) { - return resolve(null) - } - if (exporter.flush) { - exporter.flush(() => { - resolve(null) - }) - } else if (exporter._writer) { - exporter._writer.flush(() => { - resolve(null) - }) - } - }) - }) - on('task', { - 'dd:testSuiteStart': (suite) => { - if (testSuiteSpan) { - return null - } - const testSuiteSpanMetadata = getTestSuiteCommonTags(command, frameworkVersion, suite, TEST_FRAMEWORK_NAME) - testSuiteSpan = tracer.startSpan(`${TEST_FRAMEWORK_NAME}.test_suite`, { - childOf: testModuleSpan, - tags: { - [COMPONENT]: TEST_FRAMEWORK_NAME, - ...testEnvironmentMetadata, - ...testSuiteSpanMetadata - } - }) - return null - }, - 'dd:beforeEach': (test) => { - const { testName, testSuite } = test - const shouldSkip = !!testsToSkip.find(test => { - return testName === test.name && testSuite === test.suite - }) - const isUnskippable = unskippableSuites.includes(testSuite) - const isForcedToRun = shouldSkip && isUnskippable - - // skip test - if (shouldSkip && !isUnskippable) { - skippedTests.push(test) - isTestsSkipped = true - return { shouldSkip: true } - } - - if (!activeSpan) { - activeSpan = getTestSpan(testName, testSuite, isUnskippable, isForcedToRun) - } - - return activeSpan ? { traceId: activeSpan.context().toTraceId() } : {} - }, - 'dd:afterEach': ({ test, coverage }) => { - const { state, error, isRUMActive, testSourceLine, testSuite, testName } = test - if (activeSpan) { - if (coverage && isCodeCoverageEnabled && tracer._tracer._exporter && tracer._tracer._exporter.exportCoverage) { - const coverageFiles = getCoveredFilenamesFromCoverage(coverage) - const relativeCoverageFiles = coverageFiles.map(file => getTestSuitePath(file, rootDir)) - const { _traceId, _spanId } = testSuiteSpan.context() - const formattedCoverage = { - sessionId: _traceId, - suiteId: _spanId, - testId: activeSpan.context()._spanId, - files: relativeCoverageFiles - } - tracer._tracer._exporter.exportCoverage(formattedCoverage) - } - const testStatus = CYPRESS_STATUS_TO_TEST_STATUS[state] - activeSpan.setTag(TEST_STATUS, testStatus) + on('before:run', cypressPlugin.beforeRun.bind(cypressPlugin)) + on('after:spec', cypressPlugin.afterSpec.bind(cypressPlugin)) + on('after:run', cypressPlugin.afterRun.bind(cypressPlugin)) + on('task', cypressPlugin.getTasks()) - if (error) { - activeSpan.setTag('error', error) - } - if (isRUMActive) { - activeSpan.setTag(TEST_IS_RUM_ACTIVE, 'true') - } - if (testSourceLine) { - activeSpan.setTag(TEST_SOURCE_START, testSourceLine) - } - const finishedTest = { - testName, - testStatus, - finishTime: activeSpan._getTime(), // we store the finish time here - testSpan: activeSpan - } - if (finishedTestsByFile[testSuite]) { - finishedTestsByFile[testSuite].push(finishedTest) - } else { - finishedTestsByFile[testSuite] = [finishedTest] - } - // test spans are finished at after:spec - } - activeSpan = null - return null - }, - 'dd:addTags': (tags) => { - if (activeSpan) { - activeSpan.addTags(tags) - } - return null - } - }) + return cypressPlugin.init(tracer, config) } diff --git a/packages/datadog-plugin-cypress/src/support.js b/packages/datadog-plugin-cypress/src/support.js index 54def0865d6..b9a739c94e4 100644 --- a/packages/datadog-plugin-cypress/src/support.js +++ b/packages/datadog-plugin-cypress/src/support.js @@ -1,4 +1,53 @@ /* eslint-disable */ +let isEarlyFlakeDetectionEnabled = false +let knownTestsForSuite = [] +let suiteTests = [] +let earlyFlakeDetectionNumRetries = 0 + +// If the test is using multi domain with cy.origin, trying to access +// window properties will result in a cross origin error. +function safeGetRum (window) { + try { + return window.DD_RUM + } catch (e) { + return null + } +} + +function isNewTest (test) { + return !knownTestsForSuite.includes(test.fullTitle()) +} + +function retryTest (test, suiteTests) { + for (let retryIndex = 0; retryIndex < earlyFlakeDetectionNumRetries; retryIndex++) { + const clonedTest = test.clone() + // TODO: signal in framework logs that this is a retry. + // TODO: Change it so these tests are allowed to fail. + // TODO: figure out if reported duration is skewed. + suiteTests.unshift(clonedTest) + clonedTest._ddIsNew = true + clonedTest._ddIsEfdRetry = true + } +} + + +const oldRunTests = Cypress.mocha.getRunner().runTests +Cypress.mocha.getRunner().runTests = function (suite, fn) { + if (!isEarlyFlakeDetectionEnabled) { + return oldRunTests.apply(this, arguments) + } + // We copy the new tests at the beginning of the suite run (runTests), so that they're run + // multiple times. + suite.tests.forEach(test => { + if (!test._ddIsNew && !test.isPending() && isNewTest(test)) { + test._ddIsNew = true + retryTest(test, suite.tests) + } + }) + + return oldRunTests.apply(this, [suite, fn]) +} + beforeEach(function () { cy.task('dd:beforeEach', { testName: Cypress.mocha.getRunner().suite.ctx.currentTest.fullTitle(), @@ -11,20 +60,29 @@ beforeEach(function () { }) }) -before(() => { - cy.task('dd:testSuiteStart', Cypress.mocha.getRootSuite().file) +before(function () { + cy.task('dd:testSuiteStart', { + testSuite: Cypress.mocha.getRootSuite().file, + testSuiteAbsolutePath: Cypress.spec && Cypress.spec.absolute + }).then((suiteConfig) => { + if (suiteConfig) { + isEarlyFlakeDetectionEnabled = suiteConfig.isEarlyFlakeDetectionEnabled + knownTestsForSuite = suiteConfig.knownTestsForSuite + earlyFlakeDetectionNumRetries = suiteConfig.earlyFlakeDetectionNumRetries + } + }) }) after(() => { cy.window().then(win => { - if (win.DD_RUM) { + if (safeGetRum(win)) { win.dispatchEvent(new Event('beforeunload')) } }) }) -afterEach(() => { +afterEach(function () { cy.window().then(win => { const currentTest = Cypress.mocha.getRunner().suite.ctx.currentTest const testInfo = { @@ -32,14 +90,22 @@ afterEach(() => { testSuite: Cypress.mocha.getRootSuite().file, state: currentTest.state, error: currentTest.err, + isNew: currentTest._ddIsNew, + isEfdRetry: currentTest._ddIsEfdRetry } try { testInfo.testSourceLine = Cypress.mocha.getRunner().currentRunnable.invocationDetails.line } catch (e) {} - if (win.DD_RUM) { + if (safeGetRum(win)) { testInfo.isRUMActive = true } - cy.task('dd:afterEach', { test: testInfo, coverage: win.__coverage__ }) + let coverage + try { + coverage = win.__coverage__ + } catch (e) { + // ignore error and continue + } + cy.task('dd:afterEach', { test: testInfo, coverage }) }) }) diff --git a/packages/datadog-plugin-cypress/test/app-10/CODEOWNERS b/packages/datadog-plugin-cypress/test/app-10/CODEOWNERS deleted file mode 100644 index fb1bef44dc2..00000000000 --- a/packages/datadog-plugin-cypress/test/app-10/CODEOWNERS +++ /dev/null @@ -1 +0,0 @@ -cypress/integration/* @datadog diff --git a/packages/datadog-plugin-cypress/test/app/CODEOWNERS b/packages/datadog-plugin-cypress/test/app/CODEOWNERS deleted file mode 100644 index fb1bef44dc2..00000000000 --- a/packages/datadog-plugin-cypress/test/app/CODEOWNERS +++ /dev/null @@ -1 +0,0 @@ -cypress/integration/* @datadog diff --git a/packages/datadog-plugin-cypress/test/index.spec.js b/packages/datadog-plugin-cypress/test/index.spec.js index 293855ca522..9143cefdc67 100644 --- a/packages/datadog-plugin-cypress/test/index.spec.js +++ b/packages/datadog-plugin-cypress/test/index.spec.js @@ -1,5 +1,4 @@ 'use strict' -const getPort = require('get-port') const { expect } = require('chai') const semver = require('semver') @@ -31,15 +30,16 @@ describe('Plugin', function () { let agentListenPort this.retries(2) withVersions('cypress', 'cypress', (version, moduleName) => { - beforeEach(function () { + beforeEach(() => { + return agent.load() + }) + beforeEach(function (done) { this.timeout(10000) - return agent.load().then(() => { - agentListenPort = agent.server.address().port - cypressExecutable = require(`../../../versions/cypress@${version}`).get() - return getPort().then(port => { - appPort = port - appServer.listen(appPort) - }) + agentListenPort = agent.server.address().port + cypressExecutable = require(`../../../versions/cypress@${version}`).get() + appServer.listen(0, () => { + appPort = appServer.address().port + done() }) }) afterEach(() => agent.close({ ritmReset: false })) @@ -49,17 +49,22 @@ describe('Plugin', function () { describe('cypress', function () { this.timeout(testTimeout) + it('instruments tests', function (done) { process.env.DD_TRACE_AGENT_PORT = agentListenPort + const testSuiteFolder = semver.intersects(version, '>=10') + ? 'app-10' + : 'app' + cypressExecutable.run({ - project: semver.intersects(version, '>=10') - ? './packages/datadog-plugin-cypress/test/app-10' : './packages/datadog-plugin-cypress/test/app', + project: `./packages/datadog-plugin-cypress/test/${testSuiteFolder}`, config: { baseUrl: `http://localhost:${appPort}` }, quiet: true, headless: true }) + agent.use(traces => { const passedTestSpan = traces[0][0] const failedTestSpan = traces[1][0] @@ -77,14 +82,15 @@ describe('Plugin', function () { [TEST_NAME]: 'can visit a page renders a hello world', [TEST_STATUS]: 'pass', [TEST_SUITE]: 'cypress/integration/integration-test.js', - [TEST_SOURCE_FILE]: 'cypress/integration/integration-test.js', + [TEST_SOURCE_FILE]: + `packages/datadog-plugin-cypress/test/${testSuiteFolder}/cypress/integration/integration-test.js`, [TEST_TYPE]: 'browser', [ORIGIN_KEY]: CI_APP_ORIGIN, [TEST_IS_RUM_ACTIVE]: 'true', - [TEST_CODE_OWNERS]: JSON.stringify(['@datadog']), [LIBRARY_VERSION]: ddTraceVersion, [COMPONENT]: 'cypress' }) + expect(passedTestSpan.meta[TEST_CODE_OWNERS]).to.contain('@DataDog') expect(passedTestSpan.meta[TEST_FRAMEWORK_VERSION]).not.to.be.undefined expect(passedTestSpan.metrics[TEST_SOURCE_START]).to.exist @@ -102,7 +108,8 @@ describe('Plugin', function () { [TEST_NAME]: 'can visit a page will fail', [TEST_STATUS]: 'fail', [TEST_SUITE]: 'cypress/integration/integration-test.js', - [TEST_SOURCE_FILE]: 'cypress/integration/integration-test.js', + [TEST_SOURCE_FILE]: + `packages/datadog-plugin-cypress/test/${testSuiteFolder}/cypress/integration/integration-test.js`, [TEST_TYPE]: 'browser', [ORIGIN_KEY]: CI_APP_ORIGIN, [ERROR_TYPE]: 'AssertionError', diff --git a/packages/datadog-plugin-dns/test/index.spec.js b/packages/datadog-plugin-dns/test/index.spec.js index 5b2ab06ecec..1457bb869d8 100644 --- a/packages/datadog-plugin-dns/test/index.spec.js +++ b/packages/datadog-plugin-dns/test/index.spec.js @@ -5,233 +5,236 @@ const { promisify } = require('util') const { storage } = require('../../datadog-core') const { ERROR_TYPE, ERROR_MESSAGE } = require('../../dd-trace/src/constants') +const PLUGINS = ['dns', 'node:dns'] + describe('Plugin', () => { let dns let tracer + PLUGINS.forEach(plugin => { + describe(plugin, () => { + afterEach(() => { + return agent.close() + }) - describe('dns', () => { - afterEach(() => { - return agent.close() - }) - - beforeEach(() => { - return agent.load('dns') - .then(() => { - dns = require('dns') - tracer = require('../../dd-trace') - }) - }) - - it('should instrument lookup', done => { - agent - .use(traces => { - expect(traces[0][0]).to.deep.include({ - name: 'dns.lookup', - service: 'test', - resource: 'localhost' + beforeEach(() => { + return agent.load('dns') + .then(() => { + dns = require(plugin) + tracer = require('../../dd-trace') }) - expect(traces[0][0].meta).to.deep.include({ - 'component': 'dns', - 'span.kind': 'client', - 'dns.hostname': 'localhost', - 'dns.address': '127.0.0.1' - }) - }) - .then(done) - .catch(done) - - dns.lookup('localhost', 4, (err, address, family) => err && done(err)) - }) + }) - it('should instrument lookup with all addresses', done => { - agent - .use(traces => { - expect(traces[0][0]).to.deep.include({ - name: 'dns.lookup', - service: 'test', - resource: 'localhost' - }) - expect(traces[0][0].meta).to.deep.include({ - 'component': 'dns', - 'span.kind': 'client', - 'dns.hostname': 'localhost', - 'dns.address': '127.0.0.1', - 'dns.addresses': '127.0.0.1,::1' - }) - }) - .then(done) - .catch(done) + it('should instrument lookup', done => { + agent + .use(traces => { + expect(traces[0][0]).to.deep.include({ + name: 'dns.lookup', + service: 'test', + resource: 'localhost' + }) + expect(traces[0][0].meta).to.deep.include({ + component: 'dns', + 'span.kind': 'client', + 'dns.hostname': 'localhost', + 'dns.address': '127.0.0.1' + }) + }) + .then(done) + .catch(done) + + dns.lookup('localhost', 4, (err, address, family) => err && done(err)) + }) - dns.lookup('localhost', { all: true }, (err, address, family) => err && done(err)) - }) + it('should instrument lookup with all addresses', done => { + agent + .use(traces => { + expect(traces[0][0]).to.deep.include({ + name: 'dns.lookup', + service: 'test', + resource: 'localhost' + }) + expect(traces[0][0].meta).to.deep.include({ + component: 'dns', + 'span.kind': 'client', + 'dns.hostname': 'localhost', + 'dns.address': '127.0.0.1', + 'dns.addresses': '127.0.0.1,::1' + }) + }) + .then(done) + .catch(done) + + dns.lookup('localhost', { all: true }, (err, address, family) => err && done(err)) + }) - it('should instrument errors correctly', done => { - agent - .use(traces => { - expect(traces[0][0]).to.deep.include({ - name: 'dns.lookup', - service: 'test', - resource: 'fakedomain.faketld', - error: 1 - }) - expect(traces[0][0].meta).to.deep.include({ - 'component': 'dns', - 'span.kind': 'client', - 'dns.hostname': 'fakedomain.faketld', - [ERROR_TYPE]: 'Error', - [ERROR_MESSAGE]: 'getaddrinfo ENOTFOUND fakedomain.faketld' - }) + it('should instrument errors correctly', done => { + agent + .use(traces => { + expect(traces[0][0]).to.deep.include({ + name: 'dns.lookup', + service: 'test', + resource: 'fakedomain.faketld', + error: 1 + }) + expect(traces[0][0].meta).to.deep.include({ + component: 'dns', + 'span.kind': 'client', + 'dns.hostname': 'fakedomain.faketld', + [ERROR_TYPE]: 'Error', + [ERROR_MESSAGE]: 'getaddrinfo ENOTFOUND fakedomain.faketld' + }) + }) + .then(done) + .catch(done) + + dns.lookup('fakedomain.faketld', 4, (err, address, family) => { + expect(err).to.not.be.null }) - .then(done) - .catch(done) + }) - dns.lookup('fakedomain.faketld', 4, (err, address, family) => { - expect(err).to.not.be.null + it('should instrument lookupService', done => { + agent + .use(traces => { + expect(traces[0][0]).to.deep.include({ + name: 'dns.lookup_service', + service: 'test', + resource: '127.0.0.1:22' + }) + expect(traces[0][0].meta).to.deep.include({ + component: 'dns', + 'span.kind': 'client', + 'dns.address': '127.0.0.1' + }) + expect(traces[0][0].metrics).to.deep.include({ + 'dns.port': 22 + }) + }) + .then(done) + .catch(done) + + dns.lookupService('127.0.0.1', 22, err => err && done(err)) }) - }) - it('should instrument lookupService', done => { - agent - .use(traces => { - expect(traces[0][0]).to.deep.include({ - name: 'dns.lookup_service', - service: 'test', - resource: '127.0.0.1:22' - }) - expect(traces[0][0].meta).to.deep.include({ - 'component': 'dns', - 'span.kind': 'client', - 'dns.address': '127.0.0.1' - }) - expect(traces[0][0].metrics).to.deep.include({ - 'dns.port': 22 - }) - }) - .then(done) - .catch(done) + it('should instrument resolve', done => { + agent + .use(traces => { + expect(traces[0][0]).to.deep.include({ + name: 'dns.resolve', + service: 'test', + resource: 'A lvh.me' + }) + expect(traces[0][0].meta).to.deep.include({ + component: 'dns', + 'span.kind': 'client', + 'dns.hostname': 'lvh.me', + 'dns.rrtype': 'A' + }) + }) + .then(done) + .catch(done) + + dns.resolve('lvh.me', err => err && done(err)) + }) - dns.lookupService('127.0.0.1', 22, err => err && done(err)) - }) + it('should instrument resolve shorthands', done => { + agent + .use(traces => { + expect(traces[0][0]).to.deep.include({ + name: 'dns.resolve', + service: 'test', + resource: 'ANY localhost' + }) + expect(traces[0][0].meta).to.deep.include({ + component: 'dns', + 'span.kind': 'client', + 'dns.hostname': 'localhost', + 'dns.rrtype': 'ANY' + }) + }) + .then(done) + .catch(done) + + dns.resolveAny('localhost', () => done()) + }) - it('should instrument resolve', done => { - agent - .use(traces => { - expect(traces[0][0]).to.deep.include({ - name: 'dns.resolve', - service: 'test', - resource: 'A lvh.me' - }) - expect(traces[0][0].meta).to.deep.include({ - 'component': 'dns', - 'span.kind': 'client', - 'dns.hostname': 'lvh.me', - 'dns.rrtype': 'A' - }) - }) - .then(done) - .catch(done) + it('should instrument reverse', done => { + agent + .use(traces => { + expect(traces[0][0]).to.deep.include({ + name: 'dns.reverse', + service: 'test', + resource: '127.0.0.1' + }) + expect(traces[0][0].meta).to.deep.include({ + component: 'dns', + 'span.kind': 'client', + 'dns.ip': '127.0.0.1' + }) + }) + .then(done) + .catch(done) + + dns.reverse('127.0.0.1', err => err && done(err)) + }) - dns.resolve('lvh.me', err => err && done(err)) - }) + it('should preserve the parent scope in the callback', done => { + const span = tracer.startSpan('dummySpan', {}) - it('should instrument resolve shorthands', done => { - agent - .use(traces => { - expect(traces[0][0]).to.deep.include({ - name: 'dns.resolve', - service: 'test', - resource: 'ANY lvh.me' - }) - expect(traces[0][0].meta).to.deep.include({ - 'component': 'dns', - 'span.kind': 'client', - 'dns.hostname': 'lvh.me', - 'dns.rrtype': 'ANY' - }) - }) - .then(done) - .catch(done) + tracer.scope().activate(span, () => { + dns.lookup('localhost', 4, (err) => { + if (err) return done(err) - dns.resolveAny('lvh.me', err => err && done(err)) - }) + expect(tracer.scope().active()).to.equal(span) - it('should instrument reverse', done => { - agent - .use(traces => { - expect(traces[0][0]).to.deep.include({ - name: 'dns.reverse', - service: 'test', - resource: '127.0.0.1' - }) - expect(traces[0][0].meta).to.deep.include({ - 'component': 'dns', - 'span.kind': 'client', - 'dns.ip': '127.0.0.1' + done() }) }) - .then(done) - .catch(done) - - dns.reverse('127.0.0.1', err => err && done(err)) - }) - - it('should preserve the parent scope in the callback', done => { - const span = tracer.startSpan('dummySpan', {}) - - tracer.scope().activate(span, () => { - dns.lookup('localhost', 4, (err) => { - if (err) return done(err) + }) - expect(tracer.scope().active()).to.equal(span) + it('should work with promisify', () => { + const lookup = promisify(dns.lookup) - done() + return lookup('localhost', 4).then(({ address, family }) => { + expect(address).to.equal('127.0.0.1') + expect(family).to.equal(4) }) }) - }) - - it('should work with promisify', () => { - const lookup = promisify(dns.lookup) - return lookup('localhost', 4).then(({ address, family }) => { - expect(address).to.equal('127.0.0.1') - expect(family).to.equal(4) + it('should instrument Resolver', done => { + const resolver = new dns.Resolver() + + agent + .use(traces => { + expect(traces[0][0]).to.deep.include({ + name: 'dns.resolve', + service: 'test', + resource: 'A lvh.me' + }) + expect(traces[0][0].meta).to.deep.include({ + component: 'dns', + 'dns.hostname': 'lvh.me', + 'dns.rrtype': 'A' + }) + }) + .then(done) + .catch(done) + + resolver.resolve('lvh.me', err => err && done(err)) }) - }) - it('should instrument Resolver', done => { - const resolver = new dns.Resolver() + it('should skip instrumentation for noop context', done => { + const resolver = new dns.Resolver() + const timer = setTimeout(done, 200) - agent - .use(traces => { - expect(traces[0][0]).to.deep.include({ - name: 'dns.resolve', - service: 'test', - resource: 'A lvh.me' + agent + .use(() => { + done(new Error('Resolve was traced.')) + clearTimeout(timer) }) - expect(traces[0][0].meta).to.deep.include({ - 'component': 'dns', - 'dns.hostname': 'lvh.me', - 'dns.rrtype': 'A' - }) - }) - .then(done) - .catch(done) - - resolver.resolve('lvh.me', err => err && done(err)) - }) - - it('should skip instrumentation for noop context', done => { - const resolver = new dns.Resolver() - const timer = setTimeout(done, 200) - agent - .use(() => { - done(new Error('Resolve was traced.')) - clearTimeout(timer) + storage.run({ noop: true }, () => { + resolver.resolve('lvh.me', () => {}) }) - - storage.run({ noop: true }, () => { - resolver.resolve('lvh.me', () => {}) }) }) }) diff --git a/packages/datadog-plugin-dns/test/integration-test/client.spec.js b/packages/datadog-plugin-dns/test/integration-test/client.spec.js index b1578dcce79..4a2f5113458 100644 --- a/packages/datadog-plugin-dns/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-dns/test/integration-test/client.spec.js @@ -16,7 +16,7 @@ describe('esm', () => { before(async function () { this.timeout(20000) sandbox = await createSandbox([], false, [ - `./packages/datadog-plugin-dns/test/integration-test/*`]) + './packages/datadog-plugin-dns/test/integration-test/*']) }) after(async () => { diff --git a/packages/datadog-plugin-elasticsearch/test/integration-test/client.spec.js b/packages/datadog-plugin-elasticsearch/test/integration-test/client.spec.js index a676b5ab8bb..eacd384c033 100644 --- a/packages/datadog-plugin-elasticsearch/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-elasticsearch/test/integration-test/client.spec.js @@ -17,7 +17,7 @@ describe('esm', () => { before(async function () { this.timeout(20000) sandbox = await createSandbox([`'@elastic/elasticsearch@${version}'`], false, [ - `./packages/datadog-plugin-elasticsearch/test/integration-test/*`]) + './packages/datadog-plugin-elasticsearch/test/integration-test/*']) }) after(async () => { diff --git a/packages/datadog-plugin-elasticsearch/test/naming.js b/packages/datadog-plugin-elasticsearch/test/naming.js index d7c1fc2a952..737c724d81b 100644 --- a/packages/datadog-plugin-elasticsearch/test/naming.js +++ b/packages/datadog-plugin-elasticsearch/test/naming.js @@ -14,6 +14,6 @@ const rawExpectedSchema = { } module.exports = { - rawExpectedSchema: rawExpectedSchema, + rawExpectedSchema, expectedSchema: resolveNaming(rawExpectedSchema) } diff --git a/packages/datadog-plugin-express/test/index.spec.js b/packages/datadog-plugin-express/test/index.spec.js index 14299a50280..55a608f4adf 100644 --- a/packages/datadog-plugin-express/test/index.spec.js +++ b/packages/datadog-plugin-express/test/index.spec.js @@ -2,7 +2,6 @@ const { AsyncLocalStorage } = require('async_hooks') const axios = require('axios') -const getPort = require('get-port') const { ERROR_MESSAGE, ERROR_STACK, ERROR_TYPE } = require('../../dd-trace/src/constants') const agent = require('../../dd-trace/test/plugins/agent') const plugin = require('../src') @@ -45,7 +44,8 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port const timer = setTimeout(done, 100) agent.use(() => { @@ -53,11 +53,9 @@ describe('Plugin', () => { done(new Error('Agent received an unexpected trace.')) }) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -65,17 +63,18 @@ describe('Plugin', () => { const app = express() app.use(() => { throw new Error('boom') }) + // eslint-disable-next-line n/handle-callback-err app.use((err, req, res, next) => { res.status(200).send() }) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .then(() => done()) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/user`) + .then(() => done()) + .catch(done) }) }) }) @@ -100,7 +99,9 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -113,15 +114,14 @@ describe('Plugin', () => { expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user`) expect(spans[0].meta).to.have.property('http.method', 'GET') expect(spans[0].meta).to.have.property('http.status_code', '200') + expect(spans[0].meta).to.have.property('http.route', '/user') }) .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -135,7 +135,9 @@ describe('Plugin', () => { app.use('/app', router) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -152,11 +154,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/1`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app/user/1`) + .catch(done) }) }) @@ -172,7 +172,9 @@ describe('Plugin', () => { app.use('/app', router) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -189,11 +191,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/1`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app/user/1`) + .catch(done) }) }) @@ -208,7 +208,9 @@ describe('Plugin', () => { app.use(function named (req, res, next) { next() }) app.use('/app', router) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -244,11 +246,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/1`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app/user/1`) + .catch(done) }) }) @@ -270,7 +270,9 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -285,11 +287,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user/1`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user/1`) + .catch(done) }) }) @@ -309,12 +309,15 @@ describe('Plugin', () => { next = _next }) app.use(() => { throw error }) + // eslint-disable-next-line n/handle-callback-err app.use((err, req, res, next) => next()) app.get('/user/:id', (req, res) => { res.status(200).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -328,11 +331,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user/1`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user/1`) + .catch(done) }) }) @@ -346,7 +347,9 @@ describe('Plugin', () => { app.use('/app', router) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -356,11 +359,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/1`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app/user/1`) + .catch(done) }) }) @@ -374,7 +375,9 @@ describe('Plugin', () => { app.use('/app', router) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -384,11 +387,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/1`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app/user/1`) + .catch(done) }) }) @@ -408,7 +409,9 @@ describe('Plugin', () => { app.use('/foo/bar', (req, res, next) => next()) app.use('/foo', router) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -418,11 +421,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/foo/bar`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/foo/bar`) + .catch(done) }) }) @@ -436,7 +437,9 @@ describe('Plugin', () => { app.use('/app', router) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -446,11 +449,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/1`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app/user/1`) + .catch(done) }) }) @@ -465,7 +466,9 @@ describe('Plugin', () => { app.use('/app', router) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -475,11 +478,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/1`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app/user/1`) + .catch(done) }) }) @@ -493,7 +494,9 @@ describe('Plugin', () => { app.use('/parent', childApp) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -505,11 +508,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/parent/child`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/parent/child`) + .catch(done) }) }) @@ -532,12 +533,12 @@ describe('Plugin', () => { done() }) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/1`) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/app/user/1`) + .catch(done) }) }) @@ -561,7 +562,9 @@ describe('Plugin', () => { app.use('/app', router) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -571,11 +574,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/123`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app/user/123`) + .catch(done) }) }) @@ -593,7 +594,9 @@ describe('Plugin', () => { app.use('/app', router) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -603,11 +606,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/123`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app/user/123`) + .catch(done) }) }) @@ -622,7 +623,9 @@ describe('Plugin', () => { res.status(200).send(error.message) }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -632,11 +635,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app`) + .catch(done) }) }) @@ -657,7 +658,9 @@ describe('Plugin', () => { app.use('/v1', routerA) app.use('/v1', routerB) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -667,11 +670,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/v1/a`) - .catch(() => {}) - }) + axios + .get(`http://localhost:${port}/v1/a`) + .catch(() => {}) }) }) @@ -687,7 +688,9 @@ describe('Plugin', () => { res.status(200).send(req.body) }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -697,11 +700,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app`) + .catch(done) }) }) @@ -723,7 +724,9 @@ describe('Plugin', () => { res.status(200).send('') }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -733,11 +736,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/foo/bar`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/foo/bar`) + .catch(done) }) }) @@ -759,7 +760,9 @@ describe('Plugin', () => { res.status(200).send('') }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -769,11 +772,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/foo/bar`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/foo/bar`) + .catch(done) }) }) @@ -797,7 +798,9 @@ describe('Plugin', () => { res.status(200).send('') }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -807,11 +810,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/foo/bar`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/foo/bar`) + .catch(done) }) }) @@ -828,7 +829,9 @@ describe('Plugin', () => { app.use('/v1', router) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -838,11 +841,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/v1/a`) - .catch(() => {}) - }) + axios + .get(`http://localhost:${port}/v1/a`) + .catch(() => {}) }) }) @@ -871,12 +872,12 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app`) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/app`) + .catch(done) }) }) @@ -885,7 +886,9 @@ describe('Plugin', () => { app.use((req, res, next) => res.status(200).send()) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -895,11 +898,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app`) + .catch(done) }) }) @@ -925,11 +926,11 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/user`) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios.get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -955,11 +956,11 @@ describe('Plugin', () => { } ) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/user`) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios.get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -977,7 +978,9 @@ describe('Plugin', () => { app.use('/app', router) app.use('/bar', (req, res, next) => next()) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -987,10 +990,8 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/app/user/123`) - .catch(done) - }) + axios.get(`http://localhost:${port}/app/user/123`) + .catch(done) }) }) @@ -1001,7 +1002,9 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent.use(traces => { const spans = sort(traces[0]) @@ -1011,17 +1014,15 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - headers: { - 'x-datadog-trace-id': '1234', - 'x-datadog-parent-id': '5678', - 'ot-baggage-foo': 'bar' - } - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + headers: { + 'x-datadog-trace-id': '1234', + 'x-datadog-parent-id': '5678', + 'ot-baggage-foo': 'bar' + } + }) + .catch(done) }) }) @@ -1036,7 +1037,9 @@ describe('Plugin', () => { res.status(500).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent.use(traces => { const spans = sort(traces[0]) @@ -1048,13 +1051,11 @@ describe('Plugin', () => { done() }) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) @@ -1070,7 +1071,9 @@ describe('Plugin', () => { throw new Error('boom') }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent.use(traces => { const spans = sort(traces[0]) @@ -1082,13 +1085,11 @@ describe('Plugin', () => { done() }) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 400 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 400 + }) + .catch(done) }) }) @@ -1098,7 +1099,9 @@ describe('Plugin', () => { app.use(() => { throw error }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -1113,13 +1116,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) @@ -1128,9 +1129,12 @@ describe('Plugin', () => { const error = new Error('boom') app.use((req, res, next) => next(error)) + // eslint-disable-next-line n/handle-callback-err app.use((error, req, res, next) => res.status(500).send()) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -1149,13 +1153,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) @@ -1164,9 +1166,12 @@ describe('Plugin', () => { const error = new Error('boom') app.use((req, res) => { throw error }) + // eslint-disable-next-line n/handle-callback-err app.use((error, req, res, next) => res.status(500).send()) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -1185,13 +1190,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) @@ -1202,7 +1205,9 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -1213,11 +1218,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -1256,12 +1259,46 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/user`) + .catch(done) + }) + }) + + it('should handle 404 errors', done => { + const app = express() + + app.use((req, res, next) => { + next() + }) + + app.get('/does-exist', (req, res) => { + res.status(200).send('hi') + }) + + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + agent.use(traces => { + const spans = sort(traces[0]) + + expect(spans[0]).to.have.property('error', 0) + expect(spans[0]).to.have.property('resource', 'GET') + expect(spans[0].meta).to.have.property('http.status_code', '404') + expect(spans[0].meta).to.have.property('component', 'express') + expect(spans[0].meta).to.not.have.property('http.route') + + done() }) + + axios + .get(`http://localhost:${port}/does-not-exist`, { + validateStatus: status => status === 404 + }) + .catch(done) }) }) @@ -1281,7 +1318,9 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -1298,10 +1337,8 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/dd`) - .catch(done) - }) + axios.get(`http://localhost:${port}/dd`) + .catch(done) }) }) @@ -1316,7 +1353,9 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -1328,10 +1367,8 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/dd`) - .catch(done) - }) + axios.get(`http://localhost:${port}/dd`) + .catch(done) }) }) }) @@ -1362,7 +1399,9 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -1372,11 +1411,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -1387,7 +1424,9 @@ describe('Plugin', () => { res.status(400).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -1397,13 +1436,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 400 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 400 + }) + .catch(done) }) }) @@ -1414,7 +1451,9 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -1424,13 +1463,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - headers: { 'User-Agent': 'test' } - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + headers: { 'User-Agent': 'test' } + }) + .catch(done) }) }) @@ -1441,7 +1478,8 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port const spy = sinon.spy() agent @@ -1457,11 +1495,9 @@ describe('Plugin', () => { } }, 100) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/health`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/health`) + .catch(done) }) }) }) @@ -1501,11 +1537,11 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/user`) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios.get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -1520,7 +1556,9 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -1531,10 +1569,8 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/user`) - .catch(done) - }) + axios.get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -1549,7 +1585,9 @@ describe('Plugin', () => { res.status(500).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent.use(traces => { const spans = sort(traces[0]) @@ -1561,13 +1599,11 @@ describe('Plugin', () => { done() }) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) @@ -1583,7 +1619,9 @@ describe('Plugin', () => { throw new Error('boom') }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -1596,13 +1634,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 400 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 400 + }) + .catch(done) }) }) @@ -1611,9 +1647,12 @@ describe('Plugin', () => { const error = new Error('boom') app.use((req, res) => { throw error }) + // eslint-disable-next-line n/handle-callback-err app.use((error, req, res, next) => res.status(500).send()) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -1627,13 +1666,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) @@ -1643,7 +1680,9 @@ describe('Plugin', () => { app.use(() => { throw error }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -1658,13 +1697,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) }) diff --git a/packages/datadog-plugin-express/test/integration-test/client.spec.js b/packages/datadog-plugin-express/test/integration-test/client.spec.js index ec250439766..a5c08d60ecb 100644 --- a/packages/datadog-plugin-express/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-express/test/integration-test/client.spec.js @@ -17,7 +17,7 @@ describe('esm', () => { before(async function () { this.timeout(50000) sandbox = await createSandbox([`'express@${version}'`], false, - [`./packages/datadog-plugin-express/test/integration-test/*`]) + ['./packages/datadog-plugin-express/test/integration-test/*']) }) after(async function () { diff --git a/packages/datadog-plugin-express/test/leak.js b/packages/datadog-plugin-express/test/leak.js deleted file mode 100644 index cf52b1b19d6..00000000000 --- a/packages/datadog-plugin-express/test/leak.js +++ /dev/null @@ -1,29 +0,0 @@ -'use strict' - -require('../../dd-trace') - .init({ plugins: false, sampleRate: 0 }) - .use('express') - -const test = require('tape') -const express = require(`../../../versions/express`).get() -const axios = require('axios') -const getPort = require('get-port') -const profile = require('../../dd-trace/test/profile') - -test('express plugin should not leak', t => { - getPort().then(port => { - const app = express() - - app.use((req, res) => { - res.status(200).send() - }) - - const listener = app.listen(port, '127.0.0.1', () => { - profile(t, operation).then(() => listener.close()) - - function operation (done) { - axios.get(`http://localhost:${port}`).then(done) - } - }) - }) -}) diff --git a/packages/datadog-plugin-fastify/src/code_origin.js b/packages/datadog-plugin-fastify/src/code_origin.js new file mode 100644 index 00000000000..3e6f58d5624 --- /dev/null +++ b/packages/datadog-plugin-fastify/src/code_origin.js @@ -0,0 +1,31 @@ +'use strict' + +const { entryTag } = require('../../datadog-code-origin') +const Plugin = require('../../dd-trace/src/plugins/plugin') +const web = require('../../dd-trace/src/plugins/util/web') + +const kCodeOriginForSpansTagsSym = Symbol('datadog.codeOriginForSpansTags') + +class FastifyCodeOriginForSpansPlugin extends Plugin { + static get id () { + return 'fastify' + } + + constructor (...args) { + super(...args) + + this.addSub('apm:fastify:request:handle', ({ req, routeConfig }) => { + const tags = routeConfig?.[kCodeOriginForSpansTagsSym] + if (!tags) return + const context = web.getContext(req) + context.span?.addTags(tags) + }) + + this.addSub('apm:fastify:route:added', ({ routeOptions, onRoute }) => { + if (!routeOptions.config) routeOptions.config = {} + routeOptions.config[kCodeOriginForSpansTagsSym] = entryTag(onRoute) + }) + } +} + +module.exports = FastifyCodeOriginForSpansPlugin diff --git a/packages/datadog-plugin-fastify/src/index.js b/packages/datadog-plugin-fastify/src/index.js index 6b4768279f8..18371458346 100644 --- a/packages/datadog-plugin-fastify/src/index.js +++ b/packages/datadog-plugin-fastify/src/index.js @@ -1,18 +1,16 @@ 'use strict' -const RouterPlugin = require('../../datadog-plugin-router/src') +const FastifyTracingPlugin = require('./tracing') +const FastifyCodeOriginForSpansPlugin = require('./code_origin') +const CompositePlugin = require('../../dd-trace/src/plugins/composite') -class FastifyPlugin extends RouterPlugin { - static get id () { - return 'fastify' - } - - constructor (...args) { - super(...args) - - this.addSub('apm:fastify:request:handle', ({ req }) => { - this.setFramework(req, 'fastify', this.config) - }) +class FastifyPlugin extends CompositePlugin { + static get id () { return 'fastify' } + static get plugins () { + return { + tracing: FastifyTracingPlugin, + codeOriginForSpans: FastifyCodeOriginForSpansPlugin + } } } diff --git a/packages/datadog-plugin-fastify/src/tracing.js b/packages/datadog-plugin-fastify/src/tracing.js new file mode 100644 index 00000000000..90b2e5e8451 --- /dev/null +++ b/packages/datadog-plugin-fastify/src/tracing.js @@ -0,0 +1,19 @@ +'use strict' + +const RouterPlugin = require('../../datadog-plugin-router/src') + +class FastifyTracingPlugin extends RouterPlugin { + static get id () { + return 'fastify' + } + + constructor (...args) { + super(...args) + + this.addSub('apm:fastify:request:handle', ({ req }) => { + this.setFramework(req, 'fastify', this.config) + }) + } +} + +module.exports = FastifyTracingPlugin diff --git a/packages/datadog-plugin-fastify/test/code_origin.spec.js b/packages/datadog-plugin-fastify/test/code_origin.spec.js new file mode 100644 index 00000000000..711c2ffff6c --- /dev/null +++ b/packages/datadog-plugin-fastify/test/code_origin.spec.js @@ -0,0 +1,216 @@ +'use strict' + +const axios = require('axios') +const semver = require('semver') +const agent = require('../../dd-trace/test/plugins/agent') +const { NODE_MAJOR } = require('../../../version') + +const host = 'localhost' + +describe('Plugin', () => { + let fastify + let app + + describe('fastify', () => { + withVersions('fastify', 'fastify', (version, _, specificVersion) => { + if (NODE_MAJOR <= 18 && semver.satisfies(specificVersion, '>=5')) return + + afterEach(() => { + app.close() + }) + + withExports('fastify', version, ['default', 'fastify'], '>=3', getExport => { + describe('with tracer config codeOriginForSpans.enabled: true', () => { + if (semver.satisfies(specificVersion, '<4')) return // TODO: Why doesn't it work on older versions? + + before(() => { + return agent.load( + ['fastify', 'find-my-way', 'http'], + [{}, {}, { client: false }], + { codeOriginForSpans: { enabled: true } } + ) + }) + + after(() => { + return agent.close({ ritmReset: false }) + }) + + beforeEach(() => { + fastify = getExport() + app = fastify() + + if (semver.intersects(version, '>=3')) { + return app.register(require('../../../versions/middie').get()) + } + }) + + it('should add code_origin tag on entry spans when feature is enabled', done => { + let routeRegisterLine + + // Wrap in a named function to have at least one frame with a function name + function wrapperFunction () { + routeRegisterLine = getNextLineNumber() + app.get('/user', function userHandler (request, reply) { + reply.send() + }) + } + + const callWrapperLine = getNextLineNumber() + wrapperFunction() + + app.listen(() => { + const port = app.server.address().port + + agent + .use(traces => { + const spans = traces[0] + const tags = spans[0].meta + + expect(tags).to.have.property('_dd.code_origin.type', 'entry') + + expect(tags).to.have.property('_dd.code_origin.frames.0.file', __filename) + expect(tags).to.have.property('_dd.code_origin.frames.0.line', routeRegisterLine) + expect(tags).to.have.property('_dd.code_origin.frames.0.column').to.match(/^\d+$/) + expect(tags).to.have.property('_dd.code_origin.frames.0.method', 'wrapperFunction') + expect(tags).to.not.have.property('_dd.code_origin.frames.0.type') + + expect(tags).to.have.property('_dd.code_origin.frames.1.file', __filename) + expect(tags).to.have.property('_dd.code_origin.frames.1.line', callWrapperLine) + expect(tags).to.have.property('_dd.code_origin.frames.1.column').to.match(/^\d+$/) + expect(tags).to.not.have.property('_dd.code_origin.frames.1.method') + expect(tags).to.have.property('_dd.code_origin.frames.1.type', 'Context') + + expect(tags).to.not.have.property('_dd.code_origin.frames.2.file') + }) + .then(done) + .catch(done) + + axios + .get(`http://localhost:${port}/user`) + .catch(done) + }) + }) + + it('should point to where actual route handler is configured, not the prefix', done => { + let routeRegisterLine + + app.register(function v1Handler (app, opts, done) { + routeRegisterLine = getNextLineNumber() + app.get('/user', function userHandler (request, reply) { + reply.send() + }) + done() + }, { prefix: '/v1' }) + + app.listen(() => { + const port = app.server.address().port + + agent + .use(traces => { + const spans = traces[0] + const tags = spans[0].meta + + expect(tags).to.have.property('_dd.code_origin.type', 'entry') + + expect(tags).to.have.property('_dd.code_origin.frames.0.file', __filename) + expect(tags).to.have.property('_dd.code_origin.frames.0.line', routeRegisterLine) + expect(tags).to.have.property('_dd.code_origin.frames.0.column').to.match(/^\d+$/) + expect(tags).to.have.property('_dd.code_origin.frames.0.method', 'v1Handler') + expect(tags).to.not.have.property('_dd.code_origin.frames.0.type') + + expect(tags).to.not.have.property('_dd.code_origin.frames.1.file') + }) + .then(done) + .catch(done) + + axios + .get(`http://localhost:${port}/v1/user`) + .catch(done) + }) + }) + + it('should point to route handler even if passed through a middleware', function testCase (done) { + app.use(function middleware (req, res, next) { + next() + }) + + const routeRegisterLine = getNextLineNumber() + app.get('/user', function userHandler (request, reply) { + reply.send() + }) + + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + + agent + .use(traces => { + const spans = traces[0] + const tags = spans[0].meta + + expect(tags).to.have.property('_dd.code_origin.type', 'entry') + + expect(tags).to.have.property('_dd.code_origin.frames.0.file', __filename) + expect(tags).to.have.property('_dd.code_origin.frames.0.line', routeRegisterLine) + expect(tags).to.have.property('_dd.code_origin.frames.0.column').to.match(/^\d+$/) + expect(tags).to.have.property('_dd.code_origin.frames.0.method', 'testCase') + expect(tags).to.have.property('_dd.code_origin.frames.0.type', 'Context') + + expect(tags).to.not.have.property('_dd.code_origin.frames.1.file') + }) + .then(done) + .catch(done) + + axios + .get(`http://localhost:${port}/user`) + .catch(done) + }) + }) + + // TODO: In Fastify, the route is resolved before the middleware is called, so we actually can get the line + // number of where the route handler is defined. However, this might not be the right choice and it might be + // better to point to the middleware. + it.skip('should point to middleware if middleware responds early', function testCase (done) { + const middlewareRegisterLine = getNextLineNumber() + app.use(function middleware (req, res, next) { + res.end() + }) + + app.get('/user', function userHandler (request, reply) { + reply.send() + }) + + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + + agent + .use(traces => { + const spans = traces[0] + const tags = spans[0].meta + + expect(tags).to.have.property('_dd.code_origin.type', 'entry') + + expect(tags).to.have.property('_dd.code_origin.frames.0.file', __filename) + expect(tags).to.have.property('_dd.code_origin.frames.0.line', middlewareRegisterLine) + expect(tags).to.have.property('_dd.code_origin.frames.0.column').to.match(/^\d+$/) + expect(tags).to.have.property('_dd.code_origin.frames.0.method', 'testCase') + expect(tags).to.have.property('_dd.code_origin.frames.0.type', 'Context') + + expect(tags).to.not.have.property('_dd.code_origin.frames.1.file') + }) + .then(done) + .catch(done) + + axios + .get(`http://localhost:${port}/user`) + .catch(done) + }) + }) + }) + }) + }) + }) +}) + +function getNextLineNumber () { + return String(Number(new Error().stack.split('\n')[2].match(/:(\d+):/)[1]) + 1) +} diff --git a/packages/datadog-plugin-fastify/test/integration-test/client.spec.js b/packages/datadog-plugin-fastify/test/integration-test/client.spec.js index 4dce20e0255..6a04cf6912b 100644 --- a/packages/datadog-plugin-fastify/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-fastify/test/integration-test/client.spec.js @@ -1,5 +1,5 @@ 'use strict' - +const semver = require('semver') const { FakeAgent, createSandbox, @@ -8,19 +8,21 @@ const { spawnPluginIntegrationTestProc } = require('../../../../integration-tests/helpers') const { assert } = require('chai') +const { NODE_MAJOR } = require('../../../../version') describe('esm', () => { let agent let proc let sandbox - // TODO: fastify instrumentation breaks with esm for version 4.23.2 but works for commonJS, - // fix it and change the versions tested - withVersions('fastify', 'fastify', '^3', version => { + // skip older versions of fastify due to syntax differences + withVersions('fastify', 'fastify', '>=3', (version, _, specificVersion) => { + if (NODE_MAJOR <= 18 && semver.satisfies(specificVersion, '>=5')) return + before(async function () { this.timeout(20000) sandbox = await createSandbox([`'fastify@${version}'`], false, - [`./packages/datadog-plugin-fastify/test/integration-test/*`]) + ['./packages/datadog-plugin-fastify/test/integration-test/*']) }) after(async () => { diff --git a/packages/datadog-plugin-fastify/test/integration-test/helper.mjs b/packages/datadog-plugin-fastify/test/integration-test/helper.mjs index 2c9c2ee8da0..e818008c977 100644 --- a/packages/datadog-plugin-fastify/test/integration-test/helper.mjs +++ b/packages/datadog-plugin-fastify/test/integration-test/helper.mjs @@ -4,7 +4,7 @@ export async function createAndStartServer (app) { }) try { - await app.listen(0) + await app.listen({ port: 0 }) const address = app.server.address() const port = address.port process.send({ port }) diff --git a/packages/datadog-plugin-fastify/test/suite.js b/packages/datadog-plugin-fastify/test/suite.js index 2033b6e6de1..bbb0218b894 100644 --- a/packages/datadog-plugin-fastify/test/suite.js +++ b/packages/datadog-plugin-fastify/test/suite.js @@ -1,9 +1,10 @@ 'use strict' -// const suiteTest = require('../../dd-trace/test/plugins/suite') -// suiteTest({ -// modName: 'fastify', -// repoUrl: 'fastify/fastify', -// commitish: 'latest', -// testCmd: 'node_modules/.bin/tap -J test/*.test.js test/*/*.test.js --no-coverage --no-check-coverage' -// }) +const suiteTest = require('../../dd-trace/test/plugins/suite') + +suiteTest({ + modName: 'fastify', + repoUrl: 'fastify/fastify', + commitish: 'latest', + testCmd: 'node_modules/.bin/tap -J test/*.test.js test/*/*.test.js --no-coverage --no-check-coverage' +}) diff --git a/packages/datadog-plugin-fastify/test/index.spec.js b/packages/datadog-plugin-fastify/test/tracing.spec.js similarity index 81% rename from packages/datadog-plugin-fastify/test/index.spec.js rename to packages/datadog-plugin-fastify/test/tracing.spec.js index 55701272606..c8924c98dfd 100644 --- a/packages/datadog-plugin-fastify/test/index.spec.js +++ b/packages/datadog-plugin-fastify/test/tracing.spec.js @@ -2,10 +2,10 @@ const { AsyncLocalStorage } = require('async_hooks') const axios = require('axios') -const getPort = require('get-port') const semver = require('semver') const { ERROR_MESSAGE, ERROR_STACK, ERROR_TYPE } = require('../../dd-trace/src/constants') const agent = require('../../dd-trace/test/plugins/agent') +const { NODE_MAJOR } = require('../../../version') const host = 'localhost' @@ -15,7 +15,9 @@ describe('Plugin', () => { let app describe('fastify', () => { - withVersions('fastify', 'fastify', version => { + withVersions('fastify', 'fastify', (version, _, specificVersion) => { + if (NODE_MAJOR <= 18 && semver.satisfies(specificVersion, '>=5')) return + beforeEach(() => { tracer = require('../../dd-trace') }) @@ -48,7 +50,9 @@ describe('Plugin', () => { reply.send() }) - getPort().then(port => { + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + agent .use(traces => { const spans = traces[0] @@ -66,11 +70,9 @@ describe('Plugin', () => { .then(done) .catch(done) - app.listen({ host, port }, () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -83,7 +85,9 @@ describe('Plugin', () => { } }) - getPort().then(port => { + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + agent .use(traces => { const spans = traces[0] @@ -101,11 +105,9 @@ describe('Plugin', () => { .then(done) .catch(done) - app.listen({ host, port }, () => { - axios - .get(`http://localhost:${port}/user/123`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user/123`) + .catch(done) }) }) @@ -117,7 +119,9 @@ describe('Plugin', () => { } }) - getPort().then(port => { + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + agent .use(traces => { const spans = traces[0] @@ -135,11 +139,9 @@ describe('Plugin', () => { .then(done) .catch(done) - app.listen({ host, port }, () => { - axios - .get(`http://localhost:${port}/user/123`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user/123`) + .catch(done) }) }) } @@ -155,12 +157,12 @@ describe('Plugin', () => { reply.send() }) - getPort().then(port => { - app.listen({ host, port }, () => { - axios.get(`http://localhost:${port}/user`) - .then(() => done()) - .catch(done) - }) + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + + axios.get(`http://localhost:${port}/user`) + .then(() => done()) + .catch(done) }) }) @@ -172,12 +174,12 @@ describe('Plugin', () => { app.get('/user', (request, reply) => reply.send()) - getPort().then(port => { - app.listen({ host, port }, () => { - axios.get(`http://localhost:${port}/user`) - .then(() => done()) - .catch(done) - }) + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + + axios.get(`http://localhost:${port}/user`) + .then(() => done()) + .catch(done) }) }) @@ -187,12 +189,12 @@ describe('Plugin', () => { reply.send() }) - getPort().then(port => { - app.listen({ host, port }, () => { - axios.post(`http://localhost:${port}/user`, { foo: 'bar' }) - .then(() => done()) - .catch(done) - }) + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + + axios.post(`http://localhost:${port}/user`, { foo: 'bar' }) + .then(() => done()) + .catch(done) }) }) @@ -211,12 +213,12 @@ describe('Plugin', () => { } }) - getPort().then(port => { - app.listen({ host, port }, () => { - axios.post(`http://localhost:${port}/user`, { foo: 'bar' }) - .then(() => done()) - .catch(done) - }) + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + + axios.post(`http://localhost:${port}/user`, { foo: 'bar' }) + .then(() => done()) + .catch(done) }) }) @@ -248,12 +250,12 @@ describe('Plugin', () => { } }) - getPort().then(port => { - app.listen({ host, port }, () => { - axios.post(`http://localhost:${port}/user`, { foo: 'bar' }) - .then(() => done()) - .catch(done) - }) + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + + axios.post(`http://localhost:${port}/user`, { foo: 'bar' }) + .then(() => done()) + .catch(done) }) }) @@ -264,7 +266,9 @@ describe('Plugin', () => { reply.send(error = new Error('boom')) }) - getPort().then(port => { + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + agent .use(traces => { const spans = traces[0] @@ -279,11 +283,9 @@ describe('Plugin', () => { .then(done) .catch(done) - app.listen({ host, port }, () => { - axios - .get(`http://localhost:${port}/user`) - .catch(() => {}) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(() => {}) }) }) @@ -295,11 +297,11 @@ describe('Plugin', () => { reply.send() }) - getPort().then(port => { - app.listen({ host, port }, async () => { - await axios.get(`http://localhost:${port}/user`) - done() - }) + app.listen({ host, port: 0 }, async () => { + const port = app.server.address().port + + await axios.get(`http://localhost:${port}/user`) + done() }) }) @@ -324,11 +326,11 @@ describe('Plugin', () => { reply.send() }) - getPort().then(port => { - app.listen({ host, port }, () => { - axios.get(`http://localhost:${port}/user`) - .catch(done) - }) + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + + axios.get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -343,7 +345,9 @@ describe('Plugin', () => { reply.send() }) - getPort().then(port => { + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + agent .use(traces => { const spans = traces[0] @@ -359,11 +363,9 @@ describe('Plugin', () => { .then(done) .catch(done) - app.listen({ host, port }, () => { - axios - .get(`http://localhost:${port}/user`) - .catch(() => {}) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(() => {}) }) }) @@ -376,7 +378,9 @@ describe('Plugin', () => { reply.send() }) - getPort().then(port => { + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + agent .use(traces => { const spans = traces[0] @@ -389,11 +393,9 @@ describe('Plugin', () => { .then(done) .catch(done) - app.listen({ host, port }, () => { - axios - .get(`http://localhost:${port}/user`) - .catch(() => {}) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(() => {}) }) }) } @@ -407,7 +409,9 @@ describe('Plugin', () => { return Promise.reject(error = new Error('boom')) }) - getPort().then(port => { + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + agent .use(traces => { const spans = traces[0] @@ -422,17 +426,16 @@ describe('Plugin', () => { .then(done) .catch(done) - app.listen({ host, port }, () => { - axios - .get(`http://localhost:${port}/user`) - .catch(() => {}) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(() => {}) }) }) it('should handle reply exceptions', done => { let error + // eslint-disable-next-line n/handle-callback-err app.setErrorHandler((error, request, reply) => { reply.statusCode = 500 reply.send() @@ -441,7 +444,9 @@ describe('Plugin', () => { throw (error = new Error('boom')) }) - getPort().then(port => { + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + agent .use(traces => { const spans = traces[0] @@ -457,15 +462,14 @@ describe('Plugin', () => { .then(done) .catch(done) - app.listen({ host, port }, () => { - axios - .get(`http://localhost:${port}/user`) - .catch(() => {}) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(() => {}) }) }) it('should ignore reply exceptions if the request succeeds', done => { + // eslint-disable-next-line n/handle-callback-err app.setErrorHandler((error, request, reply) => { reply.statusCode = 200 reply.send() @@ -474,7 +478,9 @@ describe('Plugin', () => { throw new Error('boom') }) - getPort().then(port => { + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + agent .use(traces => { const spans = traces[0] @@ -490,11 +496,9 @@ describe('Plugin', () => { .then(done) .catch(done) - app.listen({ host, port }, () => { - axios - .get(`http://localhost:${port}/user`) - .catch(() => {}) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(() => {}) }) }) } @@ -513,7 +517,9 @@ describe('Plugin', () => { reply.send() }) - getPort().then(port => { + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + agent .use(traces => { const spans = traces[0] @@ -529,11 +535,9 @@ describe('Plugin', () => { .then(done) .catch(done) - app.listen({ host, port }, () => { - axios - .get(`http://localhost:${port}/user`) - .catch(() => {}) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(() => {}) }) }) } diff --git a/packages/datadog-plugin-fetch/src/index.js b/packages/datadog-plugin-fetch/src/index.js index 240699ec3bb..44173a561ca 100644 --- a/packages/datadog-plugin-fetch/src/index.js +++ b/packages/datadog-plugin-fetch/src/index.js @@ -4,28 +4,37 @@ const HttpClientPlugin = require('../../datadog-plugin-http/src/client') class FetchPlugin extends HttpClientPlugin { static get id () { return 'fetch' } - static get prefix () { return `apm:fetch:request` } + static get prefix () { return 'tracing:apm:fetch:request' } - addTraceSub (eventName, handler) { - this.addSub(`apm:${this.constructor.id}:${this.operation}:${eventName}`, handler) - } - - bindStart (message) { - const req = message.req + bindStart (ctx) { + const req = ctx.req const options = new URL(req.url) const headers = options.headers = Object.fromEntries(req.headers.entries()) options.method = req.method - message.args = { options } + ctx.args = { options } - const store = super.bindStart(message) + const store = super.bindStart(ctx) - message.headers = headers - message.req = new globalThis.Request(req, { headers }) + for (const name in headers) { + if (!req.headers.has(name)) { + req.headers.set(name, headers[name]) + } + } return store } + + error (ctx) { + if (ctx.error.name === 'AbortError') return + return super.error(ctx) + } + + asyncEnd (ctx) { + ctx.res = ctx.result + return this.finish(ctx) + } } module.exports = FetchPlugin diff --git a/packages/datadog-plugin-fetch/test/index.spec.js b/packages/datadog-plugin-fetch/test/index.spec.js index 3ad82148b47..b469f4a9722 100644 --- a/packages/datadog-plugin-fetch/test/index.spec.js +++ b/packages/datadog-plugin-fetch/test/index.spec.js @@ -1,6 +1,5 @@ 'use strict' -const getPort = require('get-port') const agent = require('../../dd-trace/test/plugins/agent') const tags = require('../../../ext/tags') const { expect } = require('chai') @@ -21,9 +20,9 @@ describe('Plugin', () => { let appListener describe('fetch', () => { - function server (app, port, listener) { + function server (app, listener) { const server = require('http').createServer(app) - server.listen(port, 'localhost', listener) + server.listen(0, 'localhost', () => listener(server.address().port)) return server } @@ -54,10 +53,8 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/user`) - }) + appListener = server(app, port => { + fetch(`http://localhost:${port}/user`) }) }, rawExpectedSchema.client @@ -68,7 +65,7 @@ describe('Plugin', () => { app.get('/user', (req, res) => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', SERVICE_NAME) @@ -84,9 +81,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/user`) - }) + fetch(`http://localhost:${port}/user`) }) }) @@ -95,7 +90,7 @@ describe('Plugin', () => { app.post('/user', (req, res) => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', SERVICE_NAME) @@ -111,9 +106,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch(new URL(`http://localhost:${port}/user`), { method: 'POST' }) - }) + fetch(new URL(`http://localhost:${port}/user`), { method: 'POST' }) }) }) @@ -122,7 +115,7 @@ describe('Plugin', () => { app.get('/user', (req, res) => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', SERVICE_NAME) @@ -138,9 +131,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch(new globalThis.Request(`http://localhost:${port}/user`)) - }) + fetch(new globalThis.Request(`http://localhost:${port}/user`)) }) }) @@ -149,15 +140,13 @@ describe('Plugin', () => { app.get('/user', (req, res) => { res.status(200).send() }) - getPort().then(port => { - appListener = server(app, port, () => { - fetch(new globalThis.Request(`http://localhost:${port}/user`)) - .then(res => { - expect(res).to.have.property('status', 200) - done() - }) - .catch(done) - }) + appListener = server(app, port => { + fetch(new globalThis.Request(`http://localhost:${port}/user`)) + .then(res => { + expect(res).to.have.property('status', 200) + done() + }) + .catch(done) }) }) @@ -168,7 +157,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.status_code', '200') @@ -177,9 +166,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/user?foo=bar`) - }) + fetch(`http://localhost:${port}/user?foo=bar`) }) }) @@ -193,7 +180,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.status_code', '200') @@ -201,9 +188,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/user?foo=bar`) - }) + fetch(`http://localhost:${port}/user?foo=bar`) }) }) @@ -218,7 +203,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.status_code', '200') @@ -226,133 +211,25 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/user?foo=bar`, { headers: { 'foo': 'bar' } }) - }) - }) - }) - - it('should skip injecting if the Authorization header contains an AWS signature', done => { - const app = express() - - app.get('/', (req, res) => { - try { - expect(req.get('x-datadog-trace-id')).to.be.undefined - expect(req.get('x-datadog-parent-id')).to.be.undefined - - res.status(200).send() - - done() - } catch (e) { - done(e) - } - }) - - getPort().then(port => { - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/`, { - headers: { - Authorization: 'AWS4-HMAC-SHA256 ...' - } - }) - }) - }) - }) - - it('should skip injecting if one of the Authorization headers contains an AWS signature', done => { - const app = express() - - app.get('/', (req, res) => { - try { - expect(req.get('x-datadog-trace-id')).to.be.undefined - expect(req.get('x-datadog-parent-id')).to.be.undefined - - res.status(200).send() - - done() - } catch (e) { - done(e) - } - }) - - getPort().then(port => { - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/`, { - headers: { - Authorization: ['AWS4-HMAC-SHA256 ...'] - } - }) - }) - }) - }) - - it('should skip injecting if the X-Amz-Signature header is set', done => { - const app = express() - - app.get('/', (req, res) => { - try { - expect(req.get('x-datadog-trace-id')).to.be.undefined - expect(req.get('x-datadog-parent-id')).to.be.undefined - - res.status(200).send() - - done() - } catch (e) { - done(e) - } - }) - - getPort().then(port => { - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/`, { - headers: { - 'X-Amz-Signature': 'abc123' - } - }) - }) - }) - }) - - it('should skip injecting if the X-Amz-Signature query param is set', done => { - const app = express() - - app.get('/', (req, res) => { - try { - expect(req.get('x-datadog-trace-id')).to.be.undefined - expect(req.get('x-datadog-parent-id')).to.be.undefined - - res.status(200).send() - - done() - } catch (e) { - done(e) - } - }) - - getPort().then(port => { - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/?X-Amz-Signature=abc123`) - }) + fetch(`http://localhost:${port}/user?foo=bar`, { headers: { foo: 'bar' } }) }) }) it('should handle connection errors', done => { - getPort().then(port => { - let error - - agent - .use(traces => { - expect(traces[0][0].meta).to.have.property(ERROR_TYPE, error.name) - expect(traces[0][0].meta).to.have.property(ERROR_MESSAGE, error.message || error.code) - expect(traces[0][0].meta).to.have.property(ERROR_STACK, error.stack) - expect(traces[0][0].meta).to.have.property('component', 'fetch') - }) - .then(done) - .catch(done) + let error - fetch(`http://localhost:${port}/user`).catch(err => { - error = err + agent + .use(traces => { + expect(traces[0][0].meta).to.have.property(ERROR_TYPE, error.name) + expect(traces[0][0].meta).to.have.property(ERROR_MESSAGE, error.message || error.code) + expect(traces[0][0].meta).to.have.property(ERROR_STACK, error.stack) + expect(traces[0][0].meta).to.have.property('component', 'fetch') }) + .then(done) + .catch(done) + + fetch('http://localhost:7357/user').catch(err => { + error = err }) }) @@ -363,7 +240,7 @@ describe('Plugin', () => { res.status(500).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('error', 0) @@ -371,9 +248,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/user`) - }) + fetch(`http://localhost:${port}/user`) }) }) @@ -384,7 +259,7 @@ describe('Plugin', () => { res.status(400).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('error', 1) @@ -392,9 +267,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/user`) - }) + fetch(`http://localhost:${port}/user`) }) }) @@ -403,7 +276,7 @@ describe('Plugin', () => { app.get('/user', (req, res) => {}) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('error', 0) @@ -412,15 +285,13 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const controller = new AbortController() + const controller = new AbortController() - fetch(`http://localhost:${port}/user`, { - signal: controller.signal - }).catch(e => {}) + fetch(`http://localhost:${port}/user`, { + signal: controller.signal + }).catch(e => {}) - controller.abort() - }) + controller.abort() }) }) @@ -431,7 +302,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', SERVICE_NAME) @@ -439,15 +310,13 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const controller = new AbortController() + const controller = new AbortController() - fetch(`http://localhost:${port}/user`, { - signal: controller.signal - }).catch(e => {}) + fetch(`http://localhost:${port}/user`, { + signal: controller.signal + }).catch(e => {}) - controller.abort() - }) + controller.abort() }) }) @@ -458,7 +327,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { const timer = setTimeout(done, 100) agent @@ -467,15 +336,13 @@ describe('Plugin', () => { clearTimeout(timer) }) - appListener = server(app, port, () => { - const store = storage.getStore() + const store = storage.getStore() - storage.enterWith({ noop: true }) + storage.enterWith({ noop: true }) - fetch(`http://localhost:${port}/user`).catch(() => {}) + fetch(`http://localhost:${port}/user`).catch(() => {}) - storage.enterWith(store) - }) + storage.enterWith(store) }) }) }) @@ -502,7 +369,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', 'custom') @@ -510,9 +377,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/user`).catch(() => {}) - }) + fetch(`http://localhost:${port}/user`).catch(() => {}) }) }) }) @@ -539,7 +404,7 @@ describe('Plugin', () => { res.status(500).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('error', 1) @@ -547,9 +412,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/user`).catch(() => {}) - }) + fetch(`http://localhost:${port}/user`).catch(() => {}) }) }) }) @@ -576,7 +439,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', `localhost:${port}`) @@ -584,9 +447,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/user`).catch(() => {}) - }) + fetch(`http://localhost:${port}/user`).catch(() => {}) }) }) }) @@ -614,24 +475,22 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { const meta = traces[0][0].meta - expect(meta).to.have.property(`${HTTP_REQUEST_HEADERS}.x-baz`, `qux`) + expect(meta).to.have.property(`${HTTP_REQUEST_HEADERS}.x-baz`, 'qux') expect(meta).to.have.property(`${HTTP_RESPONSE_HEADERS}.x-foo`, 'bar') }) .then(done) .catch(done) - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/user`, { - headers: { - 'x-baz': 'qux' - } - }).catch(() => {}) - }) + fetch(`http://localhost:${port}/user`, { + headers: { + 'x-baz': 'qux' + } + }).catch(() => {}) }) }) }) @@ -662,7 +521,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('foo', '/foo') @@ -670,9 +529,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/user`).catch(() => {}) - }) + fetch(`http://localhost:${port}/user`).catch(() => {}) }) }) }) @@ -708,10 +565,8 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/users`).catch(() => {}) - }) + appListener = server(app, port => { + fetch(`http://localhost:${port}/users`).catch(() => {}) }) }) }) @@ -738,7 +593,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { const timer = setTimeout(done, 100) agent @@ -748,9 +603,7 @@ describe('Plugin', () => { }) .catch(done) - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/users`).catch(() => {}) - }) + fetch(`http://localhost:${port}/users`).catch(() => {}) }) }) }) diff --git a/packages/datadog-plugin-fetch/test/integration-test/client.spec.js b/packages/datadog-plugin-fetch/test/integration-test/client.spec.js index fe2560f3072..922c889134d 100644 --- a/packages/datadog-plugin-fetch/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-fetch/test/integration-test/client.spec.js @@ -17,7 +17,7 @@ describe('esm', () => { before(async function () { this.timeout(50000) sandbox = await createSandbox(['get-port'], false, [ - `./packages/datadog-plugin-fetch/test/integration-test/*`]) + './packages/datadog-plugin-fetch/test/integration-test/*']) }) after(async function () { diff --git a/packages/datadog-plugin-fs/src/index.js b/packages/datadog-plugin-fs/src/index.js index db05cf063b8..8a437b7fc22 100644 --- a/packages/datadog-plugin-fs/src/index.js +++ b/packages/datadog-plugin-fs/src/index.js @@ -29,7 +29,7 @@ class FsPlugin extends TracingPlugin { resource: operation, kind: 'internal', meta: { - 'file.descriptor': (typeof fd === 'object' || typeof fd === 'number') ? fd.toString() : '', + 'file.descriptor': ((fd !== null && typeof fd === 'object') || typeof fd === 'number') ? fd.toString() : '', 'file.dest': params.dest || params.newPath || (params.target && params.path), 'file.flag': String(flag || defaultFlag || ''), 'file.gid': gid || '', diff --git a/packages/datadog-plugin-fs/test/index.spec.js b/packages/datadog-plugin-fs/test/index.spec.js index da1310010c9..e54f1d4ffd0 100644 --- a/packages/datadog-plugin-fs/test/index.spec.js +++ b/packages/datadog-plugin-fs/test/index.spec.js @@ -19,11 +19,14 @@ describe('Plugin', () => { describe('fs not instrumented without internal method call', () => { let fs let tracer + afterEach(() => agent.close({ ritmReset: false })) + beforeEach(() => agent.load('fs', undefined, { flushInterval: 1 }).then(() => { tracer = require('../../dd-trace') fs = require('fs') })) + describe('with parent span', () => { beforeEach((done) => { const parentSpan = tracer.startSpan('parent') @@ -64,24 +67,29 @@ describe('Plugin', () => { }) }) }) + describe('fs', () => { let fs let tmpdir let tracer + afterEach(() => agent.close({ ritmReset: false })) + beforeEach(() => agent.load('fs', undefined, { flushInterval: 1 }).then(() => { tracer = require('../../dd-trace') fs = require('fs') tracer.use('fs', { enabled: true }) })) + before(() => { tmpdir = realFS.mkdtempSync(path.join(os.tmpdir(), 'dd-trace-js-test')) - plugins['fs'] = require('../../datadog-plugin-fs/src') + plugins.fs = require('../../datadog-plugin-fs/src') channel('dd-trace:instrumentation:load').publish({ name: 'fs' }) }) + after((done) => { rimraf(tmpdir, realFS, done) - delete plugins['fs'] + delete plugins.fs }) describe('without parent span', () => { @@ -113,6 +121,7 @@ describe('Plugin', () => { describe('open', () => { let fd + afterEach(() => { if (typeof fd === 'number') { realFS.closeSync(fd) @@ -138,6 +147,7 @@ describe('Plugin', () => { describe('open', () => { let fd + afterEach(() => { if (typeof fd === 'number') { realFS.closeSync(fd) @@ -177,6 +187,7 @@ describe('Plugin', () => { it('should handle errors', (done) => { const filename = path.join(__filename, Math.random().toString()) + // eslint-disable-next-line n/handle-callback-err fs.open(filename, 'r', (err) => { expectOneSpan(agent, done, { resource: 'open', @@ -193,6 +204,7 @@ describe('Plugin', () => { if (realFS.promises) { describe('promises.open', () => { let fd + afterEach(() => { if (typeof fd === 'number') { realFS.closeSync(fd) @@ -230,6 +242,7 @@ describe('Plugin', () => { it('should handle errors', (done) => { const filename = path.join(__filename, Math.random().toString()) + // eslint-disable-next-line n/handle-callback-err fs.promises.open(filename, 'r').catch((err) => { expectOneSpan(agent, done, { resource: 'promises.open', @@ -246,6 +259,7 @@ describe('Plugin', () => { describe('openSync', () => { let fd + afterEach(() => { if (typeof fd === 'number') { realFS.closeSync(fd) @@ -697,9 +711,11 @@ describe('Plugin', () => { describe('createWriteStream', () => { let filename + beforeEach(() => { filename = path.join(tmpdir, 'createWriteStream') }) + afterEach(done => { // swallow errors since we're causing a race condition in one of the tests realFS.unlink(filename, () => done()) @@ -807,7 +823,7 @@ describe('Plugin', () => { it('should be instrumented', (done) => { expectOneSpan(agent, done, { - resource: resource, + resource, meta: { 'file.descriptor': fd.toString(), 'file.mode': mode.toString(8) @@ -1282,6 +1298,7 @@ describe('Plugin', () => { describe('mkdtemp', () => { let tmpdir + afterEach(() => { try { realFS.rmdirSync(tmpdir) @@ -1313,6 +1330,7 @@ describe('Plugin', () => { describe('mkdtempSync', () => { let tmpdir + afterEach(() => { try { realFS.rmdirSync(tmpdir) @@ -1348,11 +1366,14 @@ describe('Plugin', () => { 'file.path': __filename } }) - fs.exists(__filename, () => {}) // eslint-disable-line node/no-deprecated-api + // eslint-disable-next-line n/handle-callback-err + // eslint-disable-next-line n/no-deprecated-api + fs.exists(__filename, () => {}) }) it('should support promisification', () => { - const exists = util.promisify(fs.exists) // eslint-disable-line node/no-deprecated-api + // eslint-disable-next-line n/no-deprecated-api + const exists = util.promisify(fs.exists) return exists(__filename) }) @@ -1418,6 +1439,7 @@ describe('Plugin', () => { describe('Dir', () => { let dirname let dir + beforeEach(async () => { dirname = path.join(tmpdir, 'dir') fs.mkdirSync(dirname) @@ -1426,6 +1448,7 @@ describe('Plugin', () => { fs.writeFileSync(path.join(dirname, '3'), '3') dir = await fs.promises.opendir(dirname) }) + afterEach(async () => { try { await dir.close() @@ -1599,11 +1622,13 @@ describe('Plugin', () => { describe('FileHandle', () => { let filehandle let filename + beforeEach(async () => { filename = path.join(os.tmpdir(), 'filehandle') fs.writeFileSync(filename, 'some data') filehandle = await fs.promises.open(filename, 'w+') }) + afterEach(async () => { try { await filehandle.close() @@ -1712,6 +1737,7 @@ describe('Plugin', () => { describe('chmod', () => { let mode + beforeEach(() => { mode = realFS.statSync(__filename).mode % 0o100000 }) @@ -1735,6 +1761,7 @@ describe('Plugin', () => { describe('chown', () => { let uid let gid + beforeEach(() => { const stats = realFS.statSync(filename) uid = stats.uid @@ -1932,6 +1959,7 @@ function testHandleErrors (fs, name, tested, args, agent) { if (err) reject(err) else resolve() } + // eslint-disable-next-line n/handle-callback-err tested(fs, args, null, err => { expectOneSpan(agent, done, { resource: name, diff --git a/packages/datadog-plugin-google-cloud-pubsub/src/consumer.js b/packages/datadog-plugin-google-cloud-pubsub/src/consumer.js index 75dff9b1d83..1fd647f5a34 100644 --- a/packages/datadog-plugin-google-cloud-pubsub/src/consumer.js +++ b/packages/datadog-plugin-google-cloud-pubsub/src/consumer.js @@ -36,6 +36,8 @@ class GoogleCloudPubsubConsumerPlugin extends ConsumerPlugin { finish (message) { const span = this.activeSpan + if (!span) return + if (message.message._handled) { span.setTag('pubsub.ack', 1) } diff --git a/packages/datadog-plugin-google-cloud-pubsub/test/index.spec.js b/packages/datadog-plugin-google-cloud-pubsub/test/index.spec.js index 44c6925a4b2..1eaa68bbc82 100644 --- a/packages/datadog-plugin-google-cloud-pubsub/test/index.spec.js +++ b/packages/datadog-plugin-google-cloud-pubsub/test/index.spec.js @@ -37,9 +37,11 @@ describe('Plugin', () => { process.env.PUBSUB_EMULATOR_HOST = 'localhost:8081' process.env.DD_DATA_STREAMS_ENABLED = true }) + after(() => { delete process.env.PUBSUB_EMULATOR_HOST }) + afterEach(() => { return agent.close({ ritmReset: false }) }) @@ -55,9 +57,10 @@ describe('Plugin', () => { beforeEach(() => { return agent.load('google-cloud-pubsub', { dsmEnabled: false }) }) + beforeEach(() => { tracer = require('../../dd-trace') - gax = require(`../../../versions/google-gax@3.5.7`).get() + gax = require('../../../versions/google-gax@3.5.7').get() const lib = require(`../../../versions/@google-cloud/pubsub@${version}`).get() project = getProjectId() topicName = getTopic() @@ -65,6 +68,7 @@ describe('Plugin', () => { v1 = lib.v1 pubsub = new lib.PubSub({ projectId: project }) }) + describe('createTopic', () => { withNamingSchema( async () => pubsub.createTopic(topicName), @@ -78,7 +82,7 @@ describe('Plugin', () => { meta: { 'pubsub.method': 'createTopic', 'span.kind': 'client', - 'component': 'google-cloud-pubsub' + component: 'google-cloud-pubsub' } }) await pubsub.createTopic(topicName) @@ -100,7 +104,7 @@ describe('Plugin', () => { meta: { 'pubsub.method': 'createTopic', 'span.kind': 'client', - 'component': 'google-cloud-pubsub' + component: 'google-cloud-pubsub' } }) const name = `projects/${project}/topics/${topicName}` @@ -117,7 +121,7 @@ describe('Plugin', () => { error: 1, meta: { 'pubsub.method': 'createTopic', - 'component': 'google-cloud-pubsub' + component: 'google-cloud-pubsub' } }) const publisher = new v1.PublisherClient({ projectId: project }) @@ -148,7 +152,7 @@ describe('Plugin', () => { 'pubsub.topic': resource, 'pubsub.method': 'publish', 'span.kind': 'producer', - 'component': 'google-cloud-pubsub' + component: 'google-cloud-pubsub' } }) const [topic] = await pubsub.createTopic(topicName) @@ -183,7 +187,7 @@ describe('Plugin', () => { service: expectedSchema.receive.serviceName, type: 'worker', meta: { - 'component': 'google-cloud-pubsub', + component: 'google-cloud-pubsub', 'span.kind': 'consumer', 'pubsub.topic': resource }, @@ -233,7 +237,7 @@ describe('Plugin', () => { [ERROR_MESSAGE]: error.message, [ERROR_TYPE]: error.name, [ERROR_STACK]: error.stack, - 'component': 'google-cloud-pubsub' + component: 'google-cloud-pubsub' } }) const [topic] = await pubsub.createTopic(topicName) @@ -287,6 +291,24 @@ describe('Plugin', () => { await pubsub.createTopic(topicName) }) }) + + it('should handle manual subscription close', async () => { + const [topic] = await pubsub.createTopic(topicName) + const [sub] = await topic.createSubscription('foo') + + // message handler takes a while, subscription is closed while it's still running + sub.on('message', msg => { + setTimeout(() => { msg.ack() }, 2000) + }) + + await publish(topic, { data: Buffer.from('hello') }) + + setTimeout(() => { sub.close() }, 500) + + return new Promise((resolve) => { + sub.on('close', resolve) + }) + }) }) describe('with configuration', () => { @@ -417,7 +439,7 @@ describe('Plugin', () => { service, error: 0, meta: { - 'component': 'google-cloud-pubsub', + component: 'google-cloud-pubsub', 'gcloud.project_id': project } }, expected) diff --git a/packages/datadog-plugin-google-cloud-pubsub/test/integration-test/client.spec.js b/packages/datadog-plugin-google-cloud-pubsub/test/integration-test/client.spec.js index 6619bcfe997..0effff0795b 100644 --- a/packages/datadog-plugin-google-cloud-pubsub/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-google-cloud-pubsub/test/integration-test/client.spec.js @@ -16,8 +16,8 @@ describe('esm', () => { withVersions('google-cloud-pubsub', '@google-cloud/pubsub', '>=4.0.0', version => { before(async function () { this.timeout(20000) - sandbox = await createSandbox([`'@google-cloud/pubsub@${version}'`], false, [ './packages/dd-trace/src/id.js', - `./packages/datadog-plugin-google-cloud-pubsub/test/integration-test/*`]) + sandbox = await createSandbox([`'@google-cloud/pubsub@${version}'`], false, ['./packages/dd-trace/src/id.js', + './packages/datadog-plugin-google-cloud-pubsub/test/integration-test/*']) }) after(async () => { diff --git a/packages/datadog-plugin-google-cloud-pubsub/test/naming.js b/packages/datadog-plugin-google-cloud-pubsub/test/naming.js index 97f7f490617..da2ffd55247 100644 --- a/packages/datadog-plugin-google-cloud-pubsub/test/naming.js +++ b/packages/datadog-plugin-google-cloud-pubsub/test/naming.js @@ -34,6 +34,6 @@ const rawExpectedSchema = { } module.exports = { - rawExpectedSchema: rawExpectedSchema, + rawExpectedSchema, expectedSchema: resolveNaming(rawExpectedSchema) } diff --git a/packages/datadog-plugin-graphql/src/index.js b/packages/datadog-plugin-graphql/src/index.js index fa701da350f..1e530fb94d0 100644 --- a/packages/datadog-plugin-graphql/src/index.js +++ b/packages/datadog-plugin-graphql/src/index.js @@ -1,5 +1,6 @@ 'use strict' +const pick = require('../../datadog-core/src/utils/src/pick') const CompositePlugin = require('../../dd-trace/src/plugins/composite') const log = require('../../dd-trace/src/log') const GraphQLExecutePlugin = require('./execute') @@ -63,10 +64,4 @@ function getHooks (config) { return { execute, parse, validate } } -// non-lodash pick - -function pick (obj, selectors) { - return Object.fromEntries(Object.entries(obj).filter(([key]) => selectors.includes(key))) -} - module.exports = GraphQLPlugin diff --git a/packages/datadog-plugin-graphql/src/resolve.js b/packages/datadog-plugin-graphql/src/resolve.js index caca7c96e3e..ebf8ffb59f0 100644 --- a/packages/datadog-plugin-graphql/src/resolve.js +++ b/packages/datadog-plugin-graphql/src/resolve.js @@ -1,6 +1,7 @@ 'use strict' const TracingPlugin = require('../../dd-trace/src/plugins/tracing') +const dc = require('dc-polyfill') const collapsedPathSym = Symbol('collapsedPaths') @@ -14,8 +15,6 @@ class GraphQLResolvePlugin extends TracingPlugin { if (!shouldInstrument(this.config, path)) return const computedPathString = path.join('.') - addResolver(context, info, args) - if (this.config.collapse) { if (!context[collapsedPathSym]) { context[collapsedPathSym] = {} @@ -55,6 +54,10 @@ class GraphQLResolvePlugin extends TracingPlugin { span.setTag(`graphql.variables.${name}`, variables[name]) }) } + + if (this.resolverStartCh.hasSubscribers) { + this.resolverStartCh.publish({ context, resolverInfo: getResolverInfo(info, args) }) + } } constructor (...args) { @@ -69,12 +72,18 @@ class GraphQLResolvePlugin extends TracingPlugin { field.finishTime = span._getTime ? span._getTime() : 0 field.error = field.error || err }) + + this.resolverStartCh = dc.channel('datadog:graphql:resolver:start') } configure (config) { // this will disable resolve subscribers if `config.depth` is set to 0 super.configure(config.depth === 0 ? false : config) } + + finish (finishTime) { + this.activeSpan.finish(finishTime) + } } // helpers @@ -109,28 +118,33 @@ function withCollapse (responsePathAsArray) { } } -function addResolver (context, info, args) { - if (info.rootValue && !info.rootValue[info.fieldName]) { - return - } +function getResolverInfo (info, args) { + let resolverInfo = null + const resolverVars = {} - if (!context.resolvers) { - context.resolvers = {} + if (args && Object.keys(args).length) { + Object.assign(resolverVars, args) } - const resolvers = context.resolvers + const directives = info.fieldNodes?.[0]?.directives + if (Array.isArray(directives)) { + for (const directive of directives) { + const argList = {} + for (const argument of directive.arguments) { + argList[argument.name.value] = argument.value.value + } - if (!resolvers[info.fieldName]) { - if (args && Object.keys(args).length) { - resolvers[info.fieldName] = [args] - } else { - resolvers[info.fieldName] = [] - } - } else { - if (args && Object.keys(args).length) { - resolvers[info.fieldName].push(args) + if (Object.keys(argList).length) { + resolverVars[directive.name.value] = argList + } } } + + if (Object.keys(resolverVars).length) { + resolverInfo = { [info.fieldName]: resolverVars } + } + + return resolverInfo } module.exports = GraphQLResolvePlugin diff --git a/packages/datadog-plugin-graphql/test/index.spec.js b/packages/datadog-plugin-graphql/test/index.spec.js index f45d1a587a4..aa8c754f28a 100644 --- a/packages/datadog-plugin-graphql/test/index.spec.js +++ b/packages/datadog-plugin-graphql/test/index.spec.js @@ -7,7 +7,10 @@ const { ERROR_MESSAGE, ERROR_TYPE, ERROR_STACK } = require('../../dd-trace/src/c const { expectedSchema, rawExpectedSchema } = require('./naming') const axios = require('axios') const http = require('http') -const getPort = require('get-port') +const dc = require('dc-polyfill') +const plugin = require('../src') + +const { performance } = require('perf_hooks') describe('Plugin', () => { let tracer @@ -15,6 +18,10 @@ describe('Plugin', () => { let schema let sort + let markFast + let markSlow + let markSync + function buildSchema () { const Human = new graphql.GraphQLObjectType({ name: 'Human', @@ -78,6 +85,31 @@ describe('Plugin', () => { resolve (obj, args) { return [{}, {}, {}] } + }, + fastAsyncField: { + type: graphql.GraphQLString, + resolve (obj, args) { + return new Promise((resolve) => { + markFast = performance.now() + resolve('fast field') + }) + } + }, + slowAsyncField: { + type: graphql.GraphQLString, + resolve (obj, args) { + return new Promise((resolve) => { + markSlow = performance.now() + resolve('slow field') + }) + } + }, + syncField: { + type: graphql.GraphQLString, + resolve (obj, args) { + markSync = performance.now() + return 'sync field' + } } } }) @@ -110,7 +142,7 @@ describe('Plugin', () => { friends: { type: new graphql.GraphQLList(Human), resolve () { - return [ { name: 'alice' }, { name: 'bob' } ] + return [{ name: 'alice' }, { name: 'bob' }] } } } @@ -166,7 +198,7 @@ describe('Plugin', () => { }) describe('graphql-yoga', () => { - withVersions('graphql', 'graphql-yoga', version => { + withVersions(plugin, 'graphql-yoga', version => { let graphqlYoga let server let port @@ -198,14 +230,16 @@ describe('Plugin', () => { const yoga = graphqlYoga.createYoga({ schema }) server = http.createServer(yoga) - - getPort().then(newPort => { - port = newPort - server.listen(port) - }) }) }) + before(done => { + server.listen(0, () => { + port = server.address().port + done() + }) + }) + after(() => { server.close() return agent.close({ ritmReset: false }) @@ -257,7 +291,7 @@ describe('Plugin', () => { withNamingSchema( () => { - const source = `query MyQuery { hello(name: "world") }` + const source = 'query MyQuery { hello(name: "world") }' const variableValues = { who: 'world' } graphql.graphql({ schema, source, variableValues }) }, @@ -271,7 +305,7 @@ describe('Plugin', () => { ) it('should instrument parsing', done => { - const source = `query MyQuery { hello(name: "world") }` + const source = 'query MyQuery { hello(name: "world") }' const variableValues = { who: 'world' } agent @@ -293,7 +327,7 @@ describe('Plugin', () => { }) it('should instrument validation', done => { - const source = `query MyQuery { hello(name: "world") }` + const source = 'query MyQuery { hello(name: "world") }' const variableValues = { who: 'world' } agent @@ -315,7 +349,7 @@ describe('Plugin', () => { }) it('should instrument execution', done => { - const source = `query MyQuery { hello(name: "world") }` + const source = 'query MyQuery { hello(name: "world") }' const variableValues = { who: 'world' } agent @@ -338,7 +372,7 @@ describe('Plugin', () => { }) it('should not include variables by default', done => { - const source = `query MyQuery($who: String!) { hello(name: $who) }` + const source = 'query MyQuery($who: String!) { hello(name: $who) }' const variableValues = { who: 'world' } agent @@ -353,7 +387,7 @@ describe('Plugin', () => { }) it('should instrument schema resolvers', done => { - const source = `{ hello(name: "world") }` + const source = '{ hello(name: "world") }' agent .use(traces => { @@ -378,6 +412,73 @@ describe('Plugin', () => { graphql.graphql({ schema, source }).catch(done) }) + it('should instrument each field resolver duration independently', done => { + const source = ` + { + human { + fastAsyncField + slowAsyncField + syncField + } + } + ` + + let foundFastFieldSpan = false + let foundSlowFieldSpan = false + let foundSyncFieldSpan = false + + let fastAsyncTime + let slowAsyncTime + let syncTime + + const processTraces = (traces) => { + try { + for (const trace of traces) { + for (const span of trace) { + if (span.name !== 'graphql.resolve') { + continue + } + + if (span.resource === 'fastAsyncField:String') { + expect(fastAsyncTime).to.be.lessThan(slowAsyncTime) + foundFastFieldSpan = true + } else if (span.resource === 'slowAsyncField:String') { + expect(slowAsyncTime).to.be.lessThan(syncTime) + foundSlowFieldSpan = true + } else if (span.resource === 'syncField:String') { + expect(syncTime).to.be.greaterThan(slowAsyncTime) + foundSyncFieldSpan = true + } + + if (foundFastFieldSpan && foundSlowFieldSpan && foundSyncFieldSpan) { + agent.unsubscribe(processTraces) + done() + return + } + } + } + } catch (e) { + agent.unsubscribe(processTraces) + done(e) + } + } + + agent.subscribe(processTraces) + + const markStart = performance.now() + + graphql.graphql({ schema, source }) + .then((result) => { + fastAsyncTime = markFast - markStart + slowAsyncTime = markSlow - markStart + syncTime = markSync - markStart + }) + .catch((e) => { + agent.unsubscribe(processTraces) + done(e) + }) + }) + it('should instrument nested field resolvers', done => { const source = ` { @@ -493,7 +594,7 @@ describe('Plugin', () => { }) it('should instrument mutations', done => { - const source = `mutation { human { name } }` + const source = 'mutation { human { name } }' agent .use(traces => { @@ -508,7 +609,7 @@ describe('Plugin', () => { }) it('should instrument subscriptions', done => { - const source = `subscription { human { name } }` + const source = 'subscription { human { name } }' agent .use(traces => { @@ -523,7 +624,7 @@ describe('Plugin', () => { }) it('should handle a circular schema', done => { - const source = `{ human { pets { owner { name } } } }` + const source = '{ human { pets { owner { name } } } }' graphql.graphql({ schema, source }) .then((result) => { @@ -540,7 +641,7 @@ describe('Plugin', () => { } `) - const source = `{ hello }` + const source = '{ hello }' const rootValue = { hello: 'world' } agent @@ -564,7 +665,7 @@ describe('Plugin', () => { } `) - const source = `{ hello }` + const source = '{ hello }' const rootValue = { hello: 'world' } @@ -587,7 +688,7 @@ describe('Plugin', () => { }) it('should not instrument schema resolvers multiple times', done => { - const source = `{ hello(name: "world") }` + const source = '{ hello(name: "world") }' agent.use(() => { // skip first call agent @@ -606,7 +707,7 @@ describe('Plugin', () => { }) it('should run parsing, validation and execution in the current context', done => { - const source = `query MyQuery { hello(name: "world") }` + const source = 'query MyQuery { hello(name: "world") }' const variableValues = { who: 'world' } const span = tracer.startSpan('test.request') @@ -649,7 +750,7 @@ describe('Plugin', () => { } `) - const source = `{ hello }` + const source = '{ hello }' const rootValue = { hello () { @@ -672,7 +773,7 @@ describe('Plugin', () => { } `) - const source = `{ hello }` + const source = '{ hello }' const rootValue = { hello () { @@ -692,8 +793,8 @@ describe('Plugin', () => { }) it('should handle unsupported operations', () => { - const source = `query MyQuery { hello(name: "world") }` - const subscription = `subscription { human { name } }` + const source = 'query MyQuery { hello(name: "world") }' + const subscription = 'subscription { human { name } }' return graphql.graphql({ schema, source }) .then(() => graphql.graphql({ schema, source: subscription })) @@ -703,7 +804,7 @@ describe('Plugin', () => { }) it('should handle calling low level APIs directly', done => { - const source = `query MyQuery { hello(name: "world") }` + const source = 'query MyQuery { hello(name: "world") }' Promise .all([ @@ -731,7 +832,7 @@ describe('Plugin', () => { }) it('should handle Source objects', done => { - const source = `query MyQuery { hello(name: "world") }` + const source = 'query MyQuery { hello(name: "world") }' const document = graphql.parse(new graphql.Source(source)) agent @@ -804,7 +905,7 @@ describe('Plugin', () => { }) it('should handle validation errors', done => { - const source = `{ human { address } }` + const source = '{ human { address } }' const document = graphql.parse(source) agent @@ -827,7 +928,7 @@ describe('Plugin', () => { }) it('should handle execution exceptions', done => { - const source = `{ hello }` + const source = '{ hello }' const document = graphql.parse(source) let error @@ -856,7 +957,7 @@ describe('Plugin', () => { }) it('should handle execution errors', done => { - const source = `{ hello }` + const source = '{ hello }' const document = graphql.parse(source) const schema = graphql.buildSchema(` @@ -904,7 +1005,7 @@ describe('Plugin', () => { } `) - const source = `{ hello }` + const source = '{ hello }' const rootValue = { hello: () => { @@ -938,7 +1039,7 @@ describe('Plugin', () => { } `) - const source = `{ hello }` + const source = '{ hello }' const rootValue = { hello: () => { @@ -970,7 +1071,7 @@ describe('Plugin', () => { } `) - const source = `{ hello }` + const source = '{ hello }' const rootValue = { hello: () => 'world' @@ -985,17 +1086,35 @@ describe('Plugin', () => { }) it('should support multiple executions on a pre-parsed document', () => { - const source = `query MyQuery { hello(name: "world") }` + const source = 'query MyQuery { hello(name: "world") }' const document = graphql.parse(source) - expect(() => { graphql.execute({ schema, document }) graphql.execute({ schema, document }) }).to.not.throw() }) + it('should not fail without directives in the document ' + + 'and with subscription to datadog:graphql:resolver:start', () => { + const source = 'query MyQuery { hello(name: "world") }' + const document = graphql.parse(source) + delete document.definitions[0].directives + delete document.definitions[0].selectionSet.selections[0].directives + + function noop () {} + dc.channel('datadog:graphql:resolver:start').subscribe(noop) + + try { + expect(() => { + graphql.execute({ schema, document }) + }).to.not.throw() + } finally { + dc.channel('datadog:graphql:resolver:start').unsubscribe(noop) + } + }) + it('should support multiple validations on a pre-parsed document', () => { - const source = `query MyQuery { hello(name: "world") }` + const source = 'query MyQuery { hello(name: "world") }' const document = graphql.parse(source) expect(() => { @@ -1091,7 +1210,7 @@ describe('Plugin', () => { // https://github.com/graphql/graphql-js/pull/2904 if (!semver.intersects(version, '>=16')) { it('should instrument using positional arguments', done => { - const source = `query MyQuery { hello(name: "world") }` + const source = 'query MyQuery { hello(name: "world") }' const variableValues = { who: 'world' } agent @@ -1114,7 +1233,7 @@ describe('Plugin', () => { }) } else { it('should not support positional arguments', done => { - const source = `query MyQuery { hello(name: "world") }` + const source = 'query MyQuery { hello(name: "world") }' const variableValues = { who: 'world' } graphql.graphql(schema, source, null, null, variableValues) @@ -1181,7 +1300,7 @@ describe('Plugin', () => { }) it('should be configured with the correct values', done => { - const source = `{ hello(name: "world") }` + const source = '{ hello(name: "world") }' agent .use(traces => { @@ -1314,7 +1433,7 @@ describe('Plugin', () => { } `) - const source = `{ hello }` + const source = '{ hello }' const rootValue = { hello () { @@ -1403,7 +1522,7 @@ describe('Plugin', () => { }) it('should not collapse list field resolvers', done => { - const source = `{ friends { name } }` + const source = '{ friends { name } }' agent .use(traces => { @@ -1457,7 +1576,7 @@ describe('Plugin', () => { }) it('should fallback to the operation type and name', done => { - const source = `query WithoutSignature { friends { name } }` + const source = 'query WithoutSignature { friends { name } }' agent .use(traces => { @@ -1615,7 +1734,7 @@ describe('Plugin', () => { }) }) - withVersions('graphql', 'apollo-server-core', apolloVersion => { + withVersions(plugin, 'apollo-server-core', apolloVersion => { // The precense of graphql@^15.2.0 in the /versions folder causes graphql-tools@3.1.1 // to break in the before() hook. This test tests a library version that had its release occur 5 years ago // updating the test would require using newer version of apollo-core which have a completely different syntax @@ -1635,7 +1754,7 @@ describe('Plugin', () => { graphql = require(`../../../versions/graphql@${version}`).get() const apolloCore = require(`../../../versions/apollo-server-core@${apolloVersion}`).get() - const graphqlTools = require(`../../../versions/graphql-tools@3.1.1`).get() + const graphqlTools = require('../../../versions/graphql-tools@3.1.1').get() runQuery = apolloCore.runQuery mergeSchemas = graphqlTools.mergeSchemas diff --git a/packages/datadog-plugin-graphql/test/integration-test/client.spec.js b/packages/datadog-plugin-graphql/test/integration-test/client.spec.js index 0deed5fbc17..d0a4b0e42d3 100644 --- a/packages/datadog-plugin-graphql/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-graphql/test/integration-test/client.spec.js @@ -17,7 +17,7 @@ describe('esm', () => { before(async function () { this.timeout(50000) sandbox = await createSandbox([`'graphql@${version}'`], false, [ - `./packages/datadog-plugin-graphql/test/integration-test/*`]) + './packages/datadog-plugin-graphql/test/integration-test/*']) }) after(async function () { diff --git a/packages/datadog-plugin-graphql/test/leak.js b/packages/datadog-plugin-graphql/test/leak.js deleted file mode 100644 index d03a4384038..00000000000 --- a/packages/datadog-plugin-graphql/test/leak.js +++ /dev/null @@ -1,25 +0,0 @@ -'use strict' - -require('../../dd-trace') - .init({ plugins: false, sampleRate: 0 }) - .use('graphql') - -const test = require('tape') -const graphql = require(`../../../versions/graphql`).get() -const profile = require('../../dd-trace/test/profile') - -test('graphql plugin should not leak', t => { - const schema = graphql.buildSchema(` - type Query { - hello: String - } - `) - - const source = `{ hello }` - - profile(t, operation, 2000) - - function operation (done) { - graphql.graphql(schema, source).then(done) - } -}) diff --git a/packages/datadog-plugin-grpc/src/client.js b/packages/datadog-plugin-grpc/src/client.js index 8cc6dfdb91f..1b130a1f93e 100644 --- a/packages/datadog-plugin-grpc/src/client.js +++ b/packages/datadog-plugin-grpc/src/client.js @@ -8,7 +8,7 @@ const { addMetadataTags, getFilter, getMethodMetadata } = require('./util') class GrpcClientPlugin extends ClientPlugin { static get id () { return 'grpc' } static get operation () { return 'client:request' } - static get prefix () { return `apm:grpc:client:request` } + static get prefix () { return 'apm:grpc:client:request' } static get peerServicePrecursors () { return ['rpc.service'] } constructor (...args) { @@ -30,7 +30,7 @@ class GrpcClientPlugin extends ClientPlugin { kind: 'client', type: 'http', meta: { - 'component': 'grpc', + component: 'grpc', 'grpc.method.kind': method.kind, 'grpc.method.path': method.path, 'grpc.method.name': method.name, @@ -41,7 +41,6 @@ class GrpcClientPlugin extends ClientPlugin { 'grpc.status.code': 0 } }, false) - // needed as precursor for peer.service if (method.service && method.package) { span.setTag('rpc.service', method.package + '.' + method.service) @@ -65,10 +64,13 @@ class GrpcClientPlugin extends ClientPlugin { error ({ span, error }) { this.addCode(span, error.code) + if (error.code && !this._tracerConfig.grpc.client.error.statuses.includes(error.code)) { + return + } this.addError(error, span) } - finish ({ span, result }) { + finish ({ span, result, peer }) { if (!span) return const { code, metadata } = result || {} @@ -80,6 +82,21 @@ class GrpcClientPlugin extends ClientPlugin { addMetadataTags(span, metadata, metadataFilter, 'response') } + if (peer) { + // The only scheme we want to support here is ipv[46]:port, although + // more are supported by the library + // https://github.com/grpc/grpc/blob/v1.60.0/doc/naming.md + const parts = peer.split(':') + if (parts[parts.length - 1].match(/^\d+/)) { + const port = parts[parts.length - 1] + const ip = parts.slice(0, -1).join(':') + span.setTag('network.destination.ip', ip) + span.setTag('network.destination.port', port) + } else { + span.setTag('network.destination.ip', peer) + } + } + this.tagPeerService(span) span.finish() } diff --git a/packages/datadog-plugin-grpc/src/server.js b/packages/datadog-plugin-grpc/src/server.js index 42e00f1dafd..0b599a1283d 100644 --- a/packages/datadog-plugin-grpc/src/server.js +++ b/packages/datadog-plugin-grpc/src/server.js @@ -8,7 +8,7 @@ const { addMetadataTags, getFilter, getMethodMetadata } = require('./util') class GrpcServerPlugin extends ServerPlugin { static get id () { return 'grpc' } static get operation () { return 'server:request' } - static get prefix () { return `apm:grpc:server:request` } + static get prefix () { return 'apm:grpc:server:request' } constructor (...args) { super(...args) @@ -39,7 +39,7 @@ class GrpcServerPlugin extends ServerPlugin { kind: 'server', type: 'web', meta: { - 'component': 'grpc', + component: 'grpc', 'grpc.method.kind': method.kind, 'grpc.method.path': method.path, 'grpc.method.name': method.name, @@ -70,6 +70,9 @@ class GrpcServerPlugin extends ServerPlugin { if (!span) return this.addCode(span, error.code) + if (error.code && !this._tracerConfig.grpc.server.error.statuses.includes(error.code)) { + return + } this.addError(error) } diff --git a/packages/datadog-plugin-grpc/src/util.js b/packages/datadog-plugin-grpc/src/util.js index 1df9531b57f..1c1937e7ea7 100644 --- a/packages/datadog-plugin-grpc/src/util.js +++ b/packages/datadog-plugin-grpc/src/util.js @@ -1,6 +1,6 @@ 'use strict' -const pick = require('lodash.pick') +const pick = require('../../datadog-core/src/utils/src/pick') const log = require('../../dd-trace/src/log') module.exports = { diff --git a/packages/datadog-plugin-grpc/test/client.spec.js b/packages/datadog-plugin-grpc/test/client.spec.js index 17316b648e6..4628fb8a5f6 100644 --- a/packages/datadog-plugin-grpc/test/client.spec.js +++ b/packages/datadog-plugin-grpc/test/client.spec.js @@ -1,11 +1,13 @@ 'use strict' +const path = require('path') const agent = require('../../dd-trace/test/plugins/agent') const getPort = require('get-port') +const semver = require('semver') const Readable = require('stream').Readable const getService = require('./service') const loader = require('../../../versions/@grpc/proto-loader').get() -const { ERROR_MESSAGE, ERROR_TYPE, ERROR_STACK } = require('../../dd-trace/src/constants') +const { ERROR_MESSAGE, ERROR_TYPE, ERROR_STACK, GRPC_CLIENT_ERROR_STATUSES } = require('../../dd-trace/src/constants') const { DD_MAJOR } = require('../../../version') const nodeMajor = parseInt(process.versions.node.split('.')[0]) @@ -42,20 +44,20 @@ describe('Plugin', () => { server.addService(TestService.service, service) server.start() - resolve(new ClientService(`localhost:${port}`, grpc.credentials.createInsecure())) + resolve(new ClientService(`127.0.0.1:${port}`, grpc.credentials.createInsecure())) }) } else { server.bind(`127.0.0.1:${port}`, grpc.ServerCredentials.createInsecure()) server.addService(TestService.service, service) server.start() - resolve(new ClientService(`localhost:${port}`, grpc.credentials.createInsecure())) + resolve(new ClientService(`127.0.0.1:${port}`, grpc.credentials.createInsecure())) } }) } function buildProtoClient (service, ClientService) { - const definition = loader.loadSync(`${__dirname}/test.proto`) + const definition = loader.loadSync(path.join(__dirname, 'test.proto')) const TestService = grpc.loadPackageDefinition(definition).test.TestService return buildGenericService(service, TestService, ClientService) @@ -126,6 +128,26 @@ describe('Plugin', () => { } ) + if (semver.intersects(version, '>=1.1.4')) { + it('should provide host information', async () => { + const client = await buildClient({ + getUnary: (_, callback) => callback() + }) + + client.getUnary({ first: 'foobar' }, () => {}) + return agent + .use(traces => { + expect(traces[0][0].meta).to.include({ + 'network.destination.ip': '127.0.0.1', + 'network.destination.port': port.toString(), + 'rpc.service': 'test.TestService', + 'span.kind': 'client', + component: 'grpc' + }) + }) + }) + } + it('should handle `unary` calls', async () => { const client = await buildClient({ getUnary: (_, callback) => callback() @@ -149,7 +171,7 @@ describe('Plugin', () => { 'grpc.method.kind': 'unary', 'rpc.service': 'test.TestService', 'span.kind': 'client', - 'component': 'grpc' + component: 'grpc' }) expect(traces[0][0].metrics).to.include({ @@ -186,7 +208,7 @@ describe('Plugin', () => { 'grpc.method.kind': 'server_streaming', 'rpc.service': 'test.TestService', 'span.kind': 'client', - 'component': 'grpc' + component: 'grpc' }) expect(traces[0][0].metrics).to.include({ @@ -221,7 +243,7 @@ describe('Plugin', () => { 'grpc.method.kind': 'client_streaming', 'rpc.service': 'test.TestService', 'span.kind': 'client', - 'component': 'grpc' + component: 'grpc' }) expect(traces[0][0].metrics).to.include({ @@ -324,15 +346,32 @@ describe('Plugin', () => { 'grpc.method.kind': 'unary', 'rpc.service': 'test.TestService', 'span.kind': 'client', - 'component': 'grpc' + component: 'grpc' }) expect(traces[0][0].meta).to.have.property(ERROR_STACK) expect(traces[0][0].metrics).to.have.property('grpc.status.code', 2) }) }) + it('should ignore errors not set by DD_GRPC_CLIENT_ERROR_STATUSES', async () => { + tracer._tracer._config.grpc.client.error.statuses = [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13] + const client = await buildClient({ + getUnary: (_, callback) => callback(new Error('foobar')) + }) + + client.getUnary({ first: 'foobar' }, () => {}) + + return agent + .use(traces => { + expect(traces[0][0]).to.have.property('error', 0) + expect(traces[0][0].metrics).to.have.property('grpc.status.code', 2) + tracer._tracer._config.grpc.client.error.statuses = + GRPC_CLIENT_ERROR_STATUSES + }) + }) + it('should handle protocol errors', async () => { - const definition = loader.loadSync(`${__dirname}/invalid.proto`) + const definition = loader.loadSync(path.join(__dirname, 'invalid.proto')) const test = grpc.loadPackageDefinition(definition).test const client = await buildClient({ getUnary: (_, callback) => callback(null) @@ -352,7 +391,7 @@ describe('Plugin', () => { 'grpc.method.kind': 'unary', 'rpc.service': 'test.TestService', 'span.kind': 'client', - 'component': 'grpc' + component: 'grpc' }) expect(traces[0][0].meta).to.have.property(ERROR_STACK) expect(traces[0][0].meta[ERROR_MESSAGE]).to.match(/^13 INTERNAL:.+$/m) @@ -361,7 +400,7 @@ describe('Plugin', () => { }) it('should handle property named "service"', async () => { - const definition = loader.loadSync(`${__dirname}/hasservice.proto`) + const definition = loader.loadSync(path.join(__dirname, 'hasservice.proto')) const thing = grpc.loadPackageDefinition(definition).thing await buildClient({ getUnary: (_, callback) => callback(null) @@ -391,7 +430,7 @@ describe('Plugin', () => { 'rpc.service': 'test.TestService', 'grpc.method.kind': 'unary', 'span.kind': 'client', - 'component': 'grpc' + component: 'grpc' }) expect(traces[0][0].metrics).to.deep.include({ @@ -423,7 +462,7 @@ describe('Plugin', () => { 'grpc.method.kind': 'unary', 'rpc.service': 'test.TestService', 'span.kind': 'client', - 'component': 'grpc' + component: 'grpc' }) expect(traces[0][0].metrics).to.deep.include({ diff --git a/packages/datadog-plugin-grpc/test/integration-test/client.spec.js b/packages/datadog-plugin-grpc/test/integration-test/client.spec.js index 5b6b864d2ff..c9a31f5c65e 100644 --- a/packages/datadog-plugin-grpc/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-grpc/test/integration-test/client.spec.js @@ -17,7 +17,7 @@ describe('esm', () => { before(async function () { this.timeout(20000) sandbox = await createSandbox([`'@grpc/grpc-js@${version}'`, '@grpc/proto-loader', 'get-port@^3.2.0'], false, [ - `./packages/datadog-plugin-grpc/test/*`]) + './packages/datadog-plugin-grpc/test/*']) }) after(async () => { diff --git a/packages/datadog-plugin-grpc/test/naming.js b/packages/datadog-plugin-grpc/test/naming.js index 25e6706e10a..f84571b78e5 100644 --- a/packages/datadog-plugin-grpc/test/naming.js +++ b/packages/datadog-plugin-grpc/test/naming.js @@ -25,6 +25,6 @@ const rawExpectedSchema = { } module.exports = { - rawExpectedSchema: rawExpectedSchema, + rawExpectedSchema, expectedSchema: resolveNaming(rawExpectedSchema) } diff --git a/packages/datadog-plugin-grpc/test/server.spec.js b/packages/datadog-plugin-grpc/test/server.spec.js index a049bbfc4cb..cf695840303 100644 --- a/packages/datadog-plugin-grpc/test/server.spec.js +++ b/packages/datadog-plugin-grpc/test/server.spec.js @@ -1,10 +1,11 @@ 'use strict' +const path = require('path') const agent = require('../../dd-trace/test/plugins/agent') const getPort = require('get-port') const Readable = require('stream').Readable -const { ERROR_MESSAGE, ERROR_TYPE, ERROR_STACK } = require('../../dd-trace/src/constants') +const { ERROR_MESSAGE, ERROR_TYPE, ERROR_STACK, GRPC_SERVER_ERROR_STATUSES } = require('../../dd-trace/src/constants') const nodeMajor = parseInt(process.versions.node.split('.')[0]) const pkgs = nodeMajor > 14 ? ['@grpc/grpc-js'] : ['grpc', '@grpc/grpc-js'] @@ -27,7 +28,7 @@ describe('Plugin', () => { }, service) const loader = require('../../../versions/@grpc/proto-loader').get() - const definition = loader.loadSync(`${__dirname}/test.proto`) + const definition = loader.loadSync(path.join(__dirname, 'test.proto')) const TestService = grpc.loadPackageDefinition(definition).test.TestService server = new grpc.Server() @@ -275,6 +276,38 @@ describe('Plugin', () => { }) }) + it('should ignore errors not set by DD_GRPC_SERVER_ERROR_STATUSES', async () => { + tracer._tracer._config.grpc.server.error.statuses = [6, 7, 8, 9, 10, 11, 12, 13] + const client = await buildClient({ + getUnary: (_, callback) => { + const metadata = new grpc.Metadata() + + metadata.set('extra', 'information') + + const error = new Error('foobar') + + error.code = grpc.status.NOT_FOUND + + const childOf = tracer.scope().active() + const child = tracer.startSpan('child', { childOf }) + + // Delay trace to ensure auto-cancellation doesn't override the status code. + setTimeout(() => child.finish()) + + callback(error, {}, metadata) + } + }) + + client.getUnary({ first: 'foobar' }, () => {}) + + return agent + .use(traces => { + expect(traces[0][0]).to.have.property('error', 0) + expect(traces[0][0].metrics).to.have.property('grpc.status.code', 5) + tracer._tracer._config.grpc.server.error.statuses = GRPC_SERVER_ERROR_STATUSES + }) + }) + it('should handle custom errors', async () => { const client = await buildClient({ getUnary: (_, callback) => { diff --git a/packages/datadog-plugin-hapi/src/index.js b/packages/datadog-plugin-hapi/src/index.js index e4954acd79e..b72df6951b0 100644 --- a/packages/datadog-plugin-hapi/src/index.js +++ b/packages/datadog-plugin-hapi/src/index.js @@ -32,8 +32,8 @@ class HapiPlugin extends RouterPlugin { } }) - this.addSub('apm:hapi:extension:enter', ({ req }) => { - this.enter(this._requestSpans.get(req)) + this.addBind('apm:hapi:extension:start', ({ req }) => { + return this._requestSpans.get(req) }) } } diff --git a/packages/datadog-plugin-hapi/test/index.spec.js b/packages/datadog-plugin-hapi/test/index.spec.js index 9f2837612cd..48093e29044 100644 --- a/packages/datadog-plugin-hapi/test/index.spec.js +++ b/packages/datadog-plugin-hapi/test/index.spec.js @@ -1,10 +1,10 @@ 'use strict' const axios = require('axios') -const getPort = require('get-port') const semver = require('semver') const agent = require('../../dd-trace/test/plugins/agent') const { ERROR_MESSAGE, ERROR_TYPE, ERROR_STACK } = require('../../dd-trace/src/constants') +const { AsyncLocalStorage } = require('async_hooks') const versionRange = parseInt(process.versions.node.split('.')[0]) > 14 ? '<17 || >18' @@ -47,15 +47,13 @@ describe('Plugin', () => { if (semver.intersects(version, '>=17')) { beforeEach(() => { - return getPort() - .then(_port => { - port = _port - server = Hapi.server({ - address: '127.0.0.1', - port - }) - return server.start() - }) + server = Hapi.server({ + address: 'localhost', + port: 0 + }) + return server.start().then(() => { + port = server.listener.address().port + }) }) afterEach(() => { @@ -63,19 +61,19 @@ describe('Plugin', () => { }) } else { beforeEach(done => { - getPort() - .then(_port => { - port = _port - - if (Hapi.Server.prototype.connection) { - server = new Hapi.Server() - server.connection({ address: '127.0.0.1', port }) - } else { - server = new Hapi.Server('127.0.0.1', port) - } + if (Hapi.Server.prototype.connection) { + server = new Hapi.Server() + server.connection({ address: 'localhost', port }) + } else { + server = new Hapi.Server('localhost', port) + } - server.start(done) - }) + server.start(err => { + if (!err) { + port = server.listener.address().port + } + done(err) + }) }) afterEach(done => { @@ -351,6 +349,30 @@ describe('Plugin', () => { .get(`http://localhost:${port}/user/123`) .catch(() => {}) }) + + it('should persist AsyncLocalStorage context', (done) => { + const als = new AsyncLocalStorage() + const path = '/path' + + server.ext('onRequest', (request, h) => { + als.enterWith({ path: request.path }) + return reply(request, h) + }) + + server.route({ + method: 'GET', + path, + handler: async (request, h) => { + expect(als.getStore()).to.deep.equal({ path }) + done() + return h.response ? h.response() : h() + } + }) + + axios + .get(`http://localhost:${port}${path}`) + .catch(() => {}) + }) }) }) }) diff --git a/packages/datadog-plugin-hapi/test/integration-test/client.spec.js b/packages/datadog-plugin-hapi/test/integration-test/client.spec.js index e483b593fe5..3105f810458 100644 --- a/packages/datadog-plugin-hapi/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-hapi/test/integration-test/client.spec.js @@ -18,7 +18,7 @@ describe('esm', () => { before(async function () { this.timeout(20000) sandbox = await createSandbox([`'@hapi/hapi@${version}'`], false, [ - `./packages/datadog-plugin-hapi/test/integration-test/*`]) + './packages/datadog-plugin-hapi/test/integration-test/*']) }) after(async () => { diff --git a/packages/datadog-plugin-http/src/client.js b/packages/datadog-plugin-http/src/client.js index 42833bb896f..55a025f4970 100644 --- a/packages/datadog-plugin-http/src/client.js +++ b/packages/datadog-plugin-http/src/client.js @@ -17,7 +17,7 @@ const HTTP_RESPONSE_HEADERS = tags.HTTP_RESPONSE_HEADERS class HttpClientPlugin extends ClientPlugin { static get id () { return 'http' } - static get prefix () { return `apm:http:client:request` } + static get prefix () { return 'apm:http:client:request' } bindStart (message) { const { args, http = {} } = message @@ -58,7 +58,7 @@ class HttpClientPlugin extends ClientPlugin { span._spanContext._trace.record = false } - if (this.shouldInjectTraceHeaders(options, uri)) { + if (this.config.propagationFilter(uri)) { this.tracer.inject(span, HTTP_HEADERS, options.headers) } @@ -71,18 +71,6 @@ class HttpClientPlugin extends ClientPlugin { return message.currentStore } - shouldInjectTraceHeaders (options, uri) { - if (hasAmazonSignature(options) && !this.config.enablePropagationWithAmazonHeaders) { - return false - } - - if (!this.config.propagationFilter(uri)) { - return false - } - - return true - } - bindAsyncStart ({ parentStore }) { return parentStore } @@ -122,7 +110,7 @@ class HttpClientPlugin extends ClientPlugin { // conditions for no error: // 1. not using a custom agent instance with custom timeout specified // 2. no invocation of `req.setTimeout` - if (!args.options.agent?.options.timeout && !customRequestTimeout) return + if (!args.options.agent?.options?.timeout && !customRequestTimeout) return span.setTag('error', 1) } @@ -212,31 +200,6 @@ function getHooks (config) { return { request } } -function hasAmazonSignature (options) { - if (!options) { - return false - } - - if (options.headers) { - const headers = Object.keys(options.headers) - .reduce((prev, next) => Object.assign(prev, { - [next.toLowerCase()]: options.headers[next] - }), {}) - - if (headers['x-amz-signature']) { - return true - } - - if ([].concat(headers['authorization']).some(startsWith('AWS4-HMAC-SHA256'))) { - return true - } - } - - const search = options.search || options.path - - return search && search.toLowerCase().indexOf('x-amz-signature=') !== -1 -} - function extractSessionDetails (options) { if (typeof options === 'string') { return new URL(options).host @@ -248,8 +211,4 @@ function extractSessionDetails (options) { return { host, port } } -function startsWith (searchString) { - return value => String(value).startsWith(searchString) -} - module.exports = HttpClientPlugin diff --git a/packages/datadog-plugin-http/test/client.spec.js b/packages/datadog-plugin-http/test/client.spec.js index 7256950ac83..268aff9b238 100644 --- a/packages/datadog-plugin-http/test/client.spec.js +++ b/packages/datadog-plugin-http/test/client.spec.js @@ -1,6 +1,5 @@ 'use strict' -const getPort = require('get-port') const agent = require('../../dd-trace/test/plugins/agent') const fs = require('fs') const path = require('path') @@ -25,17 +24,25 @@ describe('Plugin', () => { let appListener let tracer - ['http', 'https'].forEach(protocol => { - describe(protocol, () => { - function server (app, port, listener) { + ['http', 'https', 'node:http', 'node:https'].forEach(pluginToBeLoaded => { + const protocol = pluginToBeLoaded.split(':')[1] || pluginToBeLoaded + describe(pluginToBeLoaded, () => { + function server (app, listener) { let server - if (protocol === 'https') { + if (pluginToBeLoaded === 'https') { process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0' server = require('https').createServer({ key, cert }, app) - } else { + } else if (pluginToBeLoaded === 'node:https') { + process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0' + server = require('node:https').createServer({ key, cert }, app) + } else if (pluginToBeLoaded === 'http') { server = require('http').createServer(app) + } else { + server = require('node:http').createServer(app) } - server.listen(port, 'localhost', listener) + server.listen(0, 'localhost', () => { + listener(server.address().port) + }) return server } @@ -55,7 +62,7 @@ describe('Plugin', () => { beforeEach(() => { return agent.load('http', { server: false }) .then(() => { - http = require(protocol) + http = require(pluginToBeLoaded) express = require('express') }) }) @@ -65,13 +72,12 @@ describe('Plugin', () => { app.get('/user', (req, res) => { res.status(200).send() }) - getPort().then(port => { - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - res.on('data', () => {}) - }) - req.end() + appListener = server(app, () => { + const port = appListener.address().port + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + res.on('data', () => {}) }) + req.end() }) } @@ -93,7 +99,8 @@ describe('Plugin', () => { app.get('/user', (req, res) => { res.status(200).send() }) - getPort().then(port => { + + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', SERVICE_NAME) @@ -109,39 +116,35 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - res.on('data', () => {}) - }) - - req.end() + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + res.on('data', () => {}) }) + + req.end() }) }) - it(`should also support get()`, done => { + it('should also support get()', done => { const app = express() app.get('/user', (req, res) => { res.status(200).send() }) - getPort().then(port => { - agent - .use(traces => { - expect(traces[0][0].meta).to.have.property('http.status_code', '200') - expect(traces[0][0]).to.not.be.undefined - }) - .then(done) - .catch(done) - - appListener = server(app, port, () => { - const req = http.get(`${protocol}://localhost:${port}/user`, res => { - res.on('data', () => {}) - }) + agent + .use(traces => { + expect(traces[0][0].meta).to.have.property('http.status_code', '200') + expect(traces[0][0]).to.not.be.undefined + }) + .then(done) + .catch(done) - req.end() + appListener = server(app, port => { + const req = http.get(`${protocol}://localhost:${port}/user`, res => { + res.on('data', () => {}) }) + + req.end() }) }) @@ -150,7 +153,8 @@ describe('Plugin', () => { app.get('/user', (req, res) => { res.status(200).send() }) - getPort().then(port => { + + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', SERVICE_NAME) @@ -166,27 +170,25 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - appListener.on('connect', (req, clientSocket, head) => { - clientSocket.write('HTTP/1.1 200 Connection Established\r\n' + + appListener.on('connect', (req, clientSocket, head) => { + clientSocket.write('HTTP/1.1 200 Connection Established\r\n' + 'Proxy-agent: Node.js-Proxy\r\n' + '\r\n') - clientSocket.end() - appListener.close() - }) - - const req = http.request({ - protocol: `${protocol}:`, - port, - method: 'CONNECT', - hostname: 'localhost', - path: '/user' - }) + clientSocket.end() + appListener.close() + }) - req.on('connect', (res, socket) => socket.end()) - req.on('error', () => {}) - req.end() + const req = http.request({ + protocol: `${protocol}:`, + port, + method: 'CONNECT', + hostname: 'localhost', + path: '/user' }) + + req.on('connect', (res, socket) => socket.end()) + req.on('error', () => {}) + req.end() }) }) @@ -195,7 +197,8 @@ describe('Plugin', () => { app.get('/user', (req, res) => { res.status(200).send() }) - getPort().then(port => { + + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', SERVICE_NAME) @@ -210,30 +213,28 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - appListener.on('upgrade', (req, socket, head) => { - socket.write('HTTP/1.1 101 Web Socket Protocol Handshake\r\n' + + appListener.on('upgrade', (req, socket, head) => { + socket.write('HTTP/1.1 101 Web Socket Protocol Handshake\r\n' + 'Upgrade: WebSocket\r\n' + 'Connection: Upgrade\r\n' + '\r\n') - socket.pipe(socket) - }) - - const req = http.request({ - protocol: `${protocol}:`, - port, - hostname: 'localhost', - path: '/user', - headers: { - 'Connection': 'Upgrade', - 'Upgrade': 'websocket' - } - }) + socket.pipe(socket) + }) - req.on('upgrade', (res, socket) => socket.end()) - req.on('error', () => {}) - req.end() + const req = http.request({ + protocol: `${protocol}:`, + port, + hostname: 'localhost', + path: '/user', + headers: { + Connection: 'Upgrade', + Upgrade: 'websocket' + } }) + + req.on('upgrade', (res, socket) => socket.end()) + req.on('error', () => {}) + req.end() }) }) @@ -243,7 +244,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.url', `${protocol}://localhost:${port}/user`) @@ -257,12 +258,9 @@ describe('Plugin', () => { port, path: '/user' } + const req = http.request(uri) - appListener = server(app, port, () => { - const req = http.request(uri) - - req.end() - }) + req.end() }) }) @@ -273,7 +271,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.status_code', '200') @@ -282,13 +280,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user?foo=bar`, res => { - res.on('data', () => {}) - }) - - req.end() + const req = http.request(`${protocol}://localhost:${port}/user?foo=bar`, res => { + res.on('data', () => {}) }) + + req.end() }) }) @@ -301,7 +297,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.status_code', '200') @@ -310,11 +306,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/another-path`, { path: '/user' }) + const req = http.request(`${protocol}://localhost:${port}/another-path`, { path: '/user' }) - req.end() - }) + req.end() }) }) } @@ -326,7 +320,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.status_code', '200') @@ -341,31 +335,28 @@ describe('Plugin', () => { port, pathname: '/another-path' } + const req = http.request(uri, { path: '/user' }) - appListener = server(app, port, () => { - const req = http.request(uri, { path: '/user' }) - - req.end() - }) + req.end() }) }) - it('should support configuration as an WHATWG URL object', async () => { + it('should support configuration as an WHATWG URL object', done => { const app = express() - const port = await getPort() - const url = new URL(`${protocol}://localhost:${port}/user`) - app.get('/user', (req, res) => res.status(200).send()) + appListener = server(app, port => { + const url = new URL(`${protocol}://localhost:${port}/user`) + + app.get('/user', (req, res) => res.status(200).send()) + + agent.use(traces => { + expect(traces[0][0].meta).to.have.property('http.status_code', '200') + expect(traces[0][0].meta).to.have.property('http.url', `${protocol}://localhost:${port}/user`) + }).then(done, done) - appListener = server(app, port, () => { const req = http.request(url) req.end() }) - - await agent.use(traces => { - expect(traces[0][0].meta).to.have.property('http.status_code', '200') - expect(traces[0][0].meta).to.have.property('http.url', `${protocol}://localhost:${port}/user`) - }) }) it('should use the correct defaults when not specified', done => { @@ -375,7 +366,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.url', `${protocol}://localhost:${port}/`) @@ -383,14 +374,12 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const req = http.request({ - protocol: `${protocol}:`, - port - }) - - req.end() + const req = http.request({ + protocol: `${protocol}:`, + port }) + + req.end() }) }) @@ -401,19 +390,17 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - agent - .use(traces => { - expect(traces[0][0]).to.not.be.undefined - }) - .then(done) - .catch(done) + agent + .use(traces => { + expect(traces[0][0]).to.not.be.undefined + }) + .then(done) + .catch(done) - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`) + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`) - req.end() - }) + req.end() }) }) @@ -424,16 +411,14 @@ describe('Plugin', () => { res.status(200).send('OK') }) - getPort().then(port => { - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - setTimeout(() => { - res.on('data', () => done()) - }) + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + setTimeout(() => { + res.on('data', () => done()) }) - - req.end() }) + + req.end() }) }) @@ -447,137 +432,17 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - agent - .use(traces => { - expect(traces[0][0].meta).to.have.property('http.status_code', '200') - }) - .then(done) - .catch(done) - - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`) - - req.end() - }) - }) - }) - - it('should skip injecting if the Authorization header contains an AWS signature', done => { - const app = express() - - app.get('/', (req, res) => { - try { - expect(req.get('x-datadog-trace-id')).to.be.undefined - expect(req.get('x-datadog-parent-id')).to.be.undefined - - res.status(200).send() - - done() - } catch (e) { - done(e) - } - }) - - getPort().then(port => { - appListener = server(app, port, () => { - const req = http.request({ - port, - headers: { - Authorization: 'AWS4-HMAC-SHA256 ...' - } - }) - - req.end() - }) - }) - }) - - it('should skip injecting if one of the Authorization headers contains an AWS signature', done => { - const app = express() - - app.get('/', (req, res) => { - try { - expect(req.get('x-datadog-trace-id')).to.be.undefined - expect(req.get('x-datadog-parent-id')).to.be.undefined - - res.status(200).send() - - done() - } catch (e) { - done(e) - } - }) - - getPort().then(port => { - appListener = server(app, port, () => { - const req = http.request({ - port, - headers: { - Authorization: ['AWS4-HMAC-SHA256 ...'] - } - }) - - req.end() + agent + .use(traces => { + expect(traces[0][0].meta).to.have.property('http.status_code', '200') }) - }) - }) - - it('should skip injecting if the X-Amz-Signature header is set', done => { - const app = express() - - app.get('/', (req, res) => { - try { - expect(req.get('x-datadog-trace-id')).to.be.undefined - expect(req.get('x-datadog-parent-id')).to.be.undefined - - res.status(200).send() - - done() - } catch (e) { - done(e) - } - }) - - getPort().then(port => { - appListener = server(app, port, () => { - const req = http.request({ - port, - headers: { - 'X-Amz-Signature': 'abc123' - } - }) - - req.end() - }) - }) - }) - - it('should skip injecting if the X-Amz-Signature query param is set', done => { - const app = express() - - app.get('/', (req, res) => { - try { - expect(req.get('x-datadog-trace-id')).to.be.undefined - expect(req.get('x-datadog-parent-id')).to.be.undefined + .then(done) + .catch(done) - res.status(200).send() - - done() - } catch (e) { - done(e) - } - }) - - getPort().then(port => { - appListener = server(app, port, () => { - const req = http.request({ - port, - path: '/?X-Amz-Signature=abc123' - }) + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`) - req.end() - }) + req.end() }) }) @@ -587,15 +452,14 @@ describe('Plugin', () => { app.get('/user', (req, res) => { res.status(200).send('OK') }) - getPort().then(port => { - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - expect(tracer.scope().active()).to.be.null - done() - }) - req.end() + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + expect(tracer.scope().active()).to.be.null + done() }) + + req.end() }) }) @@ -606,20 +470,18 @@ describe('Plugin', () => { res.status(200).send('OK') }) - getPort().then(port => { - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - const span = tracer.scope().active() + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + const span = tracer.scope().active() - res.on('data', () => {}) - res.on('end', () => { - expect(tracer.scope().active()).to.equal(span) - done() - }) + res.on('data', () => {}) + res.on('end', () => { + expect(tracer.scope().active()).to.equal(span) + done() }) - - req.end() }) + + req.end() }) }) @@ -630,42 +492,38 @@ describe('Plugin', () => { res.status(200).send('OK') }) - getPort().then(port => { - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`, () => {}) - - req.on('response', () => { - expect(tracer.scope().active()).to.not.be.null - done() - }) + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`, () => {}) - req.end() + req.on('response', () => { + expect(tracer.scope().active()).to.not.be.null + done() }) + + req.end() }) }) it('should handle connection errors', done => { - getPort().then(port => { - let error - - agent - .use(traces => { - expect(traces[0][0].meta).to.have.property(ERROR_TYPE, error.name) - expect(traces[0][0].meta).to.have.property(ERROR_MESSAGE, error.message || error.code) - expect(traces[0][0].meta).to.have.property(ERROR_STACK, error.stack) - expect(traces[0][0].meta).to.have.property('component', 'http') - }) - .then(done) - .catch(done) - - const req = http.request(`${protocol}://localhost:${port}/user`) - - req.on('error', err => { - error = err + let error + + agent + .use(traces => { + expect(traces[0][0].meta).to.have.property(ERROR_TYPE, error.name) + expect(traces[0][0].meta).to.have.property(ERROR_MESSAGE, error.message || error.code) + expect(traces[0][0].meta).to.have.property(ERROR_STACK, error.stack) + expect(traces[0][0].meta).to.have.property('component', 'http') }) + .then(done) + .catch(done) - req.end() + const req = http.request(`${protocol}://localhost:7357/user`) + + req.on('error', err => { + error = err }) + + req.end() }) it('should not record HTTP 5XX responses as errors by default', done => { @@ -675,21 +533,19 @@ describe('Plugin', () => { res.status(500).send() }) - getPort().then(port => { - agent - .use(traces => { - expect(traces[0][0]).to.have.property('error', 0) - }) - .then(done) - .catch(done) - - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - res.on('data', () => { }) - }) + agent + .use(traces => { + expect(traces[0][0]).to.have.property('error', 0) + }) + .then(done) + .catch(done) - req.end() + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + res.on('data', () => { }) }) + + req.end() }) }) @@ -700,21 +556,19 @@ describe('Plugin', () => { res.status(400).send() }) - getPort().then(port => { - agent - .use(traces => { - expect(traces[0][0]).to.have.property('error', 1) - }) - .then(done) - .catch(done) - - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - res.on('data', () => { }) - }) + agent + .use(traces => { + expect(traces[0][0]).to.have.property('error', 1) + }) + .then(done) + .catch(done) - req.end() + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + res.on('data', () => { }) }) + + req.end() }) }) @@ -723,32 +577,30 @@ describe('Plugin', () => { app.get('/user', (req, res) => {}) - getPort().then(port => { - let error - - agent - .use(traces => { - expect(traces[0][0]).to.have.property('error', 1) - expect(traces[0][0].meta).to.have.property(ERROR_MESSAGE, error.message) - expect(traces[0][0].meta).to.have.property(ERROR_TYPE, error.name) - expect(traces[0][0].meta).to.have.property(ERROR_STACK, error.stack) - expect(traces[0][0].meta).to.not.have.property('http.status_code') - expect(traces[0][0].meta).to.have.property('component', 'http') - }) - .then(done) - .catch(done) + let error - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - res.on('data', () => { }) - }) + agent + .use(traces => { + expect(traces[0][0]).to.have.property('error', 1) + expect(traces[0][0].meta).to.have.property(ERROR_MESSAGE, error.message) + expect(traces[0][0].meta).to.have.property(ERROR_TYPE, error.name) + expect(traces[0][0].meta).to.have.property(ERROR_STACK, error.stack) + expect(traces[0][0].meta).to.not.have.property('http.status_code') + expect(traces[0][0].meta).to.have.property('component', 'http') + }) + .then(done) + .catch(done) - req.on('error', err => { - error = err - }) + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + res.on('data', () => { }) + }) - req.destroy() + req.on('error', err => { + error = err }) + + req.destroy() }) }) @@ -757,24 +609,22 @@ describe('Plugin', () => { app.get('/user', (req, res) => {}) - getPort().then(port => { - agent - .use(traces => { - expect(traces[0][0]).to.have.property('error', 0) - expect(traces[0][0].meta).to.not.have.property('http.status_code') - }) - .then(done) - .catch(done) + agent + .use(traces => { + expect(traces[0][0]).to.have.property('error', 0) + expect(traces[0][0].meta).to.not.have.property('http.status_code') + }) + .then(done) + .catch(done) - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - res.on('data', () => { }) - }) + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + res.on('data', () => { }) + }) - req.on('abort', () => {}) + req.on('abort', () => {}) - req.abort() - }) + req.abort() }) }) @@ -783,25 +633,23 @@ describe('Plugin', () => { app.get('/user', (req, res) => {}) - getPort().then(port => { - agent - .use(traces => { - expect(traces[0][0]).to.have.property('error', 1) - expect(traces[0][0].meta).to.not.have.property('http.status_code') - }) - .then(done) - .catch(done) + agent + .use(traces => { + expect(traces[0][0]).to.have.property('error', 1) + expect(traces[0][0].meta).to.not.have.property('http.status_code') + }) + .then(done) + .catch(done) - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - res.on('data', () => { }) - }) + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + res.on('data', () => { }) + }) - req.on('error', () => {}) + req.on('error', () => {}) - req.setTimeout(1) - req.end() - }) + req.setTimeout(1) + req.end() }) }) @@ -816,23 +664,21 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - agent - .use(traces => { - expect(traces[0][0]).to.have.property('error', 0) - }) - .then(done) - .catch(done) + agent + .use(traces => { + expect(traces[0][0]).to.have.property('error', 0) + }) + .then(done) + .catch(done) - appListener = server(app, port, async () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - res.on('data', () => { }) - }) + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + res.on('data', () => { }) + }) - req.on('error', () => {}) + req.on('error', () => {}) - req.end() - }) + req.end() }) }).timeout(10000) @@ -846,27 +692,25 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - agent - .use(traces => { - expect(traces[0][0]).to.have.property('error', 1) - }) - .then(done) - .catch(done) + agent + .use(traces => { + expect(traces[0][0]).to.have.property('error', 1) + }) + .then(done) + .catch(done) - const options = { - agent: new http.Agent({ keepAlive: true, timeout: 5000 }) // custom agent with same default timeout - } + const options = { + agent: new http.Agent({ keepAlive: true, timeout: 5000 }) // custom agent with same default timeout + } - appListener = server(app, port, async () => { - const req = http.request(`${protocol}://localhost:${port}/user`, options, res => { - res.on('data', () => { }) - }) + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`, options, res => { + res.on('data', () => { }) + }) - req.on('error', () => {}) + req.on('error', () => {}) - req.end() - }) + req.end() }) }).timeout(10000) @@ -880,32 +724,35 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - agent - .use(traces => { - expect(traces[0][0]).to.have.property('error', 1) - }) - .then(done) - .catch(done) + agent + .use(traces => { + expect(traces[0][0]).to.have.property('error', 1) + }) + .then(done) + .catch(done) - appListener = server(app, port, async () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - res.on('data', () => { }) - }) + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + res.on('data', () => { }) + }) - req.on('error', () => {}) - req.setTimeout(5000) // match default timeout + req.on('error', () => {}) + req.setTimeout(5000) // match default timeout - req.end() - }) + req.end() }) }).timeout(10000) } it('should only record a request once', done => { // Make sure both plugins are loaded, which could cause double-counting. - require('http') - require('https') + if (pluginToBeLoaded.includes('node:')) { + require('node:http') + require('node:https') + } else { + require('http') + require('https') + } const app = express() @@ -913,33 +760,31 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - agent - .use(traces => { - const spans = traces[0] - expect(spans.length).to.equal(3) + agent + .use(traces => { + const spans = traces[0] + expect(spans.length).to.equal(3) + }) + .then(done) + .catch(done) + + appListener = server(app, port => { + // Activate a new parent span so we capture any double counting that may happen, otherwise double-counts + // would be siblings and our test would only capture 1 as a false positive. + const span = tracer.startSpan('http-test') + tracer.scope().activate(span, () => { + // Test `http(s).request + const req = http.request(`${protocol}://localhost:${port}/user?test=request`, res => { + res.on('data', () => {}) }) - .then(done) - .catch(done) - - appListener = server(app, port, () => { - // Activate a new parent span so we capture any double counting that may happen, otherwise double-counts - // would be siblings and our test would only capture 1 as a false positive. - const span = tracer.startSpan('http-test') - tracer.scope().activate(span, () => { - // Test `http(s).request - const req = http.request(`${protocol}://localhost:${port}/user?test=request`, res => { - res.on('data', () => {}) - }) - req.end() - - // Test `http(s).get` - http.get(`${protocol}://localhost:${port}/user?test=get`, res => { - res.on('data', () => {}) - }) + req.end() - span.finish() + // Test `http(s).get` + http.get(`${protocol}://localhost:${port}/user?test=get`, res => { + res.on('data', () => {}) }) + + span.finish() }) }) }) @@ -951,19 +796,17 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - agent - .use(traces => { - expect(traces[0][0]).to.have.property('service', SERVICE_NAME) - }) - .then(done) - .catch(done) + agent + .use(traces => { + expect(traces[0][0]).to.have.property('service', SERVICE_NAME) + }) + .then(done) + .catch(done) - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/abort`) + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/abort`) - req.abort() - }) + req.abort() }) }) @@ -974,7 +817,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.status_code', '200') @@ -983,17 +826,15 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const req = http.request({ - hostname: 'localhost', - port, - path: '/user?foo=bar' - }, res => { - res.on('data', () => {}) - }) - - req.end() + const req = http.request({ + hostname: 'localhost', + port, + path: '/user?foo=bar' + }, res => { + res.on('data', () => {}) }) + + req.end() }) }) @@ -1004,7 +845,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][1]).to.have.property('error', 0) @@ -1014,17 +855,15 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - tracer.trace('test.request', (span, finish) => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - res.on('data', () => {}) - res.on('end', () => { - setTimeout(finish) - }) + tracer.trace('test.request', (span, finish) => { + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + res.on('data', () => {}) + res.on('end', () => { + setTimeout(finish) }) - - req.end() }) + + req.end() }) }) }) @@ -1037,26 +876,24 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - const timer = setTimeout(done, 100) + const timer = setTimeout(done, 100) - agent - .use(() => { - done(new Error('Noop request was traced.')) - clearTimeout(timer) - }) + agent + .use(() => { + done(new Error('Noop request was traced.')) + clearTimeout(timer) + }) - appListener = server(app, port, () => { - const store = storage.getStore() + appListener = server(app, port => { + const store = storage.getStore() - storage.enterWith({ noop: true }) - const req = http.request(tracer._tracer._url.href) + storage.enterWith({ noop: true }) + const req = http.request(tracer._tracer._url.href) - req.on('error', () => {}) - req.end() + req.on('error', () => {}) + req.end() - storage.enterWith(store) - }) + storage.enterWith(store) }) }) } @@ -1072,7 +909,7 @@ describe('Plugin', () => { ch = require('dc-polyfill').channel('apm:http:client:request:start') sub = () => {} tracer = require('../../dd-trace') - http = require(protocol) + http = require(pluginToBeLoaded) }) }) @@ -1085,23 +922,21 @@ describe('Plugin', () => { res.end() } - getPort().then(port => { - appListener = server(app, port, () => { - ch.subscribe(sub) + appListener = server(app, port => { + ch.subscribe(sub) - tracer.use('http', false) + tracer.use('http', false) - const req = http.request(`${protocol}://localhost:${port}`, res => { - res.on('error', done) - res.on('data', () => {}) - res.on('end', () => done()) - }) - req.on('error', done) + const req = http.request(`${protocol}://localhost:${port}`, res => { + res.on('error', done) + res.on('data', () => {}) + res.on('end', () => done()) + }) + req.on('error', done) - tracer.use('http', true) + tracer.use('http', true) - req.end() - }) + req.end() }) }) }) @@ -1119,7 +954,7 @@ describe('Plugin', () => { return agent.load('http', config) .then(() => { - http = require(protocol) + http = require(pluginToBeLoaded) express = require('express') }) }) @@ -1131,67 +966,19 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - agent - .use(traces => { - expect(traces[0][0]).to.have.property('service', 'custom') - }) - .then(done) - .catch(done) - - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - res.on('data', () => {}) - }) - - req.end() + agent + .use(traces => { + expect(traces[0][0]).to.have.property('service', 'custom') }) - }) - }) - }) + .then(done) + .catch(done) - describe('with config enablePropagationWithAmazonHeaders enabled', () => { - let config - - beforeEach(() => { - config = { - enablePropagationWithAmazonHeaders: true - } - - return agent.load('http', config) - .then(() => { - http = require(protocol) - express = require('express') + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + res.on('data', () => {}) }) - }) - - it('should inject tracing header into AWS signed request', done => { - const app = express() - - app.get('/', (req, res) => { - try { - expect(req.get('x-datadog-trace-id')).to.be.a('string') - expect(req.get('x-datadog-parent-id')).to.be.a('string') - - res.status(200).send() - - done() - } catch (e) { - done(e) - } - }) - - getPort().then(port => { - appListener = server(app, port, () => { - const req = http.request({ - port, - headers: { - Authorization: 'AWS4-HMAC-SHA256 ...' - } - }) - req.end() - }) + req.end() }) }) }) @@ -1209,7 +996,7 @@ describe('Plugin', () => { return agent.load('http', config) .then(() => { - http = require(protocol) + http = require(pluginToBeLoaded) express = require('express') }) }) @@ -1221,21 +1008,19 @@ describe('Plugin', () => { res.status(500).send() }) - getPort().then(port => { - agent - .use(traces => { - expect(traces[0][0]).to.have.property('error', 1) - }) - .then(done) - .catch(done) - - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - res.on('data', () => { }) - }) + agent + .use(traces => { + expect(traces[0][0]).to.have.property('error', 1) + }) + .then(done) + .catch(done) - req.end() + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + res.on('data', () => { }) }) + + req.end() }) }) }) @@ -1254,7 +1039,7 @@ describe('Plugin', () => { return agent.load('http', config) .then(() => { - http = require(protocol) + http = require(pluginToBeLoaded) express = require('express') }) }) @@ -1265,14 +1050,12 @@ describe('Plugin', () => { app.get('/user', (req, res) => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { serverPort = port - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - res.on('data', () => {}) - }) - req.end() + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + res.on('data', () => {}) }) + req.end() }) }, { @@ -1294,7 +1077,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', `localhost:${port}`) @@ -1302,13 +1085,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - res.on('data', () => {}) - }) - - req.end() + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + res.on('data', () => {}) }) + + req.end() }) }) }) @@ -1326,7 +1107,7 @@ describe('Plugin', () => { return agent.load('http', config) .then(() => { - http = require(protocol) + http = require(pluginToBeLoaded) express = require('express') }) }) @@ -1340,28 +1121,26 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { const meta = traces[0][0].meta expect(meta).to.have.property(`${HTTP_REQUEST_HEADERS}.host`, `localhost:${port}`) - expect(meta).to.have.property(`http.baz`, 'baz') + expect(meta).to.have.property('http.baz', 'baz') expect(meta).to.have.property(`${HTTP_RESPONSE_HEADERS}.x-foo`, 'foo') - expect(meta).to.have.property(`http.bar`, 'bar') + expect(meta).to.have.property('http.bar', 'bar') }) .then(done) .catch(done) - appListener = server(app, port, () => { - const url = `${protocol}://localhost:${port}/user` - const headers = { 'x-baz': 'baz' } - const req = http.request(url, { headers }, res => { - res.on('data', () => {}) - }) - - req.end() + const url = `${protocol}://localhost:${port}/user` + const headers = { 'x-baz': 'baz' } + const req = http.request(url, { headers }, res => { + res.on('data', () => {}) }) + + req.end() }) }) @@ -1372,24 +1151,22 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - agent - .use(traces => { - const meta = traces[0][0].meta - - expect(meta).to.have.property(`${HTTP_REQUEST_HEADERS}.x-foo`, `bar`) - }) - .then(done) - .catch(done) + agent + .use(traces => { + const meta = traces[0][0].meta - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - res.on('data', () => {}) - }) + expect(meta).to.have.property(`${HTTP_REQUEST_HEADERS}.x-foo`, 'bar') + }) + .then(done) + .catch(done) - req.setHeader('x-foo', 'bar') - req.end() + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + res.on('data', () => {}) }) + + req.setHeader('x-foo', 'bar') + req.end() }) }) @@ -1400,24 +1177,22 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - agent - .use(traces => { - const meta = traces[0][0].meta - - expect(meta).to.have.property(`${HTTP_REQUEST_HEADERS}.x-foo`, `bar1,bar2`) - }) - .then(done) - .catch(done) + agent + .use(traces => { + const meta = traces[0][0].meta - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - res.on('data', () => { }) - }) + expect(meta).to.have.property(`${HTTP_REQUEST_HEADERS}.x-foo`, 'bar1,bar2') + }) + .then(done) + .catch(done) - req.setHeader('x-foo', ['bar1', 'bar2']) - req.end() + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + res.on('data', () => { }) }) + + req.setHeader('x-foo', ['bar1', 'bar2']) + req.end() }) }) }) @@ -1439,7 +1214,7 @@ describe('Plugin', () => { return agent.load('http', config) .then(() => { - http = require(protocol) + http = require(pluginToBeLoaded) express = require('express') }) }) @@ -1451,23 +1226,21 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - agent - .use(traces => { - expect(traces[0][0]).to.have.property('resource', 'GET /user') - }) - .then(done) - .catch(done) + agent + .use(traces => { + expect(traces[0][0]).to.have.property('resource', 'GET /user') + }) + .then(done) + .catch(done) - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - res.on('data', () => {}) - }) + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + res.on('data', () => {}) + }) - req._route = '/user' + req._route = '/user' - req.end() - }) + req.end() }) }) }) @@ -1485,7 +1258,7 @@ describe('Plugin', () => { return agent.load('http', config) .then(() => { - http = require(protocol) + http = require(pluginToBeLoaded) express = require('express') }) }) @@ -1506,15 +1279,13 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = server(app, port, () => { - const req = http.request({ - port, - path: '/users' - }) - - req.end() + appListener = server(app, port => { + const req = http.request({ + port, + path: '/users' }) + + req.end() }) }) }) @@ -1532,7 +1303,7 @@ describe('Plugin', () => { return agent.load('http', config) .then(() => { - http = require(protocol) + http = require(pluginToBeLoaded) express = require('express') }) }) @@ -1544,23 +1315,21 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - const timer = setTimeout(done, 100) - - agent - .use(() => { - clearTimeout(timer) - done(new Error('Blocklisted requests should not be recorded.')) - }) - .catch(done) + const timer = setTimeout(done, 100) - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - res.on('data', () => {}) - }) + agent + .use(() => { + clearTimeout(timer) + done(new Error('Blocklisted requests should not be recorded.')) + }) + .catch(done) - req.end() + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + res.on('data', () => {}) }) + + req.end() }) }) }) diff --git a/packages/datadog-plugin-http/test/integration-test/client.spec.js b/packages/datadog-plugin-http/test/integration-test/client.spec.js index 4b5637ae98c..2d11c13168b 100644 --- a/packages/datadog-plugin-http/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-http/test/integration-test/client.spec.js @@ -16,7 +16,7 @@ describe('esm', () => { before(async function () { this.timeout(20000) sandbox = await createSandbox([], false, [ - `./packages/datadog-plugin-http/test/integration-test/*`]) + './packages/datadog-plugin-http/test/integration-test/*']) }) after(async () => { diff --git a/packages/datadog-plugin-http/test/server.spec.js b/packages/datadog-plugin-http/test/server.spec.js index 9a0135ea967..f3b5f3964ef 100644 --- a/packages/datadog-plugin-http/test/server.spec.js +++ b/packages/datadog-plugin-http/test/server.spec.js @@ -1,7 +1,5 @@ 'use strict' -const { AbortController } = require('node-abort-controller') // AbortController is not available in node <15 -const getPort = require('get-port') const agent = require('../../dd-trace/test/plugins/agent') const axios = require('axios') const { incomingHttpRequestStart } = require('../../dd-trace/src/appsec/channels') @@ -15,248 +13,247 @@ describe('Plugin', () => { let port let app - describe('http/server', () => { - beforeEach(() => { - tracer = require('../../dd-trace') - listener = (req, res) => { - app && app(req, res) - res.writeHead(200) - res.end() - } - }) - - beforeEach(() => { - return getPort().then(newPort => { - port = newPort - }) - }) - - afterEach(() => { - appListener && appListener.close() - app = null - return agent.close({ ritmReset: false }) - }) - - describe('canceled request', () => { + ['http', 'node:http'].forEach(pluginToBeLoaded => { + describe(`${pluginToBeLoaded}/server`, () => { beforeEach(() => { + tracer = require('../../dd-trace') listener = (req, res) => { - setTimeout(() => { - app && app(req, res) - res.writeHead(200) - res.end() - }, 500) + app && app(req, res) + res.writeHead(200) + res.end() } }) - beforeEach(() => { - return agent.load('http') - .then(() => { - http = require('http') - }) + afterEach(() => { + appListener && appListener.close() + app = null + return agent.close({ ritmReset: false }) }) - beforeEach(done => { - const server = new http.Server(listener) - appListener = server - .listen(port, 'localhost', () => done()) - }) + describe('canceled request', () => { + beforeEach(() => { + listener = (req, res) => { + setTimeout(() => { + app && app(req, res) + res.writeHead(200) + res.end() + }, 500) + } + }) - it('should send traces to agent', (done) => { - app = sinon.stub() - agent - .use(traces => { - expect(app).not.to.have.been.called // request should be cancelled before call to app - expect(traces[0][0]).to.have.property('name', 'web.request') - expect(traces[0][0]).to.have.property('service', 'test') - expect(traces[0][0]).to.have.property('type', 'web') - expect(traces[0][0]).to.have.property('resource', 'GET') - expect(traces[0][0].meta).to.have.property('span.kind', 'server') - expect(traces[0][0].meta).to.have.property('http.url', `http://localhost:${port}/user`) - expect(traces[0][0].meta).to.have.property('http.method', 'GET') - expect(traces[0][0].meta).to.have.property('http.status_code', '200') - expect(traces[0][0].meta).to.have.property('component', 'http') - }) - .then(done) - .catch(done) - const source = axios.CancelToken.source() - axios.get(`http://localhost:${port}/user`, { cancelToken: source.token }) - .then(() => {}) - setTimeout(() => { source.cancel() }, 100) - }) - }) + beforeEach(() => { + return agent.load('http') + .then(() => { + http = require(pluginToBeLoaded) + }) + }) - describe('without configuration', () => { - beforeEach(() => { - return agent.load('http') - .then(() => { - http = require('http') - }) - }) + beforeEach(done => { + const server = new http.Server(listener) + appListener = server + .listen(0, 'localhost', () => { + port = appListener.address().port + done() + }) + }) - beforeEach(done => { - const server = new http.Server(listener) - appListener = server - .listen(port, 'localhost', () => done()) + it('should send traces to agent', (done) => { + app = sinon.stub() + agent + .use(traces => { + expect(app).not.to.have.been.called // request should be cancelled before call to app + expect(traces[0][0]).to.have.property('name', 'web.request') + expect(traces[0][0]).to.have.property('service', 'test') + expect(traces[0][0]).to.have.property('type', 'web') + expect(traces[0][0]).to.have.property('resource', 'GET') + expect(traces[0][0].meta).to.have.property('span.kind', 'server') + expect(traces[0][0].meta).to.have.property('http.url', `http://localhost:${port}/user`) + expect(traces[0][0].meta).to.have.property('http.method', 'GET') + expect(traces[0][0].meta).to.have.property('http.status_code', '200') + expect(traces[0][0].meta).to.have.property('component', 'http') + }) + .then(done) + .catch(done) + const source = axios.CancelToken.source() + axios.get(`http://localhost:${port}/user`, { cancelToken: source.token }) + .then(() => {}) + setTimeout(() => { source.cancel() }, 100) + }) }) - withNamingSchema( - done => { - axios.get(`http://localhost:${port}/user`).catch(done) - }, - rawExpectedSchema.server - ) - - it('should do automatic instrumentation', done => { - agent - .use(traces => { - expect(traces[0][0]).to.have.property('name', 'web.request') - expect(traces[0][0]).to.have.property('service', 'test') - expect(traces[0][0]).to.have.property('type', 'web') - expect(traces[0][0]).to.have.property('resource', 'GET') - expect(traces[0][0].meta).to.have.property('span.kind', 'server') - expect(traces[0][0].meta).to.have.property('http.url', `http://localhost:${port}/user`) - expect(traces[0][0].meta).to.have.property('http.method', 'GET') - expect(traces[0][0].meta).to.have.property('http.status_code', '200') - expect(traces[0][0].meta).to.have.property('component', 'http') - }) - .then(done) - .catch(done) - - axios.get(`http://localhost:${port}/user`).catch(done) - }) + describe('without configuration', () => { + beforeEach(() => { + return agent.load('http') + .then(() => { + http = require(pluginToBeLoaded) + }) + }) - it('should run the request listener in the request scope', done => { - const spy = sinon.spy(() => { - expect(tracer.scope().active()).to.not.be.null + beforeEach(done => { + const server = new http.Server(listener) + appListener = server + .listen(port, 'localhost', () => done()) }) - incomingHttpRequestStart.subscribe(spy) + withNamingSchema( + done => { + axios.get(`http://localhost:${port}/user`).catch(done) + }, + rawExpectedSchema.server + ) + + it('should do automatic instrumentation', done => { + agent + .use(traces => { + expect(traces[0][0]).to.have.property('name', 'web.request') + expect(traces[0][0]).to.have.property('service', 'test') + expect(traces[0][0]).to.have.property('type', 'web') + expect(traces[0][0]).to.have.property('resource', 'GET') + expect(traces[0][0].meta).to.have.property('span.kind', 'server') + expect(traces[0][0].meta).to.have.property('http.url', `http://localhost:${port}/user`) + expect(traces[0][0].meta).to.have.property('http.method', 'GET') + expect(traces[0][0].meta).to.have.property('http.status_code', '200') + expect(traces[0][0].meta).to.have.property('component', 'http') + }) + .then(done) + .catch(done) - app = (req, res) => { - expect(tracer.scope().active()).to.not.be.null + axios.get(`http://localhost:${port}/user`).catch(done) + }) - const abortController = new AbortController() - expect(spy).to.have.been.calledOnceWithExactly({ req, res, abortController }, incomingHttpRequestStart.name) + it('should run the request listener in the request scope', done => { + const spy = sinon.spy(() => { + expect(tracer.scope().active()).to.not.be.null + }) - done() - } + incomingHttpRequestStart.subscribe(spy) - axios.get(`http://localhost:${port}/user`).catch(done) - }) + app = (req, res) => { + expect(tracer.scope().active()).to.not.be.null + + const abortController = new AbortController() + expect(spy).to.have.been.calledOnceWithExactly({ req, res, abortController }, incomingHttpRequestStart.name) - it(`should run the request's close event in the correct context`, done => { - app = (req, res) => { - req.on('close', () => { - expect(tracer.scope().active()).to.equal(null) done() - }) - } + } - axios.get(`http://localhost:${port}/user`).catch(done) - }) + axios.get(`http://localhost:${port}/user`).catch(done) + }) - it(`should run the response's close event in the correct context`, done => { - app = (req, res) => { - const span = tracer.scope().active() + it('should run the request\'s close event in the correct context', done => { + app = (req, res) => { + req.on('close', () => { + expect(tracer.scope().active()).to.equal(null) + done() + }) + } - res.on('close', () => { - expect(tracer.scope().active()).to.equal(span) - done() - }) - } + axios.get(`http://localhost:${port}/user`).catch(done) + }) - axios.get(`http://localhost:${port}/user`).catch(done) - }) + it('should run the response\'s close event in the correct context', done => { + app = (req, res) => { + const span = tracer.scope().active() - it(`should run the finish event in the correct context`, done => { - app = (req, res) => { - const span = tracer.scope().active() + res.on('close', () => { + expect(tracer.scope().active()).to.equal(span) + done() + }) + } - res.on('finish', () => { - expect(tracer.scope().active()).to.equal(span) - done() - }) - } + axios.get(`http://localhost:${port}/user`).catch(done) + }) - axios.get(`http://localhost:${port}/user`).catch(done) - }) + it('should run the finish event in the correct context', done => { + app = (req, res) => { + const span = tracer.scope().active() - it('should not instrument manually instantiated server responses', () => { - const { IncomingMessage, ServerResponse } = http + res.on('finish', () => { + expect(tracer.scope().active()).to.equal(span) + done() + }) + } - const req = new IncomingMessage() - const res = new ServerResponse(req) + axios.get(`http://localhost:${port}/user`).catch(done) + }) - expect(() => res.emit('finish')).to.not.throw() - }) + it('should not instrument manually instantiated server responses', () => { + const { IncomingMessage, ServerResponse } = http - it('should not cause `end` to be called multiple times', done => { - app = (req, res) => { - res.end = sinon.spy(res.end) + const req = new IncomingMessage() + const res = new ServerResponse(req) - res.on('finish', () => { - expect(res.end).to.have.been.calledOnce - done() - }) - } + expect(() => res.emit('finish')).to.not.throw() + }) - axios.get(`http://localhost:${port}/user`).catch(done) - }) - }) + it('should not cause `end` to be called multiple times', done => { + app = (req, res) => { + res.end = sinon.spy(res.end) - describe('with a `server` configuration', () => { - beforeEach(() => { - return agent.load('http', { client: false, server: {} }) - .then(() => { - http = require('http') - }) - }) + res.on('finish', () => { + expect(res.end).to.have.been.calledOnce + done() + }) + } - beforeEach(done => { - const server = new http.Server(listener) - appListener = server - .listen(port, 'localhost', () => done()) + axios.get(`http://localhost:${port}/user`).catch(done) + }) }) - // see https://github.com/DataDog/dd-trace-js/issues/2453 - it('should not have disabled tracing', (done) => { - agent.use(() => {}) - .then(done) - .catch(done) + describe('with a `server` configuration', () => { + beforeEach(() => { + return agent.load('http', { client: false, server: {} }) + .then(() => { + http = require(pluginToBeLoaded) + }) + }) - axios.get(`http://localhost:${port}/user`).catch(done) - }) - }) + beforeEach(done => { + const server = new http.Server(listener) + appListener = server + .listen(port, 'localhost', () => done()) + }) - describe('with a blocklist configuration', () => { - beforeEach(() => { - return agent.load('http', { client: false, blocklist: '/health' }) - .then(() => { - http = require('http') - }) - }) + // see https://github.com/DataDog/dd-trace-js/issues/2453 + it('should not have disabled tracing', (done) => { + agent.use(() => {}) + .then(done) + .catch(done) - beforeEach(done => { - const server = new http.Server(listener) - appListener = server - .listen(port, 'localhost', () => done()) + axios.get(`http://localhost:${port}/user`).catch(done) + }) }) - it('should drop traces for blocklist route', done => { - const spy = sinon.spy(() => {}) + describe('with a blocklist configuration', () => { + beforeEach(() => { + return agent.load('http', { client: false, blocklist: '/health' }) + .then(() => { + http = require(pluginToBeLoaded) + }) + }) - agent - .use((traces) => { - spy() - }) - .catch(done) + beforeEach(done => { + const server = new http.Server(listener) + appListener = server + .listen(port, 'localhost', () => done()) + }) - setTimeout(() => { - expect(spy).to.not.have.been.called - done() - }, 100) + it('should drop traces for blocklist route', done => { + const spy = sinon.spy(() => {}) - axios.get(`http://localhost:${port}/health`).catch(done) + agent + .use((traces) => { + spy() + }) + .catch(done) + + setTimeout(() => { + expect(spy).to.not.have.been.called + done() + }, 100) + + axios.get(`http://localhost:${port}/health`).catch(done) + }) }) }) }) diff --git a/packages/datadog-plugin-http2/src/client.js b/packages/datadog-plugin-http2/src/client.js index 4a60ee0b4db..3f8d996fcd3 100644 --- a/packages/datadog-plugin-http2/src/client.js +++ b/packages/datadog-plugin-http2/src/client.js @@ -25,7 +25,7 @@ const HTTP2_METHOD_GET = 'GET' class Http2ClientPlugin extends ClientPlugin { static get id () { return 'http2' } - static get prefix () { return `apm:http2:client:request` } + static get prefix () { return 'apm:http2:client:request' } bindStart (message) { const { authority, options, headers = {} } = message @@ -62,9 +62,7 @@ class Http2ClientPlugin extends ClientPlugin { addHeaderTags(span, headers, HTTP_REQUEST_HEADERS, this.config) - if (!hasAmazonSignature(headers, path)) { - this.tracer.inject(span, HTTP_HEADERS, headers) - } + this.tracer.inject(span, HTTP_HEADERS, headers) message.parentStore = store message.currentStore = { ...store, span } @@ -122,7 +120,8 @@ function extractSessionDetails (authority, options) { const protocol = authority.protocol || options.protocol || 'https:' let port = '' + (authority.port !== '' - ? authority.port : (authority.protocol === 'http:' ? 80 : 443)) + ? authority.port + : (authority.protocol === 'http:' ? 80 : 443)) let host = authority.hostname || authority.host || 'localhost' if (protocol === 'https:' && options) { @@ -133,29 +132,6 @@ function extractSessionDetails (authority, options) { return { protocol, port, host } } -function hasAmazonSignature (headers, path) { - if (headers) { - headers = Object.keys(headers) - .reduce((prev, next) => Object.assign(prev, { - [next.toLowerCase()]: headers[next] - }), {}) - - if (headers['x-amz-signature']) { - return true - } - - if ([].concat(headers['authorization']).some(startsWith('AWS4-HMAC-SHA256'))) { - return true - } - } - - return path && path.toLowerCase().indexOf('x-amz-signature=') !== -1 -} - -function startsWith (searchString) { - return value => String(value).startsWith(searchString) -} - function getStatusValidator (config) { if (typeof config.validateStatus === 'function') { return config.validateStatus diff --git a/packages/datadog-plugin-http2/test/client.spec.js b/packages/datadog-plugin-http2/test/client.spec.js index 89ec4cb1ab3..cfdedcde489 100644 --- a/packages/datadog-plugin-http2/test/client.spec.js +++ b/packages/datadog-plugin-http2/test/client.spec.js @@ -1,6 +1,5 @@ 'use strict' -const getPort = require('get-port') const agent = require('../../dd-trace/test/plugins/agent') const fs = require('fs') const path = require('path') @@ -20,18 +19,20 @@ describe('Plugin', () => { let appListener let tracer - ['http', 'https'].forEach(protocol => { - describe(`http2/client, protocol ${protocol}`, () => { - function server (app, port, listener) { + ['http', 'https', 'node:http', 'node:https'].forEach(pluginToBeLoaded => { + const protocol = pluginToBeLoaded.split(':')[1] || pluginToBeLoaded + const loadPlugin = pluginToBeLoaded.includes('node:') ? 'node:http2' : 'http2' + describe(`http2/client, protocol ${pluginToBeLoaded}`, () => { + function server (app, listener) { let server - if (protocol === 'https') { + if (pluginToBeLoaded === 'https' || pluginToBeLoaded === 'node:https') { process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0' - server = require('http2').createSecureServer({ key, cert }) + server = require(loadPlugin).createSecureServer({ key, cert }) } else { - server = require('http2').createServer() + server = require(loadPlugin).createServer() } server.on('stream', app) - server.listen(port, 'localhost', listener) + server.listen(0, 'localhost', () => listener(server.address().port)) return server } @@ -51,28 +52,27 @@ describe('Plugin', () => { beforeEach(() => { return agent.load('http2', { server: false }) .then(() => { - http2 = require('http2') + http2 = require(loadPlugin) }) }) const spanProducerFn = (done) => { - getPort().then(port => { - const app = (stream, headers) => { - stream.respond({ - ':status': 200 - }) - stream.end() - } - appListener = server(app, port, () => { - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) + const app = (stream, headers) => { + stream.respond({ + ':status': 200 + }) + stream.end() + } - const req = client.request({ ':path': '/user', ':method': 'GET' }) - req.on('error', done) + appListener = server(app, port => { + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) - req.end() - }) + const req = client.request({ ':path': '/user', ':method': 'GET' }) + req.on('error', done) + + req.end() }) } @@ -97,7 +97,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', SERVICE_NAME) @@ -114,16 +114,14 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) - const req = client.request({ ':path': '/user', ':method': 'GET' }) - req.on('error', done) + const req = client.request({ ':path': '/user', ':method': 'GET' }) + req.on('error', done) - req.end() - }) + req.end() }) }) @@ -135,7 +133,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('span.kind', 'client') @@ -144,16 +142,14 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) - const req = client.request({}) - .on('error', done) + const req = client.request({}) + .on('error', done) - req.end() - }) + req.end() }) }) @@ -165,7 +161,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.url', `${protocol}://localhost:${port}/user`) @@ -179,16 +175,14 @@ describe('Plugin', () => { port } - appListener = server(app, port, () => { - const client = http2 - .connect(uri) - .on('error', done) + const client = http2 + .connect(uri) + .on('error', done) - const req = client.request({ ':path': '/user' }) - req.on('error', done) + const req = client.request({ ':path': '/user' }) + req.on('error', done) - req.end() - }) + req.end() }) }) @@ -200,7 +194,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.url', `${protocol}://localhost:${port}/user`) @@ -208,16 +202,14 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) - const req = client.request({ ':path': '/user?foo=bar' }) - req.on('error', done) + const req = client.request({ ':path': '/user?foo=bar' }) + req.on('error', done) - req.end() - }) + req.end() }) }) @@ -230,7 +222,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.url', `${protocol}://localhost:${port}/user`) @@ -250,21 +242,19 @@ describe('Plugin', () => { port: 1337 } - appListener = server(app, port, () => { - let client - if (protocol === 'https') { - client = http2.connect(incorrectConfig, correctConfig) - } else { - client = http2.connect(correctConfig, incorrectConfig) - } + let client + if (protocol === 'https') { + client = http2.connect(incorrectConfig, correctConfig) + } else { + client = http2.connect(correctConfig, incorrectConfig) + } - client.on('error', done) + client.on('error', done) - const req = client.request({ ':path': '/user' }) - req.on('error', done) + const req = client.request({ ':path': '/user' }) + req.on('error', done) - req.end() - }) + req.end() }) }) @@ -277,7 +267,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.url', `${protocol}://localhost:${port}/user`) @@ -297,21 +287,19 @@ describe('Plugin', () => { port: 1337 } - appListener = server(app, port, () => { - let client - if (protocol === 'https') { - client = http2.connect(`${protocol}://remotehost:1337`, correctConfig) - } else { - client = http2.connect(`${protocol}://localhost:${port}`, incorrectConfig) - } + let client + if (protocol === 'https') { + client = http2.connect(`${protocol}://remotehost:1337`, correctConfig) + } else { + client = http2.connect(`${protocol}://localhost:${port}`, incorrectConfig) + } - client.on('error', done) + client.on('error', done) - const req = client.request({ ':path': '/user' }) - req.on('error', done) + const req = client.request({ ':path': '/user' }) + req.on('error', done) - req.end() - }) + req.end() }) }) @@ -323,7 +311,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.url', `${protocol}://localhost:${port}/`) @@ -336,16 +324,14 @@ describe('Plugin', () => { port } - appListener = server(app, port, () => { - const client = http2 - .connect(uri) - .on('error', done) + const client = http2 + .connect(uri) + .on('error', done) - const req = client.request({}) - req.on('error', done) + const req = client.request({}) + req.on('error', done) - req.end() - }) + req.end() }) }) @@ -360,7 +346,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.status_code', '200') @@ -368,149 +354,14 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) - - const req = client.request({}) - req.on('error', done) - - req.end() - }) - }) - }) - - it('should skip injecting if the Authorization header contains an AWS signature', done => { - const app = (stream, headers) => { - try { - expect(headers['x-datadog-trace-id']).to.be.undefined - expect(headers['x-datadog-parent-id']).to.be.undefined - - stream.respond({ - ':status': 200 - }) - stream.end() - - done() - } catch (e) { - done(e) - } - } - - getPort().then(port => { - appListener = server(app, port, () => { - const headers = { - Authorization: 'AWS4-HMAC-SHA256 ...' - } - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) - - const req = client.request(headers) - req.on('error', done) - - req.end() - }) - }) - }) - - it('should skip injecting if one of the Authorization headers contains an AWS signature', done => { - const app = (stream, headers) => { - try { - expect(headers['x-datadog-trace-id']).to.be.undefined - expect(headers['x-datadog-parent-id']).to.be.undefined - - stream.respond({ - ':status': 200 - }) - stream.end() - - done() - } catch (e) { - done(e) - } - } + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) - getPort().then(port => { - appListener = server(app, port, () => { - const headers = { - Authorization: ['AWS4-HMAC-SHA256 ...'] - } - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) - - const req = client.request(headers) - req.on('error', done) - - req.end() - }) - }) - }) - - it('should skip injecting if the X-Amz-Signature header is set', done => { - const app = (stream, headers) => { - try { - expect(headers['x-datadog-trace-id']).to.be.undefined - expect(headers['x-datadog-parent-id']).to.be.undefined + const req = client.request({}) + req.on('error', done) - stream.respond({ - ':status': 200 - }) - stream.end() - - done() - } catch (e) { - done(e) - } - } - - getPort().then(port => { - appListener = server(app, port, () => { - const headers = { - 'X-Amz-Signature': 'abc123' - } - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) - - const req = client.request(headers) - req.on('error', done) - - req.end() - }) - }) - }) - - it('should skip injecting if the X-Amz-Signature query param is set', done => { - const app = (stream, headers) => { - try { - expect(headers['x-datadog-trace-id']).to.be.undefined - expect(headers['x-datadog-parent-id']).to.be.undefined - - stream.respond({ - ':status': 200 - }) - stream.end() - - done() - } catch (e) { - done(e) - } - } - - getPort().then(port => { - appListener = server(app, port, () => { - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) - - const req = client.request({ ':path': '/?X-Amz-Signature=abc123' }) - req.on('error', done) - - req.end() - }) + req.end() }) }) @@ -522,52 +373,49 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { - appListener = server(app, port, () => { - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) + appListener = server(app, port => { + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) - const span = {} + const span = {} - tracer.scope().activate(span, () => { - const req = client.request({ ':path': '/user' }) - req.on('response', (headers, flags) => { - expect(tracer.scope().active()).to.equal(span) - done() - }) + tracer.scope().activate(span, () => { + const req = client.request({ ':path': '/user' }) + req.on('response', (headers, flags) => { + expect(tracer.scope().active()).to.equal(span) + done() + }) - req.on('error', done) + req.on('error', done) - req.end() - }) + req.end() }) }) }) it('should handle connection errors', done => { - getPort().then(port => { - let error + let error - agent - .use(traces => { - expect(traces[0][0].meta).to.have.property(ERROR_TYPE, error.name) - expect(traces[0][0].meta).to.have.property(ERROR_MESSAGE, error.message) - expect(traces[0][0].meta).to.have.property(ERROR_STACK, error.stack) - expect(traces[0][0].meta).to.have.property('component', 'http2') - expect(traces[0][0].metrics).to.have.property('network.destination.port', port) - }) - .then(done) - .catch(done) + agent + .use(traces => { + expect(traces[0][0].meta).to.have.property(ERROR_TYPE, error.name) + expect(traces[0][0].meta).to.have.property(ERROR_MESSAGE, error.message) + expect(traces[0][0].meta).to.have.property(ERROR_STACK, error.stack) + expect(traces[0][0].meta).to.have.property('component', 'http2') + expect(traces[0][0].metrics).to.have.property('network.destination.port', 7357) + }) + .then(done) + .catch(done) - const client = http2.connect(`${protocol}://localhost:${port}`) - .on('error', (err) => {}) + const client = http2.connect(`${protocol}://localhost:7357`) + // eslint-disable-next-line n/handle-callback-err + .on('error', (err) => {}) - const req = client.request({ ':path': '/user' }) - .on('error', (err) => { error = err }) + const req = client.request({ ':path': '/user' }) + .on('error', (err) => { error = err }) - req.end() - }) + req.end() }) it('should not record HTTP 5XX responses as errors by default', done => { @@ -578,7 +426,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('error', 0) @@ -586,16 +434,14 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) - const req = client.request({ ':path': '/' }) - req.on('error', done) + const req = client.request({ ':path': '/' }) + req.on('error', done) - req.end() - }) + req.end() }) }) @@ -607,7 +453,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('error', 1) @@ -615,21 +461,19 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) - const req = client.request({ ':path': '/' }) - req.on('error', done) + const req = client.request({ ':path': '/' }) + req.on('error', done) - req.end() - }) + req.end() }) }) it('should only record a request once', done => { - require('http2') + require(loadPlugin) const app = (stream, headers) => { stream.respond({ ':status': 200 @@ -637,7 +481,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { const spans = traces[0] @@ -646,24 +490,22 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - // Activate a new parent span so we capture any double counting that may happen, otherwise double-counts - // would be siblings and our test would only capture 1 as a false positive. - const span = tracer.startSpan('http-test') - tracer.scope().activate(span, () => { - const client = http2.connect(`${protocol}://localhost:${port}`) - .on('error', done) + // Activate a new parent span so we capture any double counting that may happen, otherwise double-counts + // would be siblings and our test would only capture 1 as a false positive. + const span = tracer.startSpan('http-test') + tracer.scope().activate(span, () => { + const client = http2.connect(`${protocol}://localhost:${port}`) + .on('error', done) - client.request({ ':path': '/test-1' }) - .on('error', done) - .end() + client.request({ ':path': '/test-1' }) + .on('error', done) + .end() - client.request({ ':path': `/user?test=2` }) - .on('error', done) - .end() + client.request({ ':path': '/user?test=2' }) + .on('error', done) + .end() - span.finish() - }) + span.finish() }) }) }) @@ -682,7 +524,7 @@ describe('Plugin', () => { return agent.load('http2', config) .then(() => { - http2 = require('http2') + http2 = require(loadPlugin) }) }) @@ -694,7 +536,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', 'custom') @@ -702,16 +544,14 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) - const req = client.request({ ':path': '/user' }) - req.on('error', done) + const req = client.request({ ':path': '/user' }) + req.on('error', done) - req.end() - }) + req.end() }) }) }) @@ -729,7 +569,7 @@ describe('Plugin', () => { return agent.load('http2', config) .then(() => { - http2 = require('http2') + http2 = require(loadPlugin) }) }) @@ -741,7 +581,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('error', 1) @@ -749,16 +589,14 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) - const req = client.request({ ':path': '/user' }) - req.on('error', done) + const req = client.request({ ':path': '/user' }) + req.on('error', done) - req.end() - }) + req.end() }) }) }) @@ -777,30 +615,29 @@ describe('Plugin', () => { return agent.load('http2', config) .then(() => { - http2 = require('http2') + http2 = require(loadPlugin) }) }) withNamingSchema( (done) => { - getPort().then(port => { - serverPort = port - const app = (stream, headers) => { - stream.respond({ - ':status': 200 - }) - stream.end() - } - appListener = server(app, port, () => { - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) - - const req = client.request({ ':path': '/user', ':method': 'GET' }) - req.on('error', done) - - req.end() + const app = (stream, headers) => { + stream.respond({ + ':status': 200 }) + stream.end() + } + appListener = server(app, port => { + serverPort = port + + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) + + const req = client.request({ ':path': '/user', ':method': 'GET' }) + req.on('error', done) + + req.end() }) }, { @@ -823,7 +660,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', `localhost:${port}`) @@ -831,14 +668,12 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const client = http2.connect(`${protocol}://localhost:${port}`) - .on('error', done) + const client = http2.connect(`${protocol}://localhost:${port}`) + .on('error', done) - client.request({ ':path': '/user' }) - .on('error', done) - .end() - }) + client.request({ ':path': '/user' }) + .on('error', done) + .end() }) }) }) @@ -856,7 +691,7 @@ describe('Plugin', () => { return agent.load('http2', config) .then(() => { - http2 = require('http2') + http2 = require(loadPlugin) }) }) @@ -869,7 +704,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { const meta = traces[0][0].meta @@ -880,14 +715,12 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const client = http2.connect(`${protocol}://localhost:${port}`) - .on('error', done) + const client = http2.connect(`${protocol}://localhost:${port}`) + .on('error', done) - client.request({ ':path': '/user' }) - .on('error', done) - .end() - }) + client.request({ ':path': '/user' }) + .on('error', done) + .end() }) }) }) @@ -905,7 +738,7 @@ describe('Plugin', () => { return agent.load('http2', config) .then(() => { - http2 = require('http2') + http2 = require(loadPlugin) }) }) @@ -917,7 +750,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { const timer = setTimeout(done, 100) agent @@ -927,14 +760,12 @@ describe('Plugin', () => { }) .catch(done) - appListener = server(app, port, () => { - const client = http2.connect(`${protocol}://localhost:${port}`) - .on('error', done) + const client = http2.connect(`${protocol}://localhost:${port}`) + .on('error', done) - client.request({ ':path': '/user' }) - .on('error', done) - .end() - }) + client.request({ ':path': '/user' }) + .on('error', done) + .end() }) }) }) diff --git a/packages/datadog-plugin-http2/test/integration-test/client.spec.js b/packages/datadog-plugin-http2/test/integration-test/client.spec.js index 368841dc112..800dc1d0c6d 100644 --- a/packages/datadog-plugin-http2/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-http2/test/integration-test/client.spec.js @@ -16,7 +16,7 @@ describe('esm', () => { before(async function () { this.timeout(50000) sandbox = await createSandbox(['http2'], false, [ - `./packages/datadog-plugin-http2/test/integration-test/*`]) + './packages/datadog-plugin-http2/test/integration-test/*']) }) after(async function () { @@ -52,7 +52,7 @@ describe('esm', () => { }) async function curl (url) { - if (typeof url === 'object') { + if (url !== null && typeof url === 'object') { if (url.then) { return curl(await url) } diff --git a/packages/datadog-plugin-http2/test/server.spec.js b/packages/datadog-plugin-http2/test/server.spec.js index 47e54c2a29e..d86817b2860 100644 --- a/packages/datadog-plugin-http2/test/server.spec.js +++ b/packages/datadog-plugin-http2/test/server.spec.js @@ -1,7 +1,6 @@ 'use strict' const { EventEmitter } = require('events') -const getPort = require('get-port') const agent = require('../../dd-trace/test/plugins/agent') const { rawExpectedSchema } = require('./naming') @@ -9,6 +8,7 @@ class MockAbortController { constructor () { this.signal = new EventEmitter() } + abort () { this.signal.emit('abort') } @@ -51,201 +51,200 @@ describe('Plugin', () => { let port let app - describe('http2/server', () => { - beforeEach(() => { - tracer = require('../../dd-trace') - listener = (req, res) => { - app && app(req, res) - res.writeHead(200) - res.end() - } - }) - - beforeEach(() => { - return getPort().then(newPort => { - port = newPort - }) - }) - - afterEach(() => { - appListener && appListener.close() - app = null - return agent.close({ ritmReset: false }) - }) - - describe('cancelled request', () => { + ['http2', 'node:http2'].forEach(pluginToBeLoaded => { + describe(`${pluginToBeLoaded}/server`, () => { beforeEach(() => { + tracer = require('../../dd-trace') listener = (req, res) => { - setTimeout(() => { - app && app(req, res) - res.writeHead(200) - res.end() - }, 500) + app && app(req, res) + res.writeHead(200) + res.end() } }) - beforeEach(() => { - return agent.load('http2') - .then(() => { - http2 = require('http2') - }) + afterEach(() => { + appListener && appListener.close() + app = null + return agent.close({ ritmReset: false }) }) - beforeEach(done => { - const server = http2.createServer(listener) - appListener = server - .listen(port, 'localhost', () => done()) - }) + describe('cancelled request', () => { + beforeEach(() => { + listener = (req, res) => { + setTimeout(() => { + app && app(req, res) + res.writeHead(200) + res.end() + }, 500) + } + }) - it('should send traces to agent', (done) => { - app = sinon.stub() - agent - .use(traces => { - expect(app).not.to.have.been.called // request should be cancelled before call to app - expect(traces[0][0]).to.have.property('name', 'web.request') - expect(traces[0][0]).to.have.property('service', 'test') - expect(traces[0][0]).to.have.property('type', 'web') - expect(traces[0][0]).to.have.property('resource', 'GET') - expect(traces[0][0].meta).to.have.property('span.kind', 'server') - expect(traces[0][0].meta).to.have.property('http.url', `http://localhost:${port}/user`) - expect(traces[0][0].meta).to.have.property('http.method', 'GET') - expect(traces[0][0].meta).to.have.property('http.status_code', '200') - expect(traces[0][0].meta).to.have.property('component', 'http2') - }) - .then(done) - .catch(done) + beforeEach(() => { + return agent.load('http2') + .then(() => { + http2 = require(pluginToBeLoaded) + }) + }) - // Don't use real AbortController because it requires 15.x+ - const ac = new MockAbortController() - request(http2, `http://localhost:${port}/user`, { - signal: ac.signal + beforeEach(done => { + const server = http2.createServer(listener) + appListener = server + .listen(0, 'localhost', () => { + port = appListener.address().port + done() + }) }) - setTimeout(() => { ac.abort() }, 100) - }) - }) - describe('without configuration', () => { - beforeEach(() => { - return agent.load('http2') - .then(() => { - http2 = require('http2') + it('should send traces to agent', (done) => { + app = sinon.stub() + agent + .use(traces => { + expect(app).not.to.have.been.called // request should be cancelled before call to app + expect(traces[0][0]).to.have.property('name', 'web.request') + expect(traces[0][0]).to.have.property('service', 'test') + expect(traces[0][0]).to.have.property('type', 'web') + expect(traces[0][0]).to.have.property('resource', 'GET') + expect(traces[0][0].meta).to.have.property('span.kind', 'server') + expect(traces[0][0].meta).to.have.property('http.url', `http://localhost:${port}/user`) + expect(traces[0][0].meta).to.have.property('http.method', 'GET') + expect(traces[0][0].meta).to.have.property('http.status_code', '200') + expect(traces[0][0].meta).to.have.property('component', 'http2') + }) + .then(done) + .catch(done) + + // Don't use real AbortController because it requires 15.x+ + const ac = new MockAbortController() + request(http2, `http://localhost:${port}/user`, { + signal: ac.signal }) + setTimeout(() => { ac.abort() }, 100) + }) }) - beforeEach(done => { - const server = http2.createServer(listener) - appListener = server - .listen(port, 'localhost', () => done()) - }) - - const spanProducerFn = (done) => { - request(http2, `http://localhost:${port}/user`).catch(done) - } - - withNamingSchema( - spanProducerFn, - rawExpectedSchema.server - ) - - it('should do automatic instrumentation', done => { - agent - .use(traces => { - expect(traces[0][0]).to.have.property('name', 'web.request') - expect(traces[0][0]).to.have.property('service', 'test') - expect(traces[0][0]).to.have.property('type', 'web') - expect(traces[0][0]).to.have.property('resource', 'GET') - expect(traces[0][0].meta).to.have.property('span.kind', 'server') - expect(traces[0][0].meta).to.have.property('http.url', `http://localhost:${port}/user`) - expect(traces[0][0].meta).to.have.property('http.method', 'GET') - expect(traces[0][0].meta).to.have.property('http.status_code', '200') - expect(traces[0][0].meta).to.have.property('component', 'http2') - }) - .then(done) - .catch(done) + describe('without configuration', () => { + beforeEach(() => { + return agent.load('http2') + .then(() => { + http2 = require(pluginToBeLoaded) + }) + }) - request(http2, `http://localhost:${port}/user`).catch(done) - }) + beforeEach(done => { + const server = http2.createServer(listener) + appListener = server + .listen(port, 'localhost', () => done()) + }) - it(`should run the request's close event in the correct context`, done => { - app = (req, res) => { - req.on('close', () => { - expect(tracer.scope().active()).to.equal(null) - done() - }) + const spanProducerFn = (done) => { + request(http2, `http://localhost:${port}/user`).catch(done) } - request(http2, `http://localhost:${port}/user`).catch(done) - }) + withNamingSchema( + spanProducerFn, + rawExpectedSchema.server + ) + + it('should do automatic instrumentation', done => { + agent + .use(traces => { + expect(traces[0][0]).to.have.property('name', 'web.request') + expect(traces[0][0]).to.have.property('service', 'test') + expect(traces[0][0]).to.have.property('type', 'web') + expect(traces[0][0]).to.have.property('resource', 'GET') + expect(traces[0][0].meta).to.have.property('span.kind', 'server') + expect(traces[0][0].meta).to.have.property('http.url', `http://localhost:${port}/user`) + expect(traces[0][0].meta).to.have.property('http.method', 'GET') + expect(traces[0][0].meta).to.have.property('http.status_code', '200') + expect(traces[0][0].meta).to.have.property('component', 'http2') + }) + .then(done) + .catch(done) + + request(http2, `http://localhost:${port}/user`).catch(done) + }) - it(`should run the response's close event in the correct context`, done => { - app = (req, res) => { - const span = tracer.scope().active() + it('should run the request\'s close event in the correct context', done => { + app = (req, res) => { + req.on('close', () => { + expect(tracer.scope().active()).to.equal(null) + done() + }) + } - res.on('close', () => { - expect(tracer.scope().active()).to.equal(span) - done() - }) - } + request(http2, `http://localhost:${port}/user`).catch(done) + }) - request(http2, `http://localhost:${port}/user`).catch(done) - }) + it('should run the response\'s close event in the correct context', done => { + app = (req, res) => { + const span = tracer.scope().active() - it(`should run the finish event in the correct context`, done => { - app = (req, res) => { - const span = tracer.scope().active() + res.on('close', () => { + expect(tracer.scope().active()).to.equal(span) + done() + }) + } - res.on('finish', () => { - expect(tracer.scope().active()).to.equal(span) - done() - }) - } + request(http2, `http://localhost:${port}/user`).catch(done) + }) - request(http2, `http://localhost:${port}/user`).catch(done) - }) + it('should run the finish event in the correct context', done => { + app = (req, res) => { + const span = tracer.scope().active() - it('should not cause `end` to be called multiple times', done => { - app = (req, res) => { - res.end = sinon.spy(res.end) + res.on('finish', () => { + expect(tracer.scope().active()).to.equal(span) + done() + }) + } - res.on('finish', () => { - expect(res.end).to.have.been.calledOnce - done() - }) - } + request(http2, `http://localhost:${port}/user`).catch(done) + }) - request(http2, `http://localhost:${port}/user`).catch(done) - }) - }) + it('should not cause `end` to be called multiple times', done => { + app = (req, res) => { + res.end = sinon.spy(res.end) - describe('with a blocklist configuration', () => { - beforeEach(() => { - return agent.load('http2', { client: false, blocklist: '/health' }) - .then(() => { - http2 = require('http2') - }) - }) + res.on('finish', () => { + expect(res.end).to.have.been.calledOnce + done() + }) + } - beforeEach(done => { - const server = http2.createServer(listener) - appListener = server - .listen(port, 'localhost', () => done()) + request(http2, `http://localhost:${port}/user`).catch(done) + }) }) - it('should drop traces for blocklist route', done => { - const spy = sinon.spy(() => {}) + describe('with a blocklist configuration', () => { + beforeEach(() => { + return agent.load('http2', { client: false, blocklist: '/health' }) + .then(() => { + http2 = require(pluginToBeLoaded) + }) + }) + + beforeEach(done => { + const server = http2.createServer(listener) + appListener = server + .listen(port, 'localhost', () => done()) + }) - agent - .use((traces) => { - spy() - }) - .catch(done) + it('should drop traces for blocklist route', done => { + const spy = sinon.spy(() => {}) - setTimeout(() => { - expect(spy).to.not.have.been.called - done() - }, 100) + agent + .use((traces) => { + spy() + }) + .catch(done) - request(http2, `http://localhost:${port}/health`).catch(done) + setTimeout(() => { + expect(spy).to.not.have.been.called + done() + }, 100) + + request(http2, `http://localhost:${port}/health`).catch(done) + }) }) }) }) diff --git a/packages/datadog-plugin-ioredis/test/index.spec.js b/packages/datadog-plugin-ioredis/test/index.spec.js index dbf7816f768..435e801f869 100644 --- a/packages/datadog-plugin-ioredis/test/index.spec.js +++ b/packages/datadog-plugin-ioredis/test/index.spec.js @@ -26,6 +26,7 @@ describe('Plugin', () => { describe('without configuration', () => { before(() => agent.load(['ioredis'])) + after(() => agent.close({ ritmReset: false })) it('should do automatic instrumentation when using callbacks', done => { @@ -118,6 +119,7 @@ describe('Plugin', () => { splitByInstance: true, allowlist: ['get'] })) + after(() => agent.close({ ritmReset: false })) it('should be configured with the correct values', done => { @@ -162,6 +164,7 @@ describe('Plugin', () => { before(() => agent.load('ioredis', { whitelist: ['get'] })) + after(() => agent.close({ ritmReset: false })) it('should be able to filter commands', done => { diff --git a/packages/datadog-plugin-ioredis/test/integration-test/client.spec.js b/packages/datadog-plugin-ioredis/test/integration-test/client.spec.js index 27859befd1d..a328b846952 100644 --- a/packages/datadog-plugin-ioredis/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-ioredis/test/integration-test/client.spec.js @@ -16,7 +16,7 @@ describe('esm', () => { before(async function () { this.timeout(20000) sandbox = await createSandbox([`'ioredis@${version}'`], false, [ - `./packages/datadog-plugin-ioredis/test/integration-test/*`]) + './packages/datadog-plugin-ioredis/test/integration-test/*']) }) after(async () => { diff --git a/packages/datadog-plugin-ioredis/test/naming.js b/packages/datadog-plugin-ioredis/test/naming.js index 2df9f5056d8..1ed3f17e428 100644 --- a/packages/datadog-plugin-ioredis/test/naming.js +++ b/packages/datadog-plugin-ioredis/test/naming.js @@ -14,6 +14,6 @@ const rawExpectedSchema = { } module.exports = { - rawExpectedSchema: rawExpectedSchema, + rawExpectedSchema, expectedSchema: resolveNaming(rawExpectedSchema) } diff --git a/packages/datadog-plugin-jest/src/index.js b/packages/datadog-plugin-jest/src/index.js index 3eaceb034aa..4362094b0be 100644 --- a/packages/datadog-plugin-jest/src/index.js +++ b/packages/datadog-plugin-jest/src/index.js @@ -12,10 +12,31 @@ const { TEST_FRAMEWORK_VERSION, TEST_SOURCE_START, TEST_ITR_UNSKIPPABLE, - TEST_ITR_FORCED_RUN + TEST_ITR_FORCED_RUN, + TEST_CODE_OWNERS, + ITR_CORRELATION_ID, + TEST_SOURCE_FILE, + TEST_IS_NEW, + TEST_IS_RETRY, + TEST_EARLY_FLAKE_ENABLED, + TEST_EARLY_FLAKE_ABORT_REASON, + JEST_DISPLAY_NAME, + TEST_IS_RUM_ACTIVE, + TEST_BROWSER_DRIVER } = require('../../dd-trace/src/plugins/util/test') const { COMPONENT } = require('../../dd-trace/src/constants') const id = require('../../dd-trace/src/id') +const { + TELEMETRY_EVENT_CREATED, + TELEMETRY_EVENT_FINISHED, + TELEMETRY_CODE_COVERAGE_STARTED, + TELEMETRY_CODE_COVERAGE_FINISHED, + TELEMETRY_ITR_FORCED_TO_RUN, + TELEMETRY_CODE_COVERAGE_EMPTY, + TELEMETRY_ITR_UNSKIPPABLE, + TELEMETRY_CODE_COVERAGE_NUM_FILES, + TELEMETRY_TEST_SESSION +} = require('../../dd-trace/src/ci-visibility/telemetry') const isJestWorker = !!process.env.JEST_WORKER_ID @@ -27,6 +48,21 @@ class JestPlugin extends CiPlugin { return 'jest' } + // The lists are the same for every test suite, so we can cache them + getUnskippableSuites (unskippableSuitesList) { + if (!this.unskippableSuites) { + this.unskippableSuites = JSON.parse(unskippableSuitesList) + } + return this.unskippableSuites + } + + getForcedToRunSuites (forcedToRunSuitesList) { + if (!this.forcedToRunSuites) { + this.forcedToRunSuites = JSON.parse(forcedToRunSuitesList) + } + return this.forcedToRunSuites + } + constructor (...args) { super(...args) @@ -55,7 +91,10 @@ class JestPlugin extends CiPlugin { numSkippedSuites, hasUnskippableSuites, hasForcedToRunSuites, - error + error, + isEarlyFlakeDetectionEnabled, + isEarlyFlakeDetectionFaulty, + onDone }) => { this.testSessionSpan.setTag(TEST_STATUS, status) this.testModuleSpan.setTag(TEST_STATUS, status) @@ -80,30 +119,60 @@ class JestPlugin extends CiPlugin { } ) + if (isEarlyFlakeDetectionEnabled) { + this.testSessionSpan.setTag(TEST_EARLY_FLAKE_ENABLED, 'true') + } + if (isEarlyFlakeDetectionFaulty) { + this.testSessionSpan.setTag(TEST_EARLY_FLAKE_ABORT_REASON, 'faulty') + } + this.testModuleSpan.finish() + this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'module') this.testSessionSpan.finish() + this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'session') finishAllTraceSpans(this.testSessionSpan) - this.tracer._exporter.flush() + + this.telemetry.count(TELEMETRY_TEST_SESSION, { provider: this.ciProviderName }) + + this.tracer._exporter.flush(() => { + if (onDone) { + onDone() + } + }) }) // Test suites can be run in a different process from jest's main one. // This subscriber changes the configuration objects from jest to inject the trace id - // of the test session to the processes that run the test suites. + // of the test session to the processes that run the test suites, and other data. this.addSub('ci:jest:session:configuration', configs => { configs.forEach(config => { config._ddTestSessionId = this.testSessionSpan.context().toTraceId() config._ddTestModuleId = this.testModuleSpan.context().toSpanId() config._ddTestCommand = this.testSessionSpan.context()._tags[TEST_COMMAND] + config._ddItrCorrelationId = this.itrCorrelationId + config._ddIsEarlyFlakeDetectionEnabled = !!this.libraryConfig?.isEarlyFlakeDetectionEnabled + config._ddEarlyFlakeDetectionNumRetries = this.libraryConfig?.earlyFlakeDetectionNumRetries ?? 0 + config._ddRepositoryRoot = this.repositoryRoot + config._ddIsFlakyTestRetriesEnabled = this.libraryConfig?.isFlakyTestRetriesEnabled ?? false + config._ddFlakyTestRetriesCount = this.libraryConfig?.flakyTestRetriesCount }) }) - this.addSub('ci:jest:test-suite:start', ({ testSuite, testEnvironmentOptions, frameworkVersion }) => { + this.addSub('ci:jest:test-suite:start', ({ + testSuite, + testSourceFile, + testEnvironmentOptions, + frameworkVersion, + displayName + }) => { const { _ddTestSessionId: testSessionId, _ddTestCommand: testCommand, _ddTestModuleId: testModuleId, + _ddItrCorrelationId: itrCorrelationId, _ddForcedToRun, - _ddUnskippable + _ddUnskippable, + _ddTestCodeCoverageEnabled } = testEnvironmentOptions const testSessionSpanContext = this.tracer.extract('text_map', { @@ -114,11 +183,35 @@ class JestPlugin extends CiPlugin { const testSuiteMetadata = getTestSuiteCommonTags(testCommand, frameworkVersion, testSuite, 'jest') if (_ddUnskippable) { - testSuiteMetadata[TEST_ITR_UNSKIPPABLE] = 'true' + const unskippableSuites = this.getUnskippableSuites(_ddUnskippable) + if (unskippableSuites[testSuite]) { + this.telemetry.count(TELEMETRY_ITR_UNSKIPPABLE, { testLevel: 'suite' }) + testSuiteMetadata[TEST_ITR_UNSKIPPABLE] = 'true' + } if (_ddForcedToRun) { - testSuiteMetadata[TEST_ITR_FORCED_RUN] = 'true' + const forcedToRunSuites = this.getForcedToRunSuites(_ddForcedToRun) + if (forcedToRunSuites[testSuite]) { + this.telemetry.count(TELEMETRY_ITR_FORCED_TO_RUN, { testLevel: 'suite' }) + testSuiteMetadata[TEST_ITR_FORCED_RUN] = 'true' + } } } + if (itrCorrelationId) { + testSuiteMetadata[ITR_CORRELATION_ID] = itrCorrelationId + } + if (displayName) { + testSuiteMetadata[JEST_DISPLAY_NAME] = displayName + } + if (testSourceFile) { + testSuiteMetadata[TEST_SOURCE_FILE] = testSourceFile + // Test suite is the whole test file, so we can use the first line as the start + testSuiteMetadata[TEST_SOURCE_START] = 1 + } + + const codeOwners = this.getCodeOwners(testSuiteMetadata) + if (codeOwners) { + testSuiteMetadata[TEST_CODE_OWNERS] = codeOwners + } this.testSuiteSpan = this.tracer.startSpan('jest.test_suite', { childOf: testSessionSpanContext, @@ -128,6 +221,10 @@ class JestPlugin extends CiPlugin { ...testSuiteMetadata } }) + this.telemetry.ciVisEvent(TELEMETRY_EVENT_CREATED, 'suite') + if (_ddTestCodeCoverageEnabled) { + this.telemetry.ciVisEvent(TELEMETRY_CODE_COVERAGE_STARTED, 'suite', { library: 'istanbul' }) + } }) this.addSub('ci:jest:worker-report:trace', traces => { @@ -164,6 +261,7 @@ class JestPlugin extends CiPlugin { this.testSuiteSpan.setTag('error', new Error(errorMessage)) } this.testSuiteSpan.finish() + this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'suite') // Suites potentially run in a different process than the session, // so calling finishAllTraceSpans on the session span is not enough finishAllTraceSpans(this.testSuiteSpan) @@ -176,18 +274,26 @@ class JestPlugin extends CiPlugin { }) /** - * This can't use `this.itrConfig` like `ci:mocha:test-suite:code-coverage` + * This can't use `this.libraryConfig` like `ci:mocha:test-suite:code-coverage` * because this subscription happens in a different process from the one * fetching the ITR config. */ - this.addSub('ci:jest:test-suite:code-coverage', (coverageFiles) => { + this.addSub('ci:jest:test-suite:code-coverage', ({ coverageFiles, testSuite }) => { + if (!coverageFiles.length) { + this.telemetry.count(TELEMETRY_CODE_COVERAGE_EMPTY) + } + const files = [...coverageFiles, testSuite] + const { _traceId, _spanId } = this.testSuiteSpan.context() const formattedCoverage = { sessionId: _traceId, suiteId: _spanId, - files: coverageFiles + files } + this.tracer._exporter.exportCoverage(formattedCoverage) + this.telemetry.ciVisEvent(TELEMETRY_CODE_COVERAGE_FINISHED, 'suite', { library: 'istanbul' }) + this.telemetry.distribution(TELEMETRY_CODE_COVERAGE_NUM_FILES, {}, files.length) }) this.addSub('ci:jest:test:start', (test) => { @@ -203,6 +309,19 @@ class JestPlugin extends CiPlugin { if (testStartLine) { span.setTag(TEST_SOURCE_START, testStartLine) } + + const spanTags = span.context()._tags + this.telemetry.ciVisEvent( + TELEMETRY_EVENT_FINISHED, + 'test', + { + hasCodeOwners: !!spanTags[TEST_CODE_OWNERS], + isNew: spanTags[TEST_IS_NEW] === 'true', + isRum: spanTags[TEST_IS_RUM_ACTIVE] === 'true', + browserDriver: spanTags[TEST_BROWSER_DRIVER] + } + ) + span.finish() finishAllTraceSpans(span) }) @@ -226,7 +345,19 @@ class JestPlugin extends CiPlugin { } startTestSpan (test) { - const { suite, name, runner, testParameters, frameworkVersion, testStartLine } = test + const { + suite, + name, + runner, + displayName, + testParameters, + frameworkVersion, + testStartLine, + testSourceFile, + isNew, + isEfdRetry, + isJestRetry + } = test const extraTags = { [JEST_TEST_RUNNER]: runner, @@ -236,6 +367,23 @@ class JestPlugin extends CiPlugin { if (testStartLine) { extraTags[TEST_SOURCE_START] = testStartLine } + // If for whatever we don't have the source file, we'll fall back to the suite name + extraTags[TEST_SOURCE_FILE] = testSourceFile || suite + + if (displayName) { + extraTags[JEST_DISPLAY_NAME] = displayName + } + + if (isNew) { + extraTags[TEST_IS_NEW] = 'true' + if (isEfdRetry) { + extraTags[TEST_IS_RETRY] = 'true' + } + } + + if (isJestRetry) { + extraTags[TEST_IS_RETRY] = 'true' + } return super.startTestSpan(name, suite, this.testSuiteSpan, extraTags) } diff --git a/packages/datadog-plugin-jest/src/util.js b/packages/datadog-plugin-jest/src/util.js index 0fb9d79cb40..0608e540e6e 100644 --- a/packages/datadog-plugin-jest/src/util.js +++ b/packages/datadog-plugin-jest/src/util.js @@ -77,30 +77,52 @@ function isMarkedAsUnskippable (test) { } function getJestSuitesToRun (skippableSuites, originalTests, rootDir) { - return originalTests.reduce((acc, test) => { + const unskippableSuites = {} + const forcedToRunSuites = {} + + const skippedSuites = [] + const suitesToRun = [] + + for (const test of originalTests) { const relativePath = getTestSuitePath(test.path, rootDir) const shouldBeSkipped = skippableSuites.includes(relativePath) - if (isMarkedAsUnskippable(test)) { - acc.suitesToRun.push(test) - if (test?.context?.config?.testEnvironmentOptions) { - test.context.config.testEnvironmentOptions['_ddUnskippable'] = true - acc.hasUnskippableSuites = true - if (shouldBeSkipped) { - test.context.config.testEnvironmentOptions['_ddForcedToRun'] = true - acc.hasForcedToRunSuites = true - } + suitesToRun.push(test) + unskippableSuites[relativePath] = true + if (shouldBeSkipped) { + forcedToRunSuites[relativePath] = true } - return acc + continue } - if (shouldBeSkipped) { - acc.skippedSuites.push(relativePath) + skippedSuites.push(relativePath) } else { - acc.suitesToRun.push(test) + suitesToRun.push(test) } - return acc - }, { skippedSuites: [], suitesToRun: [], hasUnskippableSuites: false, hasForcedToRunSuites: false }) + } + + const hasUnskippableSuites = Object.keys(unskippableSuites).length > 0 + const hasForcedToRunSuites = Object.keys(forcedToRunSuites).length > 0 + + if (originalTests.length) { + // The config object is shared by all tests, so we can just take the first one + const [test] = originalTests + if (test?.context?.config?.testEnvironmentOptions) { + if (hasUnskippableSuites) { + test.context.config.testEnvironmentOptions._ddUnskippable = JSON.stringify(unskippableSuites) + } + if (hasForcedToRunSuites) { + test.context.config.testEnvironmentOptions._ddForcedToRun = JSON.stringify(forcedToRunSuites) + } + } + } + + return { + skippedSuites, + suitesToRun, + hasUnskippableSuites, + hasForcedToRunSuites + } } module.exports = { getFormattedJestTestParameters, getJestTestName, getJestSuitesToRun, isMarkedAsUnskippable } diff --git a/packages/datadog-plugin-jest/test/circus.spec.js b/packages/datadog-plugin-jest/test/circus.spec.js index 6137c113522..4be0492d663 100644 --- a/packages/datadog-plugin-jest/test/circus.spec.js +++ b/packages/datadog-plugin-jest/test/circus.spec.js @@ -97,6 +97,7 @@ describe('Plugin', function () { jestExecutable = loadedAgent.jestExecutable jestCommonOptions = loadedAgent.jestCommonOptions }) + it('should create test spans for sync, async, integration, parameterized and retried tests', (done) => { const tests = [ { @@ -321,6 +322,7 @@ describe('Plugin', function () { jestExecutable = loadedAgent.jestExecutable jestCommonOptions = loadedAgent.jestCommonOptions }) + it('should create events for session, suite and test', (done) => { const events = [ { diff --git a/packages/datadog-plugin-jest/test/jasmine2.spec.js b/packages/datadog-plugin-jest/test/jasmine2.spec.js deleted file mode 100644 index 7ea4b80000b..00000000000 --- a/packages/datadog-plugin-jest/test/jasmine2.spec.js +++ /dev/null @@ -1,228 +0,0 @@ -'use strict' -const fs = require('fs') -const path = require('path') -const nock = require('nock') - -const { ORIGIN_KEY, COMPONENT, ERROR_MESSAGE } = require('../../dd-trace/src/constants') -const agent = require('../../dd-trace/test/plugins/agent') -const { - TEST_FRAMEWORK, - TEST_TYPE, - TEST_NAME, - TEST_SUITE, - TEST_SOURCE_FILE, - TEST_STATUS, - CI_APP_ORIGIN, - TEST_FRAMEWORK_VERSION, - JEST_TEST_RUNNER, - TEST_CODE_OWNERS, - LIBRARY_VERSION -} = require('../../dd-trace/src/plugins/util/test') - -const { version: ddTraceVersion } = require('../../../package.json') -const { DD_MAJOR } = require('../../../version') - -const describeFunction = DD_MAJOR < 4 ? describe : describe.skip - -describeFunction('Plugin', function () { - this.retries(2) - let jestExecutable - - const jestCommonOptions = { - projects: [__dirname], - testPathIgnorePatterns: ['/node_modules/'], - coverageReporters: [], - reporters: [], - cache: false, - maxWorkers: '50%', - testEnvironment: 'node' - } - - withVersions('jest', ['jest-jasmine2'], (version) => { - afterEach(() => { - const jestTestFile = fs.readdirSync(__dirname).filter(name => name.startsWith('jest-')) - jestTestFile.forEach((testFile) => { - delete require.cache[require.resolve(path.join(__dirname, testFile))] - }) - return agent.close() - }) - beforeEach(() => { - // for http integration tests - nock('http://test:123') - .get('/') - .reply(200, 'OK') - - agent.setAvailableEndpoints([]) - - return agent.load( - ['jest', 'http'], { service: 'test' }, { experimental: { exporter: 'agent_proxy' } } - ).then(() => { - jestCommonOptions.testRunner = - require(`../../../versions/jest@${version}`).getPath('jest-jasmine2') - - jestExecutable = require(`../../../versions/jest@${version}`).get() - }) - }) - describe('jest with jasmine', function () { - this.timeout(20000) - it('instruments async, sync and integration tests', function (done) { - const tests = [ - { - name: 'jest-test-suite tracer and active span are available', - status: 'pass', - extraTags: { 'test.add.stuff': 'stuff' } - }, - { name: 'jest-test-suite done', status: 'pass' }, - { name: 'jest-test-suite done fail', status: 'fail' }, - { name: 'jest-test-suite done fail uncaught', status: 'fail' }, - { name: 'jest-test-suite can do integration http', status: 'pass' }, - { name: 'jest-test-suite promise passes', status: 'pass' }, - { name: 'jest-test-suite promise fails', status: 'fail' }, - { name: 'jest-test-suite timeout', status: 'fail' }, - { name: 'jest-test-suite passes', status: 'pass' }, - { name: 'jest-test-suite fails', status: 'fail' }, - { name: 'jest-test-suite does not crash with missing stack', status: 'fail' }, - { name: 'jest-test-suite skips', status: 'skip' }, - { name: 'jest-test-suite skips todo', status: 'skip' } - ] - const assertionPromises = tests.map(({ name, status, error, extraTags }) => { - return agent.use(trace => { - const testSpan = trace[0][0] - expect(testSpan.parent_id.toString()).to.equal('0') - expect(testSpan.meta).to.contain({ - language: 'javascript', - service: 'test', - [ORIGIN_KEY]: CI_APP_ORIGIN, - [TEST_FRAMEWORK]: 'jest', - [TEST_NAME]: name, - [TEST_STATUS]: status, - [TEST_SUITE]: 'packages/datadog-plugin-jest/test/jest-test.js', - [TEST_SOURCE_FILE]: 'packages/datadog-plugin-jest/test/jest-test.js', - [TEST_TYPE]: 'test', - [JEST_TEST_RUNNER]: 'jest-jasmine2', - [LIBRARY_VERSION]: ddTraceVersion, - [COMPONENT]: 'jest' - }) - // reads from dd-trace-js' CODEOWNERS - expect(testSpan.meta[TEST_CODE_OWNERS]).to.contain('@DataDog') - if (extraTags) { - expect(testSpan.meta).to.contain(extraTags) - } - if (error) { - expect(testSpan.meta[ERROR_MESSAGE]).to.include(error) - } - // TODO: add assertions on http spans when stealthy-require issue is resolved - if (name === 'jest-test-suite can do integration http') { - const httpSpan = trace[0].find(span => span.name === 'http.request') - expect(httpSpan.meta[ORIGIN_KEY]).to.equal(CI_APP_ORIGIN) - expect(httpSpan.meta['http.url']).to.equal('http://test:123/') - expect(httpSpan.parent_id.toString()).to.equal(testSpan.span_id.toString()) - } - expect(testSpan.type).to.equal('test') - expect(testSpan.name).to.equal('jest.test') - expect(testSpan.service).to.equal('test') - expect(testSpan.resource).to.equal(`packages/datadog-plugin-jest/test/jest-test.js.${name}`) - expect(testSpan.meta[TEST_FRAMEWORK_VERSION]).not.to.be.undefined - }, { timeoutMs: 10000, spanResourceMatch: new RegExp(`${name}$`) }) - }) - - Promise.all(assertionPromises).then(() => done()).catch(done) - - const options = { - ...jestCommonOptions, - testRegex: 'jest-test.js' - } - - jestExecutable.runCLI( - options, - options.projects - ) - }) - - it('works when there is a hook error', (done) => { - const tests = [ - { name: 'jest-hook-failure will not run', error: 'hey, hook error before' }, - { name: 'jest-hook-failure-after will not run', error: 'hey, hook error after' } - ] - - const assertionPromises = tests.map(({ name, error }) => { - return agent.use(trace => { - const testSpan = trace[0][0] - expect(testSpan.parent_id.toString()).to.equal('0') - expect(testSpan.meta).to.contain({ - language: 'javascript', - service: 'test', - [ORIGIN_KEY]: CI_APP_ORIGIN, - [TEST_FRAMEWORK]: 'jest', - [TEST_NAME]: name, - [TEST_STATUS]: 'fail', - [TEST_SUITE]: 'packages/datadog-plugin-jest/test/jest-hook-failure.js', - [TEST_SOURCE_FILE]: 'packages/datadog-plugin-jest/test/jest-hook-failure.js', - [TEST_TYPE]: 'test', - [JEST_TEST_RUNNER]: 'jest-jasmine2', - [COMPONENT]: 'jest' - }) - expect(testSpan.meta[ERROR_MESSAGE]).to.equal(error) - expect(testSpan.type).to.equal('test') - expect(testSpan.name).to.equal('jest.test') - expect(testSpan.service).to.equal('test') - expect(testSpan.resource).to.equal( - `packages/datadog-plugin-jest/test/jest-hook-failure.js.${name}` - ) - expect(testSpan.meta[TEST_FRAMEWORK_VERSION]).not.to.be.undefined - }, { timeoutMs: 10000, spanResourceMatch: new RegExp(`${name}$`) }) - }) - - Promise.all(assertionPromises).then(() => done()).catch(done) - - const options = { - ...jestCommonOptions, - testRegex: 'jest-hook-failure.js' - } - - jestExecutable.runCLI( - options, - options.projects - ) - }) - - it('should work with focused tests', (done) => { - const tests = [ - { name: 'jest-test-focused will be skipped', status: 'skip' }, - { name: 'jest-test-focused-2 will be skipped too', status: 'skip' }, - { name: 'jest-test-focused can do focused test', status: 'pass' } - ] - - const assertionPromises = tests.map(({ name, status }) => { - return agent.use(trace => { - const testSpan = trace[0].find(span => span.type === 'test') - expect(testSpan.parent_id.toString()).to.equal('0') - expect(testSpan.meta[ORIGIN_KEY]).to.equal(CI_APP_ORIGIN) - expect(testSpan.meta).to.contain({ - language: 'javascript', - service: 'test', - [TEST_NAME]: name, - [TEST_STATUS]: status, - [TEST_FRAMEWORK]: 'jest', - [TEST_SUITE]: 'packages/datadog-plugin-jest/test/jest-focus.js', - [TEST_SOURCE_FILE]: 'packages/datadog-plugin-jest/test/jest-focus.js', - [COMPONENT]: 'jest' - }) - }, { timeoutMs: 10000, spanResourceMatch: new RegExp(`${name}$`) }) - }) - - Promise.all(assertionPromises).then(() => done()).catch(done) - - const options = { - ...jestCommonOptions, - testRegex: 'jest-focus.js' - } - - jestExecutable.runCLI( - options, - options.projects - ) - }) - }) - }) -}) diff --git a/packages/datadog-plugin-jest/test/jest-hook-failure.js b/packages/datadog-plugin-jest/test/jest-hook-failure.js index 73d20e09e5c..d17df4a983f 100644 --- a/packages/datadog-plugin-jest/test/jest-hook-failure.js +++ b/packages/datadog-plugin-jest/test/jest-hook-failure.js @@ -2,6 +2,7 @@ describe('jest-hook-failure', () => { beforeEach(() => { throw new Error('hey, hook error before') }) + it('will not run', () => { expect(true).toEqual(true) }) @@ -11,6 +12,7 @@ describe('jest-hook-failure-after', () => { afterEach(() => { throw new Error('hey, hook error after') }) + it('will not run', () => { expect(true).toEqual(true) }) diff --git a/packages/datadog-plugin-jest/test/jest-test.js b/packages/datadog-plugin-jest/test/jest-test.js index 8129585075a..d8155bcdc9a 100644 --- a/packages/datadog-plugin-jest/test/jest-test.js +++ b/packages/datadog-plugin-jest/test/jest-test.js @@ -4,18 +4,21 @@ const tracer = require('dd-trace') describe('jest-test-suite', () => { // eslint-disable-next-line jest.setTimeout(400) + it('tracer and active span are available', () => { expect(global._ddtrace).not.toEqual(undefined) const testSpan = tracer.scope().active() expect(testSpan).not.toEqual(null) testSpan.setTag('test.add.stuff', 'stuff') }) + it('done', (done) => { setTimeout(() => { expect(100).toEqual(100) done() }, 50) }) + it('done fail', (done) => { setTimeout(() => { try { @@ -26,12 +29,14 @@ describe('jest-test-suite', () => { } }, 50) }) + it('done fail uncaught', (done) => { setTimeout(() => { expect(100).toEqual(200) done() }, 50) }) + it('can do integration http', (done) => { const req = http.request('http://test:123', (res) => { expect(res.statusCode).toEqual(200) @@ -50,6 +55,7 @@ describe('jest-test-suite', () => { expect(parameters[1]).toEqual([2, 3, 5]) }) } + it('promise passes', () => { return new Promise((resolve) => setTimeout(() => { @@ -58,6 +64,7 @@ describe('jest-test-suite', () => { }, 50) ) }) + it('promise fails', () => { return new Promise((resolve) => setTimeout(() => { @@ -68,6 +75,7 @@ describe('jest-test-suite', () => { }) // eslint-disable-next-line jest.setTimeout(200) + it('timeout', () => { return new Promise((resolve) => setTimeout(() => { @@ -76,9 +84,11 @@ describe('jest-test-suite', () => { }, 300) ) }, 200) + it('passes', () => { expect(true).toEqual(true) }) + it('fails', () => { expect(true).toEqual(false) }) @@ -103,6 +113,7 @@ if (jest.retryTimes) { // eslint-disable-next-line jest.retryTimes(2) let retryAttempt = 0 + it('can retry', () => { expect(retryAttempt++).toEqual(2) }) diff --git a/packages/datadog-plugin-jest/test/util.spec.js b/packages/datadog-plugin-jest/test/util.spec.js index 6149df4c658..297f1f74161 100644 --- a/packages/datadog-plugin-jest/test/util.spec.js +++ b/packages/datadog-plugin-jest/test/util.spec.js @@ -6,10 +6,12 @@ describe('getFormattedJestTestParameters', () => { const result = getFormattedJestTestParameters([[[1, 2], [3, 4]]]) expect(result).to.eql([[1, 2], [3, 4]]) }) + it('returns formatted parameters for strings', () => { const result = getFormattedJestTestParameters([['\n a | b | expected\n '], 1, 2, 3, 3, 5, 8, 0, 1, 1]) expect(result).to.eql([{ a: 1, b: 2, expected: 3 }, { a: 3, b: 5, expected: 8 }, { a: 0, b: 1, expected: 1 }]) }) + it('does not crash for invalid inputs', () => { const resultUndefined = getFormattedJestTestParameters(undefined) const resultEmptyArray = getFormattedJestTestParameters([]) @@ -153,18 +155,24 @@ describe('getJestSuitesToRun', () => { it('adds extra `testEnvironmentOptions` if suite is unskippable or forced to run', () => { const skippableSuites = ['fixtures/test-unskippable.js'] - const testContext = { config: { testEnvironmentOptions: {} } } + // tests share a config object + const globalConfig = { testEnvironmentOptions: {} } const tests = [ - { path: path.join(__dirname, './fixtures/test-to-run.js') }, + { + path: path.join(__dirname, './fixtures/test-to-run.js'), + context: { config: globalConfig } + }, { path: path.join(__dirname, './fixtures/test-unskippable.js'), - context: testContext + context: { config: globalConfig } } ] const rootDir = __dirname getJestSuitesToRun(skippableSuites, tests, rootDir) - expect(testContext.config.testEnvironmentOptions['_ddUnskippable']).to.equal(true) - expect(testContext.config.testEnvironmentOptions['_ddForcedToRun']).to.equal(true) + expect(globalConfig.testEnvironmentOptions._ddUnskippable) + .to.eql(JSON.stringify({ 'fixtures/test-unskippable.js': true })) + expect(globalConfig.testEnvironmentOptions._ddForcedToRun) + .to.eql(JSON.stringify({ 'fixtures/test-unskippable.js': true })) }) }) diff --git a/packages/datadog-plugin-kafkajs/src/batch-consumer.js b/packages/datadog-plugin-kafkajs/src/batch-consumer.js new file mode 100644 index 00000000000..e0228a018c2 --- /dev/null +++ b/packages/datadog-plugin-kafkajs/src/batch-consumer.js @@ -0,0 +1,23 @@ +const ConsumerPlugin = require('../../dd-trace/src/plugins/consumer') +const { getMessageSize } = require('../../dd-trace/src/datastreams/processor') + +class KafkajsBatchConsumerPlugin extends ConsumerPlugin { + static get id () { return 'kafkajs' } + static get operation () { return 'consume-batch' } + + start ({ topic, partition, messages, groupId, clusterId }) { + if (!this.config.dsmEnabled) return + for (const message of messages) { + if (!message || !message.headers) continue + const payloadSize = getMessageSize(message) + this.tracer.decodeDataStreamsContext(message.headers) + const edgeTags = ['direction:in', `group:${groupId}`, `topic:${topic}`, 'type:kafka'] + if (clusterId) { + edgeTags.push(`kafka_cluster_id:${clusterId}`) + } + this.tracer.setCheckpoint(edgeTags, null, payloadSize) + } + } +} + +module.exports = KafkajsBatchConsumerPlugin diff --git a/packages/datadog-plugin-kafkajs/src/consumer.js b/packages/datadog-plugin-kafkajs/src/consumer.js index c29cb389e10..ee04c5eb60c 100644 --- a/packages/datadog-plugin-kafkajs/src/consumer.js +++ b/packages/datadog-plugin-kafkajs/src/consumer.js @@ -1,34 +1,105 @@ 'use strict' -const { getMessageSize, CONTEXT_PROPAGATION_KEY } = require('../../dd-trace/src/datastreams/processor') +const dc = require('dc-polyfill') +const { getMessageSize } = require('../../dd-trace/src/datastreams/processor') const ConsumerPlugin = require('../../dd-trace/src/plugins/consumer') +const afterStartCh = dc.channel('dd-trace:kafkajs:consumer:afterStart') +const beforeFinishCh = dc.channel('dd-trace:kafkajs:consumer:beforeFinish') + class KafkajsConsumerPlugin extends ConsumerPlugin { static get id () { return 'kafkajs' } static get operation () { return 'consume' } - start ({ topic, partition, message, groupId }) { + constructor () { + super(...arguments) + this.addSub('apm:kafkajs:consume:commit', message => this.commit(message)) + } + + /** + * Transform individual commit details sent by kafkajs' event reporter + * into actionable backlog items for DSM + * + * @typedef {object} ConsumerBacklog + * @property {number} type + * @property {string} consumer_group + * @property {string} topic + * @property {number} partition + * @property {number} offset + * + * @typedef {object} CommitEventItem + * @property {string} groupId + * @property {string} topic + * @property {number} partition + * @property {import('kafkajs/utils/long').Long} offset + * + * @param {CommitEventItem} commit + * @returns {ConsumerBacklog} + */ + transformCommit (commit) { + const { groupId, partition, offset, topic } = commit + return { + partition, + topic, + type: 'kafka_commit', + offset: Number(offset), + consumer_group: groupId + } + } + + commit (commitList) { + if (!this.config.dsmEnabled) return + const keys = [ + 'consumer_group', + 'type', + 'partition', + 'offset', + 'topic' + ] + for (const commit of commitList.map(this.transformCommit)) { + if (keys.some(key => !commit.hasOwnProperty(key))) continue + this.tracer.setOffset(commit) + } + } + + start ({ topic, partition, message, groupId, clusterId }) { const childOf = extract(this.tracer, message.headers) const span = this.startSpan({ childOf, resource: topic, type: 'worker', meta: { - 'component': 'kafkajs', + component: 'kafkajs', 'kafka.topic': topic, - 'kafka.message.offset': message.offset + 'kafka.message.offset': message.offset, + 'kafka.cluster_id': clusterId }, metrics: { 'kafka.partition': partition } }) - if (this.config.dsmEnabled) { + if (this.config.dsmEnabled && message?.headers) { const payloadSize = getMessageSize(message) - this.tracer.decodeDataStreamsContext(message.headers[CONTEXT_PROPAGATION_KEY]) - this.tracer - .setCheckpoint(['direction:in', `group:${groupId}`, `topic:${topic}`, 'type:kafka'], span, payloadSize) + this.tracer.decodeDataStreamsContext(message.headers) + const edgeTags = ['direction:in', `group:${groupId}`, `topic:${topic}`, 'type:kafka'] + if (clusterId) { + edgeTags.push(`kafka_cluster_id:${clusterId}`) + } + this.tracer.setCheckpoint(edgeTags, span, payloadSize) + } + + if (afterStartCh.hasSubscribers) { + afterStartCh.publish({ topic, partition, message, groupId }) } } + + finish () { + if (beforeFinishCh.hasSubscribers) { + beforeFinishCh.publish() + } + + super.finish() + } } function extract (tracer, bufferMap) { diff --git a/packages/datadog-plugin-kafkajs/src/index.js b/packages/datadog-plugin-kafkajs/src/index.js index 9e5aec80606..3d20e8af67e 100644 --- a/packages/datadog-plugin-kafkajs/src/index.js +++ b/packages/datadog-plugin-kafkajs/src/index.js @@ -2,6 +2,7 @@ const ProducerPlugin = require('./producer') const ConsumerPlugin = require('./consumer') +const BatchConsumerPlugin = require('./batch-consumer') const CompositePlugin = require('../../dd-trace/src/plugins/composite') class KafkajsPlugin extends CompositePlugin { @@ -9,7 +10,8 @@ class KafkajsPlugin extends CompositePlugin { static get plugins () { return { producer: ProducerPlugin, - consumer: ConsumerPlugin + consumer: ConsumerPlugin, + batchConsumer: BatchConsumerPlugin } } } diff --git a/packages/datadog-plugin-kafkajs/src/producer.js b/packages/datadog-plugin-kafkajs/src/producer.js index a753021440c..aa12357b4cf 100644 --- a/packages/datadog-plugin-kafkajs/src/producer.js +++ b/packages/datadog-plugin-kafkajs/src/producer.js @@ -1,8 +1,8 @@ 'use strict' const ProducerPlugin = require('../../dd-trace/src/plugins/producer') -const { encodePathwayContext } = require('../../dd-trace/src/datastreams/pathway') -const { getMessageSize, CONTEXT_PROPAGATION_KEY } = require('../../dd-trace/src/datastreams/processor') +const { DsmPathwayCodec } = require('../../dd-trace/src/datastreams/pathway') +const { getMessageSize } = require('../../dd-trace/src/datastreams/processor') const BOOTSTRAP_SERVERS_KEY = 'messaging.kafka.bootstrap.servers' @@ -11,13 +11,68 @@ class KafkajsProducerPlugin extends ProducerPlugin { static get operation () { return 'produce' } static get peerServicePrecursors () { return [BOOTSTRAP_SERVERS_KEY] } - start ({ topic, messages, bootstrapServers }) { - let pathwayCtx + constructor () { + super(...arguments) + this.addSub('apm:kafkajs:produce:commit', message => this.commit(message)) + } + + /** + * Transform individual commit details sent by kafkajs' event reporter + * into actionable backlog items for DSM + * + * @typedef {object} ProducerBacklog + * @property {number} type + * @property {string} topic + * @property {number} partition + * @property {number} offset + * + * @typedef {object} ProducerResponseItem + * @property {string} topic + * @property {number} partition + * @property {import('kafkajs/utils/long').Long} [offset] + * @property {import('kafkajs/utils/long').Long} [baseOffset] + * + * @param {ProducerResponseItem} response + * @returns {ProducerBacklog} + */ + transformProduceResponse (response) { + // In produce protocol >=v3, the offset key changes from `offset` to `baseOffset` + const { topicName: topic, partition, offset, baseOffset } = response + const offsetAsLong = offset || baseOffset + return { + type: 'kafka_produce', + partition, + offset: offsetAsLong ? Number(offsetAsLong) : undefined, + topic + } + } + + /** + * + * @param {ProducerResponseItem[]} commitList + * @returns {void} + */ + commit (commitList) { + if (!this.config.dsmEnabled) return + const keys = [ + 'type', + 'partition', + 'offset', + 'topic' + ] + for (const commit of commitList.map(this.transformProduceResponse)) { + if (keys.some(key => !commit.hasOwnProperty(key))) continue + this.tracer.setOffset(commit) + } + } + + start ({ topic, messages, bootstrapServers, clusterId }) { const span = this.startSpan({ resource: topic, meta: { - 'component': 'kafkajs', - 'kafka.topic': topic + component: 'kafkajs', + 'kafka.topic': topic, + 'kafka.cluster_id': clusterId }, metrics: { 'kafka.batch_size': messages.length @@ -27,14 +82,18 @@ class KafkajsProducerPlugin extends ProducerPlugin { span.setTag(BOOTSTRAP_SERVERS_KEY, bootstrapServers) } for (const message of messages) { - if (typeof message === 'object') { + if (message !== null && typeof message === 'object') { this.tracer.inject(span, 'text_map', message.headers) if (this.config.dsmEnabled) { const payloadSize = getMessageSize(message) - const dataStreamsContext = this.tracer - .setCheckpoint(['direction:out', `topic:${topic}`, 'type:kafka'], span, payloadSize) - pathwayCtx = encodePathwayContext(dataStreamsContext) - message.headers[CONTEXT_PROPAGATION_KEY] = pathwayCtx + const edgeTags = ['direction:out', `topic:${topic}`, 'type:kafka'] + + if (clusterId) { + edgeTags.push(`kafka_cluster_id:${clusterId}`) + } + + const dataStreamsContext = this.tracer.setCheckpoint(edgeTags, span, payloadSize) + DsmPathwayCodec.encode(dataStreamsContext, message.headers) } } } diff --git a/packages/datadog-plugin-kafkajs/test/index.spec.js b/packages/datadog-plugin-kafkajs/test/index.spec.js index a797f83b94d..f67279bdd9f 100644 --- a/packages/datadog-plugin-kafkajs/test/index.spec.js +++ b/packages/datadog-plugin-kafkajs/test/index.spec.js @@ -1,6 +1,8 @@ 'use strict' const { expect } = require('chai') +const semver = require('semver') +const dc = require('dc-polyfill') const agent = require('../../dd-trace/test/plugins/agent') const { expectSomeSpan, withDefaults } = require('../../dd-trace/test/plugins/helpers') const { ERROR_MESSAGE, ERROR_TYPE, ERROR_STACK } = require('../../dd-trace/src/constants') @@ -11,22 +13,28 @@ const { computePathwayHash } = require('../../dd-trace/src/datastreams/pathway') const { ENTRY_PARENT_HASH, DataStreamsProcessor } = require('../../dd-trace/src/datastreams/processor') const testTopic = 'test-topic' -const expectedProducerHash = computePathwayHash( - 'test', - 'tester', - ['direction:out', 'topic:' + testTopic, 'type:kafka'], - ENTRY_PARENT_HASH -) -const expectedConsumerHash = computePathwayHash( - 'test', - 'tester', - ['direction:in', 'group:test-group', 'topic:' + testTopic, 'type:kafka'], - expectedProducerHash -) +const testKafkaClusterId = '5L6g3nShT-eMCtK--X86sw' + +const getDsmPathwayHash = (clusterIdAvailable, isProducer, parentHash) => { + let edgeTags + if (isProducer) { + edgeTags = ['direction:out', 'topic:' + testTopic, 'type:kafka'] + } else { + edgeTags = ['direction:in', 'group:test-group', 'topic:' + testTopic, 'type:kafka'] + } + + if (clusterIdAvailable) { + edgeTags.push(`kafka_cluster_id:${testKafkaClusterId}`) + } + edgeTags.sort() + return computePathwayHash('test', 'tester', edgeTags, parentHash) +} describe('Plugin', () => { describe('kafkajs', function () { - this.timeout(10000) // TODO: remove when new internal trace has landed + // TODO: remove when new internal trace has landed + this.timeout(10000) + afterEach(() => { return agent.close({ ritmReset: false }) }) @@ -34,29 +42,45 @@ describe('Plugin', () => { let kafka let tracer let Kafka + let clusterIdAvailable + let expectedProducerHash + let expectedConsumerHash + + before(() => { + clusterIdAvailable = semver.intersects(version, '>=1.13') + expectedProducerHash = getDsmPathwayHash(clusterIdAvailable, true, ENTRY_PARENT_HASH) + expectedConsumerHash = getDsmPathwayHash(clusterIdAvailable, false, expectedProducerHash) + }) + describe('without configuration', () => { const messages = [{ key: 'key1', value: 'test2' }] + beforeEach(async () => { - process.env['DD_DATA_STREAMS_ENABLED'] = 'true' + process.env.DD_DATA_STREAMS_ENABLED = 'true' tracer = require('../../dd-trace') await agent.load('kafkajs') - Kafka = require(`../../../versions/kafkajs@${version}`).get().Kafka - + const lib = require(`../../../versions/kafkajs@${version}`).get() + Kafka = lib.Kafka kafka = new Kafka({ clientId: `kafkajs-test-${version}`, - brokers: ['127.0.0.1:9092'] + brokers: ['127.0.0.1:9092'], + logLevel: lib.logLevel.WARN }) }) + describe('producer', () => { it('should be instrumented', async () => { + const meta = { + 'span.kind': 'producer', + component: 'kafkajs', + 'pathway.hash': expectedProducerHash.readBigUInt64BE(0).toString() + } + if (clusterIdAvailable) meta['kafka.cluster_id'] = testKafkaClusterId + const expectedSpanPromise = expectSpanWithDefaults({ name: expectedSchema.send.opName, service: expectedSchema.send.serviceName, - meta: { - 'span.kind': 'producer', - 'component': 'kafkajs', - 'pathway.hash': expectedProducerHash.readBigUInt64BE(0).toString() - }, + meta, metrics: { 'kafka.batch_size': messages.length }, @@ -97,7 +121,7 @@ describe('Plugin', () => { [ERROR_TYPE]: error.name, [ERROR_MESSAGE]: error.message, [ERROR_STACK]: error.stack, - 'component': 'kafkajs' + component: 'kafkajs' }) }) @@ -113,7 +137,8 @@ describe('Plugin', () => { return expectedSpanPromise } }) - if (version !== '1.4.0') { + // Dynamic broker list support added in 1.14/2.0 (https://github.com/tulios/kafkajs/commit/62223) + if (semver.intersects(version, '>=1.14')) { it('should not extract bootstrap servers when initialized with a function', async () => { const expectedSpanPromise = agent.use(traces => { const span = traces[0][0] @@ -136,8 +161,10 @@ describe('Plugin', () => { rawExpectedSchema.send ) }) - describe('consumer', () => { + + describe('consumer (eachMessage)', () => { let consumer + beforeEach(async () => { consumer = kafka.consumer({ groupId: 'test-group' }) await consumer.connect() @@ -154,7 +181,7 @@ describe('Plugin', () => { service: expectedSchema.receive.serviceName, meta: { 'span.kind': 'consumer', - 'component': 'kafkajs', + component: 'kafkajs', 'pathway.hash': expectedConsumerHash.readBigUInt64BE(0).toString() }, resource: testTopic, @@ -166,7 +193,6 @@ describe('Plugin', () => { eachMessage: () => {} }) await sendMessages(kafka, testTopic, messages) - return expectedSpanPromise }) @@ -219,7 +245,7 @@ describe('Plugin', () => { [ERROR_TYPE]: fakeError.name, [ERROR_MESSAGE]: fakeError.message, [ERROR_STACK]: fakeError.stack, - 'component': 'kafkajs' + component: 'kafkajs' }, resource: testTopic, error: 1, @@ -261,6 +287,69 @@ describe('Plugin', () => { .catch(done) }) + it('should publish on afterStart channel', (done) => { + const afterStart = dc.channel('dd-trace:kafkajs:consumer:afterStart') + + const spy = sinon.spy(() => { + expect(tracer.scope().active()).to.not.be.null + afterStart.unsubscribe(spy) + }) + afterStart.subscribe(spy) + + let eachMessage = async ({ topic, partition, message }) => { + try { + expect(spy).to.have.been.calledOnce + + const channelMsg = spy.firstCall.args[0] + expect(channelMsg).to.not.undefined + expect(channelMsg.topic).to.eq(testTopic) + expect(channelMsg.message.key).to.not.undefined + expect(channelMsg.message.key.toString()).to.eq(messages[0].key) + expect(channelMsg.message.value).to.not.undefined + expect(channelMsg.message.value.toString()).to.eq(messages[0].value) + + const name = spy.firstCall.args[1] + expect(name).to.eq(afterStart.name) + + done() + } catch (e) { + done(e) + } finally { + eachMessage = () => {} + } + } + + consumer.run({ eachMessage: (...args) => eachMessage(...args) }) + .then(() => sendMessages(kafka, testTopic, messages)) + }) + + it('should publish on beforeFinish channel', (done) => { + const beforeFinish = dc.channel('dd-trace:kafkajs:consumer:beforeFinish') + + const spy = sinon.spy(() => { + expect(tracer.scope().active()).to.not.be.null + beforeFinish.unsubscribe(spy) + }) + beforeFinish.subscribe(spy) + + let eachMessage = async ({ topic, partition, message }) => { + setImmediate(() => { + try { + expect(spy).to.have.been.calledOnceWith(undefined, beforeFinish.name) + + done() + } catch (e) { + done(e) + } + }) + + eachMessage = () => {} + } + + consumer.run({ eachMessage: (...args) => eachMessage(...args) }) + .then(() => sendMessages(kafka, testTopic, messages)) + }) + withNamingSchema( async () => { await consumer.run({ eachMessage: () => {} }) @@ -272,6 +361,7 @@ describe('Plugin', () => { describe('data stream monitoring', () => { let consumer + beforeEach(async () => { tracer.init() tracer.use('kafkajs', { dsmEnabled: true }) @@ -280,52 +370,148 @@ describe('Plugin', () => { await consumer.subscribe({ topic: testTopic }) }) + before(() => { + clusterIdAvailable = semver.intersects(version, '>=1.13') + expectedProducerHash = getDsmPathwayHash(clusterIdAvailable, true, ENTRY_PARENT_HASH) + expectedConsumerHash = getDsmPathwayHash(clusterIdAvailable, false, expectedProducerHash) + }) + afterEach(async () => { await consumer.disconnect() }) - it('Should set a checkpoint on produce', async () => { - const messages = [{ key: 'consumerDSM1', value: 'test2' }] - const setDataStreamsContextSpy = sinon.spy(DataStreamsContext, 'setDataStreamsContext') - await sendMessages(kafka, testTopic, messages) - expect(setDataStreamsContextSpy.args[0][0].hash).to.equal(expectedProducerHash) - setDataStreamsContextSpy.restore() - }) + describe('checkpoints', () => { + let setDataStreamsContextSpy - it('Should set a checkpoint on consume', async () => { - await sendMessages(kafka, testTopic, messages) - const setDataStreamsContextSpy = sinon.spy(DataStreamsContext, 'setDataStreamsContext') - await consumer.run({ - eachMessage: async ({ topic, partition, message, heartbeat, pause }) => { - expect(setDataStreamsContextSpy.args[0][0].hash).to.equal(expectedConsumerHash) + beforeEach(() => { + setDataStreamsContextSpy = sinon.spy(DataStreamsContext, 'setDataStreamsContext') + }) + + afterEach(() => { + setDataStreamsContextSpy.restore() + }) + + it('Should set a checkpoint on produce', async () => { + const messages = [{ key: 'consumerDSM1', value: 'test2' }] + await sendMessages(kafka, testTopic, messages) + expect(setDataStreamsContextSpy.args[0][0].hash).to.equal(expectedProducerHash) + }) + + it('Should set a checkpoint on consume (eachMessage)', async () => { + const runArgs = [] + await consumer.run({ + eachMessage: async () => { + runArgs.push(setDataStreamsContextSpy.lastCall.args[0]) + } + }) + await sendMessages(kafka, testTopic, messages) + await consumer.disconnect() + for (const runArg of runArgs) { + expect(runArg.hash).to.equal(expectedConsumerHash) } }) - setDataStreamsContextSpy.restore() - }) - it('Should set a message payload size when producing a message', async () => { - const messages = [{ key: 'key1', value: 'test2' }] - if (DataStreamsProcessor.prototype.recordCheckpoint.isSinonProxy) { - DataStreamsProcessor.prototype.recordCheckpoint.restore() - } - const recordCheckpointSpy = sinon.spy(DataStreamsProcessor.prototype, 'recordCheckpoint') - await sendMessages(kafka, testTopic, messages) - expect(recordCheckpointSpy.args[0][0].hasOwnProperty('payloadSize')) - recordCheckpointSpy.restore() + it('Should set a checkpoint on consume (eachBatch)', async () => { + const runArgs = [] + await consumer.run({ + eachBatch: async () => { + runArgs.push(setDataStreamsContextSpy.lastCall.args[0]) + } + }) + await sendMessages(kafka, testTopic, messages) + await consumer.disconnect() + for (const runArg of runArgs) { + expect(runArg.hash).to.equal(expectedConsumerHash) + } + }) + + it('Should set a message payload size when producing a message', async () => { + const messages = [{ key: 'key1', value: 'test2' }] + if (DataStreamsProcessor.prototype.recordCheckpoint.isSinonProxy) { + DataStreamsProcessor.prototype.recordCheckpoint.restore() + } + const recordCheckpointSpy = sinon.spy(DataStreamsProcessor.prototype, 'recordCheckpoint') + await sendMessages(kafka, testTopic, messages) + expect(recordCheckpointSpy.args[0][0].hasOwnProperty('payloadSize')) + recordCheckpointSpy.restore() + }) + + it('Should set a message payload size when consuming a message', async () => { + const messages = [{ key: 'key1', value: 'test2' }] + if (DataStreamsProcessor.prototype.recordCheckpoint.isSinonProxy) { + DataStreamsProcessor.prototype.recordCheckpoint.restore() + } + const recordCheckpointSpy = sinon.spy(DataStreamsProcessor.prototype, 'recordCheckpoint') + await sendMessages(kafka, testTopic, messages) + await consumer.run({ + eachMessage: async () => { + expect(recordCheckpointSpy.args[0][0].hasOwnProperty('payloadSize')) + recordCheckpointSpy.restore() + } + }) + }) }) - it('Should set a message payload size when consuming a message', async () => { - const messages = [{ key: 'key1', value: 'test2' }] - if (DataStreamsProcessor.prototype.recordCheckpoint.isSinonProxy) { - DataStreamsProcessor.prototype.recordCheckpoint.restore() + describe('backlogs', () => { + let setOffsetSpy + + beforeEach(() => { + setOffsetSpy = sinon.spy(tracer._tracer._dataStreamsProcessor, 'setOffset') + }) + + afterEach(() => { + setOffsetSpy.restore() + }) + + if (semver.intersects(version, '>=1.10')) { + it('Should add backlog on consumer explicit commit', async () => { + // Send a message, consume it, and record the last consumed offset + let commitMeta + await sendMessages(kafka, testTopic, messages) + await consumer.run({ + eachMessage: async payload => { + const { topic, partition, message } = payload + commitMeta = { + topic, + partition, + offset: Number(message.offset) + } + }, + autoCommit: false + }) + await new Promise(resolve => setTimeout(resolve, 50)) // Let eachMessage be called + await consumer.disconnect() // Flush ongoing `eachMessage` calls + for (const call of setOffsetSpy.getCalls()) { + expect(call.args[0]).to.not.have.property('type', 'kafka_commit') + } + + /** + * No choice but to reinitialize everything, because the only way to flush eachMessage + * calls is to disconnect. + */ + consumer.connect() + await sendMessages(kafka, testTopic, messages) + await consumer.run({ eachMessage: async () => {}, autoCommit: false }) + setOffsetSpy.resetHistory() + await consumer.commitOffsets([commitMeta]) + await consumer.disconnect() + + // Check our work + const runArg = setOffsetSpy.lastCall.args[0] + expect(setOffsetSpy).to.be.calledOnce + expect(runArg).to.have.property('offset', commitMeta.offset) + expect(runArg).to.have.property('partition', commitMeta.partition) + expect(runArg).to.have.property('topic', commitMeta.topic) + expect(runArg).to.have.property('type', 'kafka_commit') + expect(runArg).to.have.property('consumer_group', 'test-group') + }) } - const recordCheckpointSpy = sinon.spy(DataStreamsProcessor.prototype, 'recordCheckpoint') - await sendMessages(kafka, testTopic, messages) - await consumer.run({ - eachMessage: async () => { - expect(recordCheckpointSpy.args[0][0].hasOwnProperty('payloadSize')) - recordCheckpointSpy.restore() - } + + it('Should add backlog on producer response', async () => { + await sendMessages(kafka, testTopic, messages) + expect(setOffsetSpy).to.be.calledOnce + const { topic } = setOffsetSpy.lastCall.args[0] + expect(topic).to.equal(testTopic) }) }) }) diff --git a/packages/datadog-plugin-kafkajs/test/integration-test/client.spec.js b/packages/datadog-plugin-kafkajs/test/integration-test/client.spec.js index 34196638047..af231c00915 100644 --- a/packages/datadog-plugin-kafkajs/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-kafkajs/test/integration-test/client.spec.js @@ -16,7 +16,7 @@ describe('esm', () => { before(async function () { this.timeout(20000) sandbox = await createSandbox([`'kafkajs@${version}'`], false, [ - `./packages/datadog-plugin-kafkajs/test/integration-test/*`]) + './packages/datadog-plugin-kafkajs/test/integration-test/*']) }) after(async () => { diff --git a/packages/datadog-plugin-kafkajs/test/naming.js b/packages/datadog-plugin-kafkajs/test/naming.js index 80d44c86b87..78b1f5f3a1c 100644 --- a/packages/datadog-plugin-kafkajs/test/naming.js +++ b/packages/datadog-plugin-kafkajs/test/naming.js @@ -24,6 +24,6 @@ const rawExpectedSchema = { } module.exports = { - rawExpectedSchema: rawExpectedSchema, + rawExpectedSchema, expectedSchema: resolveNaming(rawExpectedSchema) } diff --git a/packages/datadog-plugin-koa/test/index.spec.js b/packages/datadog-plugin-koa/test/index.spec.js index 4ccbbcf71ad..c2bc08c135e 100644 --- a/packages/datadog-plugin-koa/test/index.spec.js +++ b/packages/datadog-plugin-koa/test/index.spec.js @@ -2,7 +2,6 @@ const { AsyncLocalStorage } = require('async_hooks') const axios = require('axios') -const getPort = require('get-port') const semver = require('semver') const { ERROR_TYPE } = require('../../dd-trace/src/constants') const agent = require('../../dd-trace/test/plugins/agent') @@ -16,14 +15,9 @@ describe('Plugin', () => { describe('koa', () => { withVersions('koa', 'koa', version => { - let port - beforeEach(() => { tracer = require('../../dd-trace') Koa = require(`../../../versions/koa@${version}`).get() - return getPort().then(newPort => { - port = newPort - }) }) afterEach(done => { @@ -32,6 +26,7 @@ describe('Plugin', () => { describe('without configuration', () => { before(() => agent.load(['koa', 'http'], [{}, { client: false }])) + after(() => agent.close({ ritmReset: false })) it('should do automatic instrumentation on 2.x middleware', done => { @@ -41,29 +36,31 @@ describe('Plugin', () => { ctx.body = '' }) - agent - .use(traces => { - const spans = sort(traces[0]) - - expect(spans[0]).to.have.property('name', 'koa.request') - expect(spans[0]).to.have.property('service', 'test') - expect(spans[0]).to.have.property('type', 'web') - expect(spans[0]).to.have.property('resource', 'GET') - expect(spans[0].meta).to.have.property('span.kind', 'server') - expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user`) - expect(spans[0].meta).to.have.property('http.method', 'GET') - expect(spans[0].meta).to.have.property('http.status_code', '200') - expect(spans[0].meta).to.have.property('component', 'koa') - - expect(spans[1]).to.have.property('name', 'koa.middleware') - expect(spans[1]).to.have.property('service', 'test') - expect(spans[1]).to.have.property('resource', 'handle') - expect(spans[1].meta).to.have.property('component', 'koa') - }) - .then(done) - .catch(done) - - appListener = app.listen(port, 'localhost', () => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + agent + .use(traces => { + const spans = sort(traces[0]) + + expect(spans[0]).to.have.property('name', 'koa.request') + expect(spans[0]).to.have.property('service', 'test') + expect(spans[0]).to.have.property('type', 'web') + expect(spans[0]).to.have.property('resource', 'GET') + expect(spans[0].meta).to.have.property('span.kind', 'server') + expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user`) + expect(spans[0].meta).to.have.property('http.method', 'GET') + expect(spans[0].meta).to.have.property('http.status_code', '200') + expect(spans[0].meta).to.have.property('component', 'koa') + + expect(spans[1]).to.have.property('name', 'koa.middleware') + expect(spans[1]).to.have.property('service', 'test') + expect(spans[1]).to.have.property('resource', 'handle') + expect(spans[1].meta).to.have.property('component', 'koa') + }) + .then(done) + .catch(done) + axios .get(`http://localhost:${port}/user`) .catch(done) @@ -78,29 +75,31 @@ describe('Plugin', () => { yield next }) - agent - .use(traces => { - const spans = sort(traces[0]) - - expect(spans[0]).to.have.property('name', 'koa.request') - expect(spans[0]).to.have.property('service', 'test') - expect(spans[0]).to.have.property('type', 'web') - expect(spans[0]).to.have.property('resource', 'GET') - expect(spans[0].meta).to.have.property('span.kind', 'server') - expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user`) - expect(spans[0].meta).to.have.property('http.method', 'GET') - expect(spans[0].meta).to.have.property('http.status_code', '200') - expect(spans[0].meta).to.have.property('component', 'koa') - - expect(spans[1]).to.have.property('name', 'koa.middleware') - expect(spans[1]).to.have.property('service', 'test') - expect(spans[1]).to.have.property('resource', 'converted') - expect(spans[1].meta).to.have.property('component', 'koa') - }) - .then(done) - .catch(done) - - appListener = app.listen(port, 'localhost', () => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + agent + .use(traces => { + const spans = sort(traces[0]) + + expect(spans[0]).to.have.property('name', 'koa.request') + expect(spans[0]).to.have.property('service', 'test') + expect(spans[0]).to.have.property('type', 'web') + expect(spans[0]).to.have.property('resource', 'GET') + expect(spans[0].meta).to.have.property('span.kind', 'server') + expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user`) + expect(spans[0].meta).to.have.property('http.method', 'GET') + expect(spans[0].meta).to.have.property('http.status_code', '200') + expect(spans[0].meta).to.have.property('component', 'koa') + + expect(spans[1]).to.have.property('name', 'koa.middleware') + expect(spans[1]).to.have.property('service', 'test') + expect(spans[1]).to.have.property('resource', 'converted') + expect(spans[1].meta).to.have.property('component', 'koa') + }) + .then(done) + .catch(done) + axios .get(`http://localhost:${port}/user`) .catch(done) @@ -123,7 +122,9 @@ describe('Plugin', () => { .catch(done) }) - appListener = app.listen(port, 'localhost', () => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + axios .get(`http://localhost:${port}/app/user/123`) .catch(done) @@ -151,11 +152,11 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/user`) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios.get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -206,12 +207,12 @@ describe('Plugin', () => { return next() }) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/1`) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/app/user/1`) + .catch(done) }) }) @@ -232,17 +233,19 @@ describe('Plugin', () => { app .use(koaRouter.get('/user/:id', getUser)) - agent - .use(traces => { - const spans = sort(traces[0]) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[0]).to.have.property('resource', 'GET /user/:id') - expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user/123`) - }) - .then(done) - .catch(done) + agent + .use(traces => { + const spans = sort(traces[0]) + + expect(spans[0]).to.have.property('resource', 'GET /user/:id') + expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user/123`) + }) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', (e) => { axios .get(`http://localhost:${port}/user/123`) .catch(done) @@ -269,21 +272,23 @@ describe('Plugin', () => { .use(router.routes()) .use(router.allowedMethods()) - agent - .use(traces => { - const spans = sort(traces[0]) - expect(spans[0]).to.have.property('resource', 'GET /user/:id') - expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user/123`) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[1]).to.have.property('resource') - expect(spans[1].resource).to.match(/^dispatch/) + agent + .use(traces => { + const spans = sort(traces[0]) + expect(spans[0]).to.have.property('resource', 'GET /user/:id') + expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user/123`) - expect(spans[2]).to.have.property('resource', 'handle') - }) - .then(done) - .catch(done) + expect(spans[1]).to.have.property('resource') + expect(spans[1].resource).to.match(/^dispatch/) + + expect(spans[2]).to.have.property('resource', 'handle') + }) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', (e) => { axios .get(`http://localhost:${port}/user/123`) .catch(done) @@ -303,16 +308,18 @@ describe('Plugin', () => { .use(router.routes()) .use(router.allowedMethods()) - agent - .use(traces => { - const spans = sort(traces[0]) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[0]).to.have.property('resource', 'GET /user/:id') - }) - .then(done) - .catch(done) + agent + .use(traces => { + const spans = sort(traces[0]) + + expect(spans[0]).to.have.property('resource', 'GET /user/:id') + }) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', (e) => { axios .get(`http://localhost:${port}/user/123`) .catch(done) @@ -332,16 +339,18 @@ describe('Plugin', () => { .use(router.routes()) .use(router.allowedMethods()) - agent - .use(traces => { - const spans = sort(traces[0]) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[0]).to.have.property('resource', 'GET /user/:id') - }) - .then(done) - .catch(done) + agent + .use(traces => { + const spans = sort(traces[0]) + + expect(spans[0]).to.have.property('resource', 'GET /user/:id') + }) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', (e) => { axios .get(`http://localhost:${port}/user/123`) .catch(done) @@ -360,16 +369,18 @@ describe('Plugin', () => { .use(router.routes()) .use(router.allowedMethods()) - agent - .use(traces => { - const spans = sort(traces[0]) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[0]).to.have.property('resource', 'GET /user/:id') - }) - .then(done) - .catch(done) + agent + .use(traces => { + const spans = sort(traces[0]) + + expect(spans[0]).to.have.property('resource', 'GET /user/:id') + }) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', (e) => { axios .get(`http://localhost:${port}/user/123`) .catch(done) @@ -390,16 +401,18 @@ describe('Plugin', () => { app.use(router1.routes()) - agent - .use(traces => { - const spans = sort(traces[0]) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[0]).to.have.property('resource', 'GET /public/plop') - }) - .then(done) - .catch(done) + agent + .use(traces => { + const spans = sort(traces[0]) + + expect(spans[0]).to.have.property('resource', 'GET /public/plop') + }) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', (e) => { axios .get(`http://localhost:${port}/public/plop`) .catch(done) @@ -422,18 +435,20 @@ describe('Plugin', () => { app.use(forums.routes()) - agent - .use(traces => { - const spans = sort(traces[0]) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[0]).to.have.property('resource', 'GET /forums/:fid/discussions/:did/posts/:pid') - expect(spans[0].meta) - .to.have.property('http.url', `http://localhost:${port}/forums/123/discussions/456/posts/789`) - }) - .then(done) - .catch(done) + agent + .use(traces => { + const spans = sort(traces[0]) + + expect(spans[0]).to.have.property('resource', 'GET /forums/:fid/discussions/:did/posts/:pid') + expect(spans[0].meta) + .to.have.property('http.url', `http://localhost:${port}/forums/123/discussions/456/posts/789`) + }) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', () => { axios .get(`http://localhost:${port}/forums/123/discussions/456/posts/789`) .catch(done) @@ -457,18 +472,20 @@ describe('Plugin', () => { app.use(first.routes()) app.use(second.routes()) - agent - .use(traces => { - const spans = sort(traces[0]) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[0]).to.have.property('resource', 'GET /first/child') - expect(spans[0].meta) - .to.have.property('http.url', `http://localhost:${port}/first/child`) - }) - .then(done) - .catch(done) + agent + .use(traces => { + const spans = sort(traces[0]) + + expect(spans[0]).to.have.property('resource', 'GET /first/child') + expect(spans[0].meta) + .to.have.property('http.url', `http://localhost:${port}/first/child`) + }) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', () => { axios .get(`http://localhost:${port}/first/child`) .catch(done) @@ -492,17 +509,19 @@ describe('Plugin', () => { app.use(forums.routes()) - agent - .use(traces => { - const spans = sort(traces[0]) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[0]).to.have.property('resource', 'GET /forums/:fid/posts/:pid') - expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/forums/123/posts/456`) - }) - .then(done) - .catch(done) + agent + .use(traces => { + const spans = sort(traces[0]) + + expect(spans[0]).to.have.property('resource', 'GET /forums/:fid/posts/:pid') + expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/forums/123/posts/456`) + }) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', () => { axios .get(`http://localhost:${port}/forums/123/posts/456`) .catch(done) @@ -523,17 +542,19 @@ describe('Plugin', () => { .use(router.routes()) .use(router.allowedMethods()) - agent - .use(traces => { - const spans = sort(traces[0]) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[0]).to.have.property('resource', 'GET /user/:id') - expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user/123`) - }) - .then(done) - .catch(done) + agent + .use(traces => { + const spans = sort(traces[0]) + + expect(spans[0]).to.have.property('resource', 'GET /user/:id') + expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user/123`) + }) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', () => { axios .get(`http://localhost:${port}/user/123`) .catch(done) @@ -556,26 +577,28 @@ describe('Plugin', () => { .use(router.routes()) .use(router.allowedMethods()) - agent - .use(traces => { - const spans = sort(traces[0]) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[0]).to.have.property('resource', 'GET /user/:id') - expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user/123`) - expect(spans[0].error).to.equal(1) + agent + .use(traces => { + const spans = sort(traces[0]) + + expect(spans[0]).to.have.property('resource', 'GET /user/:id') + expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user/123`) + expect(spans[0].error).to.equal(1) - expect(spans[1]).to.have.property('resource') - expect(spans[1].resource).to.match(/^dispatch/) - expect(spans[1].meta).to.include({ - [ERROR_TYPE]: error.name, - 'component': 'koa' + expect(spans[1]).to.have.property('resource') + expect(spans[1].resource).to.match(/^dispatch/) + expect(spans[1].meta).to.include({ + [ERROR_TYPE]: error.name, + component: 'koa' + }) + expect(spans[1].error).to.equal(1) }) - expect(spans[1].error).to.equal(1) - }) - .then(done) - .catch(done) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', () => { axios .get(`http://localhost:${port}/user/123`) .catch(() => {}) @@ -588,7 +611,7 @@ describe('Plugin', () => { let ws beforeEach(() => { - WebSocket = require(`../../../versions/ws@6.1.0`).get() + WebSocket = require('../../../versions/ws@6.1.0').get() websockify = require(`../../../versions/koa-websocket@${wsVersion}`).get() }) @@ -609,7 +632,9 @@ describe('Plugin', () => { .use(router.routes()) .use(router.allowedMethods()) - appListener = app.listen(port, 'localhost', () => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + ws = new WebSocket(`ws://localhost:${port}/message`) ws.on('error', done) ws.on('open', () => { @@ -628,6 +653,7 @@ describe('Plugin', () => { describe('with configuration', () => { before(() => agent.load(['koa', 'http'], [{ middleware: false }, { client: false }])) + after(() => agent.close({ ritmReset: false })) describe('middleware set to false', () => { @@ -638,26 +664,28 @@ describe('Plugin', () => { ctx.body = '' }) - agent - .use(traces => { - const spans = sort(traces[0]) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[0]).to.have.property('name', 'koa.request') - expect(spans[0]).to.have.property('service', 'test') - expect(spans[0]).to.have.property('type', 'web') - expect(spans[0]).to.have.property('resource', 'GET') - expect(spans[0].meta).to.have.property('span.kind', 'server') - expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user`) - expect(spans[0].meta).to.have.property('http.method', 'GET') - expect(spans[0].meta).to.have.property('http.status_code', '200') - expect(spans[0].meta).to.have.property('component', 'koa') + agent + .use(traces => { + const spans = sort(traces[0]) - expect(spans).to.have.length(1) - }) - .then(done) - .catch(done) + expect(spans[0]).to.have.property('name', 'koa.request') + expect(spans[0]).to.have.property('service', 'test') + expect(spans[0]).to.have.property('type', 'web') + expect(spans[0]).to.have.property('resource', 'GET') + expect(spans[0].meta).to.have.property('span.kind', 'server') + expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user`) + expect(spans[0].meta).to.have.property('http.method', 'GET') + expect(spans[0].meta).to.have.property('http.status_code', '200') + expect(spans[0].meta).to.have.property('component', 'koa') + + expect(spans).to.have.length(1) + }) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', () => { axios .get(`http://localhost:${port}/user`) .catch(done) @@ -672,26 +700,28 @@ describe('Plugin', () => { yield next }) - agent - .use(traces => { - const spans = sort(traces[0]) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[0]).to.have.property('name', 'koa.request') - expect(spans[0]).to.have.property('service', 'test') - expect(spans[0]).to.have.property('type', 'web') - expect(spans[0]).to.have.property('resource', 'GET') - expect(spans[0].meta).to.have.property('span.kind', 'server') - expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user`) - expect(spans[0].meta).to.have.property('http.method', 'GET') - expect(spans[0].meta).to.have.property('http.status_code', '200') - expect(spans[0].meta).to.have.property('component', 'koa') + agent + .use(traces => { + const spans = sort(traces[0]) - expect(spans).to.have.length(1) - }) - .then(done) - .catch(done) + expect(spans[0]).to.have.property('name', 'koa.request') + expect(spans[0]).to.have.property('service', 'test') + expect(spans[0]).to.have.property('type', 'web') + expect(spans[0]).to.have.property('resource', 'GET') + expect(spans[0].meta).to.have.property('span.kind', 'server') + expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user`) + expect(spans[0].meta).to.have.property('http.method', 'GET') + expect(spans[0].meta).to.have.property('http.status_code', '200') + expect(spans[0].meta).to.have.property('component', 'koa') + + expect(spans).to.have.length(1) + }) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', () => { axios .get(`http://localhost:${port}/user`) .catch(done) @@ -714,7 +744,9 @@ describe('Plugin', () => { .catch(done) }) - appListener = app.listen(port, 'localhost', () => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + axios .get(`http://localhost:${port}/app/user/123`) .catch(done) @@ -742,11 +774,11 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/user`) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios.get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -770,11 +802,11 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/user`) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios.get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -801,19 +833,21 @@ describe('Plugin', () => { .use(router.routes()) .use(router.allowedMethods()) - agent - .use(traces => { - const spans = sort(traces[0]) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[0]).to.have.property('resource', 'GET /user/:id') - expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user/123`) - expect(spans[0].error).to.equal(1) - expect(spans[0].meta).to.have.property('component', 'koa') - }) - .then(done) - .catch(done) + agent + .use(traces => { + const spans = sort(traces[0]) + + expect(spans[0]).to.have.property('resource', 'GET /user/:id') + expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user/123`) + expect(spans[0].error).to.equal(1) + expect(spans[0].meta).to.have.property('component', 'koa') + }) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', () => { axios .get(`http://localhost:${port}/user/123`) .catch(() => {}) diff --git a/packages/datadog-plugin-koa/test/integration-test/client.spec.js b/packages/datadog-plugin-koa/test/integration-test/client.spec.js index 45e8c7344a9..2216dd6129e 100644 --- a/packages/datadog-plugin-koa/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-koa/test/integration-test/client.spec.js @@ -17,7 +17,7 @@ describe('esm', () => { before(async function () { this.timeout(50000) sandbox = await createSandbox([`'koa@${version}'`], false, - [`./packages/datadog-plugin-koa/test/integration-test/*`]) + ['./packages/datadog-plugin-koa/test/integration-test/*']) }) after(async function () { diff --git a/packages/datadog-plugin-limitd-client/test/integration-test/client.spec.js b/packages/datadog-plugin-limitd-client/test/integration-test/client.spec.js index 7cc554bf9a1..5ed8cd12c9d 100644 --- a/packages/datadog-plugin-limitd-client/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-limitd-client/test/integration-test/client.spec.js @@ -17,7 +17,7 @@ describe('esm', () => { before(async function () { this.timeout(20000) sandbox = await createSandbox([`'limitd-client@${version}'`], false, [ - `./packages/datadog-plugin-limitd-client/test/integration-test/*`]) + './packages/datadog-plugin-limitd-client/test/integration-test/*']) }) after(async () => { diff --git a/packages/datadog-plugin-mariadb/test/index.spec.js b/packages/datadog-plugin-mariadb/test/index.spec.js index 967a5c9dc52..65828c17b25 100644 --- a/packages/datadog-plugin-mariadb/test/index.spec.js +++ b/packages/datadog-plugin-mariadb/test/index.spec.js @@ -67,6 +67,7 @@ describe('Plugin', () => { tracer.scope().activate(span, () => { const span = tracer.scope().active() + // eslint-disable-next-line n/handle-callback-err connection.query('SELECT 1 + 1 AS solution', (err, results, fields) => { try { expect(results).to.not.be.null diff --git a/packages/datadog-plugin-mariadb/test/integration-test/client.spec.js b/packages/datadog-plugin-mariadb/test/integration-test/client.spec.js index d41905c29dd..8ff3272fba9 100644 --- a/packages/datadog-plugin-mariadb/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-mariadb/test/integration-test/client.spec.js @@ -18,7 +18,7 @@ describe('esm', () => { before(async function () { this.timeout(20000) sandbox = await createSandbox([`'mariadb@${version}'`], false, [ - `./packages/datadog-plugin-mariadb/test/integration-test/*`]) + './packages/datadog-plugin-mariadb/test/integration-test/*']) }) after(async () => { diff --git a/packages/datadog-plugin-mariadb/test/naming.js b/packages/datadog-plugin-mariadb/test/naming.js index 2f01944c66f..81bc7d60f0d 100644 --- a/packages/datadog-plugin-mariadb/test/naming.js +++ b/packages/datadog-plugin-mariadb/test/naming.js @@ -14,6 +14,6 @@ const rawExpectedSchema = { } module.exports = { - rawExpectedSchema: rawExpectedSchema, + rawExpectedSchema, expectedSchema: resolveNaming(rawExpectedSchema) } diff --git a/packages/datadog-plugin-memcached/test/index.spec.js b/packages/datadog-plugin-memcached/test/index.spec.js index 4b9a3c452f2..1751a7067c6 100644 --- a/packages/datadog-plugin-memcached/test/index.spec.js +++ b/packages/datadog-plugin-memcached/test/index.spec.js @@ -32,6 +32,7 @@ describe('Plugin', () => { 'localhost', 'out.host' ) + it('should do automatic instrumentation when using callbacks', done => { memcached = new Memcached('localhost:11211', { retries: 0 }) diff --git a/packages/datadog-plugin-memcached/test/integration-test/client.spec.js b/packages/datadog-plugin-memcached/test/integration-test/client.spec.js index 277e330f9d0..78012aa8b3a 100644 --- a/packages/datadog-plugin-memcached/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-memcached/test/integration-test/client.spec.js @@ -17,7 +17,7 @@ describe('esm', () => { before(async function () { this.timeout(50000) sandbox = await createSandbox([`'memcached@${version}'`], false, [ - `./packages/datadog-plugin-memcached/test/integration-test/*`]) + './packages/datadog-plugin-memcached/test/integration-test/*']) }) after(async function () { diff --git a/packages/datadog-plugin-memcached/test/leak.js b/packages/datadog-plugin-memcached/test/leak.js deleted file mode 100644 index 9fde9661bc8..00000000000 --- a/packages/datadog-plugin-memcached/test/leak.js +++ /dev/null @@ -1,19 +0,0 @@ -'use strict' - -require('../../dd-trace') - .init({ plugins: false, sampleRate: 0 }) - .use('memcached') - -const test = require('tape') -const Memcached = require('../../../versions/memcached').get() -const profile = require('../../dd-trace/test/profile') - -test('memcached plugin should not leak', t => { - const memcached = new Memcached('localhost:11211', { retries: 0 }) - - profile(t, operation).then(() => memcached.end()) - - function operation (done) { - memcached.get('foo', done) - } -}) diff --git a/packages/datadog-plugin-memcached/test/naming.js b/packages/datadog-plugin-memcached/test/naming.js index a92fd57a43e..80a164f42d4 100644 --- a/packages/datadog-plugin-memcached/test/naming.js +++ b/packages/datadog-plugin-memcached/test/naming.js @@ -14,6 +14,6 @@ const rawExpectedSchema = { } module.exports = { - rawExpectedSchema: rawExpectedSchema, + rawExpectedSchema, expectedSchema: resolveNaming(rawExpectedSchema) } diff --git a/packages/datadog-plugin-microgateway-core/test/index.spec.js b/packages/datadog-plugin-microgateway-core/test/index.spec.js index c6beea05ec6..1b76c947122 100644 --- a/packages/datadog-plugin-microgateway-core/test/index.spec.js +++ b/packages/datadog-plugin-microgateway-core/test/index.spec.js @@ -2,7 +2,6 @@ const axios = require('axios') const http = require('http') -const getPort = require('get-port') const os = require('os') const semver = require('semver') const agent = require('../../dd-trace/test/plugins/agent') @@ -22,7 +21,11 @@ describe('Plugin', () => { const api = http.createServer((req, res) => res.end('OK')) api.listen(apiPort, function () { + const apiPort = api.address().port + proxy.listen(proxyPort, function () { + const proxyPort = proxy.address().port + gateway = Gateway({ edgemicro: { port: gatewayPort, @@ -34,7 +37,10 @@ describe('Plugin', () => { ] }) - gateway.start(cb) + gateway.start((err, server) => { + gatewayPort = server.address().port + cb(err) + }) }) }) } @@ -47,12 +53,6 @@ describe('Plugin', () => { describe('microgateway-core', () => { withVersions('microgateway-core', 'microgateway-core', (version) => { - beforeEach(async () => { - gatewayPort = await getPort() - proxyPort = await getPort() - apiPort = await getPort() - }) - afterEach(() => { stopGateway() }) diff --git a/packages/datadog-plugin-microgateway-core/test/integration-test/client.spec.js b/packages/datadog-plugin-microgateway-core/test/integration-test/client.spec.js index 33a27585833..ad54a7e0ad2 100644 --- a/packages/datadog-plugin-microgateway-core/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-microgateway-core/test/integration-test/client.spec.js @@ -23,7 +23,7 @@ describe('esm', () => { before(async function () { this.timeout(20000) sandbox = await createSandbox([`'microgateway-core@${version}'`, 'get-port'], false, [ - `./packages/datadog-plugin-microgateway-core/test/integration-test/*`]) + './packages/datadog-plugin-microgateway-core/test/integration-test/*']) }) after(async () => { diff --git a/packages/datadog-plugin-microgateway-core/test/proxy.js b/packages/datadog-plugin-microgateway-core/test/proxy.js index 42e73b340de..dd9f0fdad4f 100644 --- a/packages/datadog-plugin-microgateway-core/test/proxy.js +++ b/packages/datadog-plugin-microgateway-core/test/proxy.js @@ -36,6 +36,7 @@ module.exports = http.createServer((req, res) => { cltSocket.pipe(targetConnection) } + // eslint-disable-next-line n/no-deprecated-api const targetUrl = url.parse(util.format('%s://%s', proto, req.url)) let targetConnection diff --git a/packages/datadog-plugin-mocha/src/index.js b/packages/datadog-plugin-mocha/src/index.js index c8af76247b1..0513a4a95d6 100644 --- a/packages/datadog-plugin-mocha/src/index.js +++ b/packages/datadog-plugin-mocha/src/index.js @@ -13,9 +13,53 @@ const { addIntelligentTestRunnerSpanTags, TEST_SOURCE_START, TEST_ITR_UNSKIPPABLE, - TEST_ITR_FORCED_RUN + TEST_ITR_FORCED_RUN, + TEST_CODE_OWNERS, + ITR_CORRELATION_ID, + TEST_SOURCE_FILE, + removeEfdStringFromTestName, + TEST_IS_NEW, + TEST_IS_RETRY, + TEST_EARLY_FLAKE_ENABLED, + TEST_EARLY_FLAKE_ABORT_REASON, + TEST_SESSION_ID, + TEST_MODULE_ID, + TEST_MODULE, + TEST_SUITE_ID, + TEST_COMMAND, + TEST_SUITE, + MOCHA_IS_PARALLEL, + TEST_IS_RUM_ACTIVE, + TEST_BROWSER_DRIVER } = require('../../dd-trace/src/plugins/util/test') const { COMPONENT } = require('../../dd-trace/src/constants') +const { + TELEMETRY_EVENT_CREATED, + TELEMETRY_EVENT_FINISHED, + TELEMETRY_CODE_COVERAGE_STARTED, + TELEMETRY_CODE_COVERAGE_FINISHED, + TELEMETRY_ITR_FORCED_TO_RUN, + TELEMETRY_CODE_COVERAGE_EMPTY, + TELEMETRY_ITR_UNSKIPPABLE, + TELEMETRY_CODE_COVERAGE_NUM_FILES, + TELEMETRY_TEST_SESSION +} = require('../../dd-trace/src/ci-visibility/telemetry') +const id = require('../../dd-trace/src/id') +const log = require('../../dd-trace/src/log') + +function getTestSuiteLevelVisibilityTags (testSuiteSpan) { + const testSuiteSpanContext = testSuiteSpan.context() + const suiteTags = { + [TEST_SUITE_ID]: testSuiteSpanContext.toSpanId(), + [TEST_SESSION_ID]: testSuiteSpanContext.toTraceId(), + [TEST_COMMAND]: testSuiteSpanContext._tags[TEST_COMMAND], + [TEST_MODULE]: 'mocha' + } + if (testSuiteSpanContext._parentId) { + suiteTags[TEST_MODULE_ID] = testSuiteSpanContext._parentId.toString(10) + } + return suiteTags +} class MochaPlugin extends CiPlugin { static get id () { @@ -26,14 +70,19 @@ class MochaPlugin extends CiPlugin { super(...args) this._testSuites = new Map() - this._testNameToParams = {} + this._testTitleToParams = {} this.sourceRoot = process.cwd() this.addSub('ci:mocha:test-suite:code-coverage', ({ coverageFiles, suiteFile }) => { - if (!this.itrConfig || !this.itrConfig.isCodeCoverageEnabled) { + if (!this.libraryConfig?.isCodeCoverageEnabled) { return } - const testSuiteSpan = this._testSuites.get(suiteFile) + const testSuite = getTestSuitePath(suiteFile, this.sourceRoot) + const testSuiteSpan = this._testSuites.get(testSuite) + + if (!coverageFiles.length) { + this.telemetry.count(TELEMETRY_CODE_COVERAGE_EMPTY) + } const relativeCoverageFiles = [...coverageFiles, suiteFile] .map(filename => getTestSuitePath(filename, this.sourceRoot)) @@ -47,21 +96,47 @@ class MochaPlugin extends CiPlugin { } this.tracer._exporter.exportCoverage(formattedCoverage) + this.telemetry.ciVisEvent(TELEMETRY_CODE_COVERAGE_FINISHED, 'suite', { library: 'istanbul' }) + this.telemetry.distribution(TELEMETRY_CODE_COVERAGE_NUM_FILES, {}, relativeCoverageFiles.length) }) - this.addSub('ci:mocha:test-suite:start', ({ testSuite, isUnskippable, isForcedToRun }) => { - const store = storage.getStore() + this.addSub('ci:mocha:test-suite:start', ({ + testSuiteAbsolutePath, + isUnskippable, + isForcedToRun, + itrCorrelationId + }) => { + // If the test module span is undefined, the plugin has not been initialized correctly and we bail out + if (!this.testModuleSpan) { + return + } + const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.sourceRoot) const testSuiteMetadata = getTestSuiteCommonTags( this.command, this.frameworkVersion, - getTestSuitePath(testSuite, this.sourceRoot), + testSuite, 'mocha' ) if (isUnskippable) { testSuiteMetadata[TEST_ITR_UNSKIPPABLE] = 'true' + this.telemetry.count(TELEMETRY_ITR_UNSKIPPABLE, { testLevel: 'suite' }) } if (isForcedToRun) { testSuiteMetadata[TEST_ITR_FORCED_RUN] = 'true' + this.telemetry.count(TELEMETRY_ITR_FORCED_TO_RUN, { testLevel: 'suite' }) + } + if (this.repositoryRoot !== this.sourceRoot && !!this.repositoryRoot) { + testSuiteMetadata[TEST_SOURCE_FILE] = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) + } else { + testSuiteMetadata[TEST_SOURCE_FILE] = testSuite + } + if (testSuiteMetadata[TEST_SOURCE_FILE]) { + testSuiteMetadata[TEST_SOURCE_START] = 1 + } + + const codeOwners = this.getCodeOwners(testSuiteMetadata) + if (codeOwners) { + testSuiteMetadata[TEST_CODE_OWNERS] = codeOwners } const testSuiteSpan = this.tracer.startSpan('mocha.test_suite', { @@ -72,6 +147,14 @@ class MochaPlugin extends CiPlugin { ...testSuiteMetadata } }) + this.telemetry.ciVisEvent(TELEMETRY_EVENT_CREATED, 'suite') + if (this.libraryConfig?.isCodeCoverageEnabled) { + this.telemetry.ciVisEvent(TELEMETRY_CODE_COVERAGE_STARTED, 'suite', { library: 'istanbul' }) + } + if (itrCorrelationId) { + testSuiteSpan.setTag(ITR_CORRELATION_ID, itrCorrelationId) + } + const store = storage.getStore() this.enter(testSuiteSpan, store) this._testSuites.set(testSuite, testSuiteSpan) }) @@ -85,6 +168,7 @@ class MochaPlugin extends CiPlugin { span.setTag(TEST_STATUS, status) } span.finish() + this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'suite') } }) @@ -97,40 +181,58 @@ class MochaPlugin extends CiPlugin { } }) - this.addSub('ci:mocha:test:start', ({ test, testStartLine }) => { + this.addSub('ci:mocha:test:start', (testInfo) => { const store = storage.getStore() - const span = this.startTestSpan(test, testStartLine) + const span = this.startTestSpan(testInfo) this.enter(span, store) }) - this.addSub('ci:mocha:test:finish', (status) => { - const store = storage.getStore() + this.addSub('ci:mocha:worker:finish', () => { + this.tracer._exporter.flush() + }) - if (store && store.span) { - const span = store.span + this.addSub('ci:mocha:test:finish', ({ status, hasBeenRetried }) => { + const store = storage.getStore() + const span = store?.span + if (span) { span.setTag(TEST_STATUS, status) + if (hasBeenRetried) { + span.setTag(TEST_IS_RETRY, 'true') + } + + const spanTags = span.context()._tags + this.telemetry.ciVisEvent( + TELEMETRY_EVENT_FINISHED, + 'test', + { + hasCodeOwners: !!spanTags[TEST_CODE_OWNERS], + isNew: spanTags[TEST_IS_NEW] === 'true', + isRum: spanTags[TEST_IS_RUM_ACTIVE] === 'true', + browserDriver: spanTags[TEST_BROWSER_DRIVER] + } + ) span.finish() finishAllTraceSpans(span) } }) - this.addSub('ci:mocha:test:skip', (test) => { + this.addSub('ci:mocha:test:skip', (testInfo) => { const store = storage.getStore() // skipped through it.skip, so the span is not created yet // for this test if (!store) { - const testSpan = this.startTestSpan(test) + const testSpan = this.startTestSpan(testInfo) this.enter(testSpan, store) } }) this.addSub('ci:mocha:test:error', (err) => { const store = storage.getStore() - if (err && store && store.span) { - const span = store.span + const span = store?.span + if (err && span) { if (err.constructor.name === 'Pending' && !this.forbidPending) { span.setTag(TEST_STATUS, 'skip') } else { @@ -140,8 +242,37 @@ class MochaPlugin extends CiPlugin { } }) - this.addSub('ci:mocha:test:parameterize', ({ name, params }) => { - this._testNameToParams[name] = params + this.addSub('ci:mocha:test:retry', ({ isFirstAttempt, err }) => { + const store = storage.getStore() + const span = store?.span + if (span) { + span.setTag(TEST_STATUS, 'fail') + if (!isFirstAttempt) { + span.setTag(TEST_IS_RETRY, 'true') + } + if (err) { + span.setTag('error', err) + } + + const spanTags = span.context()._tags + this.telemetry.ciVisEvent( + TELEMETRY_EVENT_FINISHED, + 'test', + { + hasCodeOwners: !!spanTags[TEST_CODE_OWNERS], + isNew: spanTags[TEST_IS_NEW] === 'true', + isRum: spanTags[TEST_IS_RUM_ACTIVE] === 'true', + browserDriver: spanTags[TEST_BROWSER_DRIVER] + } + ) + + span.finish() + finishAllTraceSpans(span) + } + }) + + this.addSub('ci:mocha:test:parameterize', ({ title, params }) => { + this._testTitleToParams[title] = params }) this.addSub('ci:mocha:session:finish', ({ @@ -151,10 +282,13 @@ class MochaPlugin extends CiPlugin { numSkippedSuites, hasForcedToRunSuites, hasUnskippableSuites, - error + error, + isEarlyFlakeDetectionEnabled, + isEarlyFlakeDetectionFaulty, + isParallel }) => { if (this.testSessionSpan) { - const { isSuitesSkippingEnabled, isCodeCoverageEnabled } = this.itrConfig || {} + const { isSuitesSkippingEnabled, isCodeCoverageEnabled } = this.libraryConfig || {} this.testSessionSpan.setTag(TEST_STATUS, status) this.testModuleSpan.setTag(TEST_STATUS, status) @@ -163,6 +297,10 @@ class MochaPlugin extends CiPlugin { this.testModuleSpan.setTag('error', error) } + if (isParallel) { + this.testSessionSpan.setTag(MOCHA_IS_PARALLEL, 'true') + } + addIntelligentTestRunnerSpanTags( this.testSessionSpan, this.testModuleSpan, @@ -178,21 +316,70 @@ class MochaPlugin extends CiPlugin { } ) + if (isEarlyFlakeDetectionEnabled) { + this.testSessionSpan.setTag(TEST_EARLY_FLAKE_ENABLED, 'true') + } + if (isEarlyFlakeDetectionFaulty) { + this.testSessionSpan.setTag(TEST_EARLY_FLAKE_ABORT_REASON, 'faulty') + } + this.testModuleSpan.finish() + this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'module') this.testSessionSpan.finish() + this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'session') finishAllTraceSpans(this.testSessionSpan) + this.telemetry.count(TELEMETRY_TEST_SESSION, { provider: this.ciProviderName }) } - this.itrConfig = null + this.libraryConfig = null this.tracer._exporter.flush() }) + + this.addSub('ci:mocha:worker-report:trace', (traces) => { + const formattedTraces = JSON.parse(traces).map(trace => + trace.map(span => { + const formattedSpan = { + ...span, + span_id: id(span.span_id), + trace_id: id(span.trace_id), + parent_id: id(span.parent_id) + } + if (formattedSpan.name === 'mocha.test') { + const testSuite = span.meta[TEST_SUITE] + const testSuiteSpan = this._testSuites.get(testSuite) + if (!testSuiteSpan) { + log.warn(`Test suite span not found for test span with test suite ${testSuite}`) + return formattedSpan + } + const suiteTags = getTestSuiteLevelVisibilityTags(testSuiteSpan) + formattedSpan.meta = { + ...formattedSpan.meta, + ...suiteTags + } + } + return formattedSpan + }) + ) + + formattedTraces.forEach(trace => { + this.tracer._exporter.export(trace) + }) + }) } - startTestSpan (test, testStartLine) { - const testName = test.fullTitle() - const { file: testSuiteAbsolutePath, title } = test + startTestSpan (testInfo) { + const { + testSuiteAbsolutePath, + title, + isNew, + isEfdRetry, + testStartLine, + isParallel + } = testInfo + + const testName = removeEfdStringFromTestName(testInfo.testName) const extraTags = {} - const testParametersString = getTestParametersString(this._testNameToParams, title) + const testParametersString = getTestParametersString(this._testTitleToParams, title) if (testParametersString) { extraTags[TEST_PARAMETERS] = testParametersString } @@ -201,8 +388,25 @@ class MochaPlugin extends CiPlugin { extraTags[TEST_SOURCE_START] = testStartLine } + if (isParallel) { + extraTags[MOCHA_IS_PARALLEL] = 'true' + } + const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.sourceRoot) - const testSuiteSpan = this._testSuites.get(testSuiteAbsolutePath) + const testSuiteSpan = this._testSuites.get(testSuite) + + if (this.repositoryRoot !== this.sourceRoot && !!this.repositoryRoot) { + extraTags[TEST_SOURCE_FILE] = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) + } else { + extraTags[TEST_SOURCE_FILE] = testSuite + } + + if (isNew) { + extraTags[TEST_IS_NEW] = 'true' + if (isEfdRetry) { + extraTags[TEST_IS_RETRY] = 'true' + } + } return super.startTestSpan(testName, testSuite, testSuiteSpan, extraTags) } diff --git a/packages/datadog-plugin-mocha/test/index.spec.js b/packages/datadog-plugin-mocha/test/index.spec.js index ce35816b5bd..aef1c7555a2 100644 --- a/packages/datadog-plugin-mocha/test/index.spec.js +++ b/packages/datadog-plugin-mocha/test/index.spec.js @@ -4,6 +4,7 @@ const path = require('path') const fs = require('fs') const nock = require('nock') +const semver = require('semver') const agent = require('../../dd-trace/test/plugins/agent') const { ORIGIN_KEY, COMPONENT, ERROR_MESSAGE, ERROR_STACK, ERROR_TYPE } = require('../../dd-trace/src/constants') @@ -77,7 +78,7 @@ const ASYNC_TESTS = [ describe('Plugin', () => { let Mocha - withVersions('mocha', 'mocha', version => { + withVersions('mocha', 'mocha', (version, _, specificVersion) => { afterEach(() => { // This needs to be done when using the programmatic API: // https://github.com/mochajs/mocha/wiki/Using-Mocha-programmatically @@ -142,6 +143,7 @@ describe('Plugin', () => { mocha.addFile(testFilePath) mocha.run() }) + it('works with failing tests', (done) => { const testFilePath = path.join(__dirname, 'mocha-test-fail.js') const testSuite = testFilePath.replace(`${process.cwd()}/`, '') @@ -178,6 +180,7 @@ describe('Plugin', () => { mocha.addFile(testFilePath) mocha.run() }) + it('works with skipping tests', (done) => { const testFilePath = path.join(__dirname, 'mocha-test-skip.js') const testNames = [ @@ -323,12 +326,12 @@ describe('Plugin', () => { }) expect(testSpan.meta[COMPONENT]).to.equal('mocha') expect(testSpan.meta[ERROR_TYPE]).to.equal('TypeError') - const beginning = `mocha-fail-hook-sync "before each" hook for "will not run but be reported as failed": ` + const beginning = 'mocha-fail-hook-sync "before each" hook for "will not run but be reported as failed": ' expect(testSpan.meta[ERROR_MESSAGE].startsWith(beginning)).to.equal(true) const errorMsg = testSpan.meta[ERROR_MESSAGE].replace(beginning, '') expect( - errorMsg === `Cannot set property 'error' of undefined` || - errorMsg === `Cannot set properties of undefined (setting 'error')` + errorMsg === 'Cannot set property \'error\' of undefined' || + errorMsg === 'Cannot set properties of undefined (setting \'error\')' ).to.equal(true) expect(testSpan.meta[ERROR_STACK]).not.to.be.undefined }).then(done, done) @@ -447,13 +450,27 @@ describe('Plugin', () => { }) it('works with retries', (done) => { + let testNames = [] + // retry listener did not happen until 6.0.0 + if (semver.satisfies(specificVersion, '>=6.0.0')) { + testNames = [ + ['mocha-test-retries will be retried and pass', 'fail'], + ['mocha-test-retries will be retried and pass', 'fail'], + ['mocha-test-retries will be retried and pass', 'pass'], + ['mocha-test-retries will be retried and fail', 'fail'], + ['mocha-test-retries will be retried and fail', 'fail'], + ['mocha-test-retries will be retried and fail', 'fail'], + ['mocha-test-retries will be retried and fail', 'fail'], + ['mocha-test-retries will be retried and fail', 'fail'] + ] + } else { + testNames = [ + ['mocha-test-retries will be retried and pass', 'pass'], + ['mocha-test-retries will be retried and fail', 'fail'] + ] + } const testFilePath = path.join(__dirname, 'mocha-test-retries.js') - const testNames = [ - ['mocha-test-retries will be retried and pass', 'pass'], - ['mocha-test-retries will be retried and fail', 'fail'] - ] - const assertionPromises = testNames.map(([testName, status]) => { return agent.use(trace => { const testSpan = trace[0][0] @@ -463,15 +480,15 @@ describe('Plugin', () => { }) }) - Promise.all(assertionPromises) - .then(() => done()) - .catch(done) - const mocha = new Mocha({ reporter: function () {} // silent on internal tests }) mocha.addFile(testFilePath) - mocha.run() + mocha.run().on('end', () => { + Promise.all(assertionPromises) + .then(() => done()) + .catch(done) + }) }) it('works when skipping suites', function (done) { diff --git a/packages/datadog-plugin-mocha/test/mocha-active-span-in-hooks.js b/packages/datadog-plugin-mocha/test/mocha-active-span-in-hooks.js index da145328a6e..e5382e3de43 100644 --- a/packages/datadog-plugin-mocha/test/mocha-active-span-in-hooks.js +++ b/packages/datadog-plugin-mocha/test/mocha-active-span-in-hooks.js @@ -6,18 +6,23 @@ describe('mocha-active-span-in-hooks', function () { before(() => { expect(global._ddtrace.scope().active()).to.equal(null) }) + after(() => { expect(global._ddtrace.scope().active()).to.equal(null) }) + beforeEach(() => { currentTestTraceId = global._ddtrace.scope().active().context().toTraceId() }) + afterEach(() => { expect(currentTestTraceId).to.equal(global._ddtrace.scope().active().context().toTraceId()) }) + it('first test', () => { expect(currentTestTraceId).to.equal(global._ddtrace.scope().active().context().toTraceId()) }) + it('second test', () => { expect(currentTestTraceId).to.equal(global._ddtrace.scope().active().context().toTraceId()) }) diff --git a/packages/datadog-plugin-mocha/test/mocha-fail-hook-async.js b/packages/datadog-plugin-mocha/test/mocha-fail-hook-async.js index 9e3cf900a6f..9f2e4aa7bba 100644 --- a/packages/datadog-plugin-mocha/test/mocha-fail-hook-async.js +++ b/packages/datadog-plugin-mocha/test/mocha-fail-hook-async.js @@ -4,6 +4,7 @@ describe('mocha-fail-before-all', function () { before((done) => { done(new Error('this should not stop execution')) }) + it('will not be reported because it will not run', () => { expect(true).to.equal(true) }) @@ -15,6 +16,7 @@ describe('mocha-fail-hook-async', function () { done(new Error('yeah error')) }, 200) }) + it('will run but be reported as failed', () => { expect(true).to.equal(true) }) @@ -24,6 +26,7 @@ describe('mocha-fail-hook-async-other', function () { afterEach((done) => { done() }) + it('will run and be reported as passed', () => { expect(true).to.equal(true) }) @@ -35,6 +38,7 @@ describe('mocha-fail-hook-async-other-before', function () { done(new Error('yeah error')) }, 200) }) + it('will not run and be reported as failed', () => { expect(true).to.equal(true) }) @@ -44,12 +48,14 @@ describe('mocha-fail-hook-async-other-second-after', function () { afterEach((done) => { done() }) + afterEach((done) => { // the second afterEach will fail setTimeout(() => { done(new Error('yeah error')) }, 200) }) + it('will run and be reported as failed', () => { expect(true).to.equal(true) }) @@ -59,6 +65,7 @@ describe('mocha-fail-test-after-each-passes', function () { afterEach((done) => { done() }) + it('will fail and be reported as failed', () => { expect(true).to.equal(false) }) diff --git a/packages/datadog-plugin-mocha/test/mocha-fail-hook-sync.js b/packages/datadog-plugin-mocha/test/mocha-fail-hook-sync.js index 25f9e88be42..18f618b3578 100644 --- a/packages/datadog-plugin-mocha/test/mocha-fail-hook-sync.js +++ b/packages/datadog-plugin-mocha/test/mocha-fail-hook-sync.js @@ -5,6 +5,7 @@ describe('mocha-fail-hook-sync', () => { const value = '' value.unsafe.error = '' }) + it('will not run but be reported as failed', () => { expect(true).to.equal(true) }) diff --git a/packages/datadog-plugin-mocha/test/mocha-test-pass.js b/packages/datadog-plugin-mocha/test/mocha-test-pass.js index cce444190aa..6d40be6f5a6 100644 --- a/packages/datadog-plugin-mocha/test/mocha-test-pass.js +++ b/packages/datadog-plugin-mocha/test/mocha-test-pass.js @@ -4,6 +4,7 @@ describe('mocha-test-pass', () => { it('can pass', () => { expect(true).to.equal(true) }) + it('can pass two', () => { expect(true).to.equal(true) }) @@ -13,6 +14,7 @@ describe('mocha-test-pass-two', () => { it('can pass', () => { expect(true).to.equal(true) }) + it('can pass two', () => { expect(true).to.equal(true) }) diff --git a/packages/datadog-plugin-mocha/test/mocha-test-retries.js b/packages/datadog-plugin-mocha/test/mocha-test-retries.js index 760b8734c7f..6417f21a4f0 100644 --- a/packages/datadog-plugin-mocha/test/mocha-test-retries.js +++ b/packages/datadog-plugin-mocha/test/mocha-test-retries.js @@ -3,9 +3,11 @@ const { expect } = require('chai') let attempt = 0 describe('mocha-test-retries', function () { this.retries(4) + it('will be retried and pass', () => { expect(attempt++).to.equal(2) }) + it('will be retried and fail', () => { expect(attempt++).to.equal(8) }) diff --git a/packages/datadog-plugin-mocha/test/mocha-test-skip-describe.js b/packages/datadog-plugin-mocha/test/mocha-test-skip-describe.js index 27f6dfde738..1e0f4228337 100644 --- a/packages/datadog-plugin-mocha/test/mocha-test-skip-describe.js +++ b/packages/datadog-plugin-mocha/test/mocha-test-skip-describe.js @@ -4,6 +4,7 @@ describe('mocha-test-skip-describe', () => { before(function () { this.skip() }) + it('will be skipped', () => { expect(true).to.equal(true) }) diff --git a/packages/datadog-plugin-mocha/test/mocha-test-suite-level-fail-after-each.js b/packages/datadog-plugin-mocha/test/mocha-test-suite-level-fail-after-each.js index 72fee028782..5e948dde33e 100644 --- a/packages/datadog-plugin-mocha/test/mocha-test-suite-level-fail-after-each.js +++ b/packages/datadog-plugin-mocha/test/mocha-test-suite-level-fail-after-each.js @@ -10,6 +10,7 @@ describe('mocha-test-suite-level-fail', function () { afterEach(() => { throw new Error() }) + it('will pass', () => { expect(2).to.equal(2) }) diff --git a/packages/datadog-plugin-mocha/test/mocha-test-suite-level-fail-skip-describe.js b/packages/datadog-plugin-mocha/test/mocha-test-suite-level-fail-skip-describe.js index 6d20638f18c..ccd960b50a1 100644 --- a/packages/datadog-plugin-mocha/test/mocha-test-suite-level-fail-skip-describe.js +++ b/packages/datadog-plugin-mocha/test/mocha-test-suite-level-fail-skip-describe.js @@ -4,6 +4,7 @@ describe('mocha-test-suite-level-fail', function () { it('will pass', () => { expect(2).to.equal(2) }) + it('will fail', () => { expect(2).to.equal(8) }) diff --git a/packages/datadog-plugin-mocha/test/mocha-test-suite-level-fail-test.js b/packages/datadog-plugin-mocha/test/mocha-test-suite-level-fail-test.js index 2d51e22c52c..50ca4d85f12 100644 --- a/packages/datadog-plugin-mocha/test/mocha-test-suite-level-fail-test.js +++ b/packages/datadog-plugin-mocha/test/mocha-test-suite-level-fail-test.js @@ -4,6 +4,7 @@ describe('mocha-test-suite-level-fail', function () { it('will pass', () => { expect(2).to.equal(2) }) + it('will fail', () => { expect(2).to.equal(8) }) diff --git a/packages/datadog-plugin-moleculer/test/index.spec.js b/packages/datadog-plugin-moleculer/test/index.spec.js index 7cc52c5d437..2b6463619cc 100644 --- a/packages/datadog-plugin-moleculer/test/index.spec.js +++ b/packages/datadog-plugin-moleculer/test/index.spec.js @@ -47,8 +47,11 @@ describe('Plugin', () => { describe('server', () => { describe('without configuration', () => { before(() => agent.load('moleculer', { client: false })) + before(() => startBroker()) + after(() => broker.stop()) + after(() => agent.close({ ritmReset: false })) it('should do automatic instrumentation', done => { @@ -97,8 +100,11 @@ describe('Plugin', () => { params: true, meta: true })) + before(() => startBroker()) + after(() => broker.stop()) + after(() => agent.close({ ritmReset: false })) it('should have the configured service name', done => { @@ -131,6 +137,7 @@ describe('Plugin', () => { let tracer beforeEach(() => startBroker()) + afterEach(() => broker.stop()) beforeEach(done => { @@ -139,6 +146,7 @@ describe('Plugin', () => { .then(done) .catch(done) }) + afterEach(() => agent.close({ ritmReset: false })) withPeerService( @@ -185,8 +193,11 @@ describe('Plugin', () => { params: true, meta: true })) + before(() => startBroker()) + after(() => broker.stop()) + after(() => agent.close({ ritmReset: false })) it('should have the configured service name', done => { @@ -215,8 +226,11 @@ describe('Plugin', () => { describe('client + server (local)', () => { before(() => agent.load('moleculer')) + before(() => startBroker()) + after(() => broker.stop()) + after(() => agent.close({ ritmReset: false })) it('should propagate context', async () => { @@ -254,6 +268,7 @@ describe('Plugin', () => { let clientBroker before(() => agent.load('moleculer')) + before(() => startBroker()) before(function () { @@ -281,7 +296,9 @@ describe('Plugin', () => { }) after(() => clientBroker.stop()) + after(() => broker.stop()) + after(() => agent.close({ ritmReset: false })) it('should propagate context', async () => { diff --git a/packages/datadog-plugin-moleculer/test/integration-test/client.spec.js b/packages/datadog-plugin-moleculer/test/integration-test/client.spec.js index 34e663bfeb8..ba6618863d5 100644 --- a/packages/datadog-plugin-moleculer/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-moleculer/test/integration-test/client.spec.js @@ -21,7 +21,7 @@ describe('esm', () => { before(async function () { this.timeout(20000) sandbox = await createSandbox([`'moleculer@${version}'`, 'get-port'], false, [ - `./packages/datadog-plugin-moleculer/test/integration-test/*`]) + './packages/datadog-plugin-moleculer/test/integration-test/*']) }) after(async () => { diff --git a/packages/datadog-plugin-moleculer/test/naming.js b/packages/datadog-plugin-moleculer/test/naming.js index a19dc44bedf..ee339847550 100644 --- a/packages/datadog-plugin-moleculer/test/naming.js +++ b/packages/datadog-plugin-moleculer/test/naming.js @@ -24,6 +24,6 @@ const rawExpectedSchema = { } module.exports = { - rawExpectedSchema: rawExpectedSchema, + rawExpectedSchema, expectedSchema: resolveNaming(rawExpectedSchema) } diff --git a/packages/datadog-plugin-mongodb-core/src/index.js b/packages/datadog-plugin-mongodb-core/src/index.js index 80d7a58bb56..076d65917b5 100644 --- a/packages/datadog-plugin-mongodb-core/src/index.js +++ b/packages/datadog-plugin-mongodb-core/src/index.js @@ -115,7 +115,7 @@ function limitDepth (input) { } function isObject (val) { - return typeof val === 'object' && val !== null && !(val instanceof Array) + return val !== null && typeof val === 'object' && !Array.isArray(val) } function isBSON (val) { diff --git a/packages/datadog-plugin-mongodb-core/test/core.spec.js b/packages/datadog-plugin-mongodb-core/test/core.spec.js index 31f1c0e6d7e..13a346077cf 100644 --- a/packages/datadog-plugin-mongodb-core/test/core.spec.js +++ b/packages/datadog-plugin-mongodb-core/test/core.spec.js @@ -62,7 +62,7 @@ describe('Plugin', () => { const Server = getServer() server = new Server({ - host: 'localhost', + host: '127.0.0.1', port: 27017, reconnect: false }) @@ -86,7 +86,7 @@ describe('Plugin', () => { expect(span).to.have.property('type', 'mongodb') expect(span.meta).to.have.property('span.kind', 'client') expect(span.meta).to.have.property('db.name', `test.${collection}`) - expect(span.meta).to.have.property('out.host', 'localhost') + expect(span.meta).to.have.property('out.host', '127.0.0.1') expect(span.meta).to.have.property('component', 'mongodb') }) .then(done) @@ -117,7 +117,7 @@ describe('Plugin', () => { .use(traces => { const span = traces[0][0] const resource = `find test.${collection}` - const query = `{"_id":"?"}` + const query = '{"_id":"?"}' expect(span).to.have.property('resource', resource) expect(span.meta).to.have.property('mongodb.query', query) @@ -138,7 +138,7 @@ describe('Plugin', () => { .use(traces => { const span = traces[0][0] const resource = `find test.${collection}` - const query = `{"_id":"9999999999999999999999"}` + const query = '{"_id":"9999999999999999999999"}' expect(span).to.have.property('resource', resource) expect(span.meta).to.have.property('mongodb.query', query) @@ -167,7 +167,7 @@ describe('Plugin', () => { }) it('should stringify BSON objects', done => { - const BSON = require(`../../../versions/bson@4.0.0`).get() + const BSON = require('../../../versions/bson@4.0.0').get() const id = '123456781234567812345678' agent @@ -195,7 +195,7 @@ describe('Plugin', () => { .use(traces => { const span = traces[0][0] const resource = `find test.${collection}` - const query = `{"_id":"1234"}` + const query = '{"_id":"1234"}' expect(span).to.have.property('resource', resource) expect(span.meta).to.have.property('mongodb.query', query) @@ -284,7 +284,7 @@ describe('Plugin', () => { .use(traces => { const span = traces[0][0] const resource = `find test.${collection}` - const query = `{"foo":1,"bar":{"baz":[1,2,3]}}` + const query = '{"foo":1,"bar":{"baz":[1,2,3]}}' expect(span).to.have.property('resource', resource) expect(span.meta).to.have.property('mongodb.query', query) @@ -360,7 +360,7 @@ describe('Plugin', () => { const Server = getServer() server = new Server({ - host: 'localhost', + host: '127.0.0.1', port: 27017, reconnect: false }) diff --git a/packages/datadog-plugin-mongodb-core/test/integration-test/client.spec.js b/packages/datadog-plugin-mongodb-core/test/integration-test/client.spec.js index 950ef6fb7e8..710e3be2fb9 100644 --- a/packages/datadog-plugin-mongodb-core/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-mongodb-core/test/integration-test/client.spec.js @@ -24,7 +24,7 @@ describe('esm', () => { before(async function () { this.timeout(30000) sandbox = await createSandbox([`'mongodb@${version}'`], false, [ - `./packages/datadog-plugin-mongodb-core/test/integration-test/*`]) + './packages/datadog-plugin-mongodb-core/test/integration-test/*']) }) after(async function () { @@ -58,7 +58,7 @@ describe('esm', () => { before(async function () { this.timeout(30000) sandbox = await createSandbox([`'mongodb-core@${version}'`], false, [ - `./packages/datadog-plugin-mongodb-core/test/integration-test/*`]) + './packages/datadog-plugin-mongodb-core/test/integration-test/*']) }) after(async function () { diff --git a/packages/datadog-plugin-mongodb-core/test/leak.js b/packages/datadog-plugin-mongodb-core/test/leak.js deleted file mode 100644 index 427e4bc3f0c..00000000000 --- a/packages/datadog-plugin-mongodb-core/test/leak.js +++ /dev/null @@ -1,29 +0,0 @@ -'use strict' - -require('../../dd-trace') - .init({ plugins: false, sampleRate: 0 }) - .use('mongodb-core') - -const test = require('tape') -const mongo = require('../../../versions/mongodb-core').get() -const profile = require('../../dd-trace/test/profile') - -test('mongodb-core plugin should not leak', t => { - const server = new mongo.Server({ - host: 'localhost', - port: 27017, - reconnect: false - }) - - server.on('connect', () => { - profile(t, operation).then(() => server.destroy()) - }) - - server.on('error', t.fail) - - server.connect() - - function operation (done) { - server.insert(`test.1234`, [{ a: 1 }], {}, done) - } -}) diff --git a/packages/datadog-plugin-mongodb-core/test/mongodb.spec.js b/packages/datadog-plugin-mongodb-core/test/mongodb.spec.js index 3160e2a28df..0e16a3fd71a 100644 --- a/packages/datadog-plugin-mongodb-core/test/mongodb.spec.js +++ b/packages/datadog-plugin-mongodb-core/test/mongodb.spec.js @@ -53,7 +53,7 @@ describe('Plugin', () => { collectionName = id().toString() - BSON = require(`../../../versions/bson@4.0.0`).get() + BSON = require('../../../versions/bson@4.0.0').get() }) afterEach(() => { @@ -108,8 +108,8 @@ describe('Plugin', () => { agent .use(traces => { const span = traces[0][0] - const resource = `planCacheListPlans test.$cmd` - const query = `{}` + const resource = 'planCacheListPlans test.$cmd' + const query = '{}' expect(span).to.have.property('resource', resource) expect(span.meta).to.have.property('mongodb.query', query) @@ -128,7 +128,7 @@ describe('Plugin', () => { .use(traces => { const span = traces[0][0] const resource = `find test.${collectionName}` - const query = `{"_id":"?"}` + const query = '{"_id":"?"}' expect(span).to.have.property('resource', resource) expect(span.meta).to.have.property('mongodb.query', query) @@ -146,7 +146,7 @@ describe('Plugin', () => { .use(traces => { const span = traces[0][0] const resource = `find test.${collectionName}` - const query = `{"_bin":"?"}` + const query = '{"_bin":"?"}' expect(span).to.have.property('resource', resource) expect(span.meta).to.have.property('mongodb.query', query) @@ -184,7 +184,7 @@ describe('Plugin', () => { .use(traces => { const span = traces[0][0] const resource = `find test.${collectionName}` - const query = `{"_time":{"$timestamp":"0"}}` + const query = '{"_time":{"$timestamp":"0"}}' expect(span).to.have.property('resource', resource) expect(span.meta).to.have.property('mongodb.query', query) @@ -202,7 +202,7 @@ describe('Plugin', () => { .use(traces => { const span = traces[0][0] const resource = `find test.${collectionName}` - const query = `{"_id":"?"}` + const query = '{"_id":"?"}' expect(span).to.have.property('resource', resource) expect(span.meta).to.have.property('mongodb.query', query) @@ -220,7 +220,7 @@ describe('Plugin', () => { .use(traces => { const span = traces[0][0] const resource = `find test.${collectionName}` - const query = `{"_id":"1234"}` + const query = '{"_id":"1234"}' expect(span).to.have.property('resource', resource) expect(span.meta).to.have.property('mongodb.query', query) diff --git a/packages/datadog-plugin-mongodb-core/test/naming.js b/packages/datadog-plugin-mongodb-core/test/naming.js index 570301247fe..c5113333a29 100644 --- a/packages/datadog-plugin-mongodb-core/test/naming.js +++ b/packages/datadog-plugin-mongodb-core/test/naming.js @@ -14,6 +14,6 @@ const rawExpectedSchema = { } module.exports = { - rawExpectedSchema: rawExpectedSchema, + rawExpectedSchema, expectedSchema: resolveNaming(rawExpectedSchema) } diff --git a/packages/datadog-plugin-mongoose/test/integration-test/client.spec.js b/packages/datadog-plugin-mongoose/test/integration-test/client.spec.js index d393fdd774b..819bbfae63f 100644 --- a/packages/datadog-plugin-mongoose/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-mongoose/test/integration-test/client.spec.js @@ -25,7 +25,7 @@ describe('esm', () => { before(async function () { this.timeout(20000) sandbox = await createSandbox([`'mongoose@${version}'`], false, [ - `./packages/datadog-plugin-mongoose/test/integration-test/*`]) + './packages/datadog-plugin-mongoose/test/integration-test/*']) }) after(async () => { diff --git a/packages/datadog-plugin-mysql/test/index.spec.js b/packages/datadog-plugin-mysql/test/index.spec.js index 12b4e45457a..244b0d61a4c 100644 --- a/packages/datadog-plugin-mysql/test/index.spec.js +++ b/packages/datadog-plugin-mysql/test/index.spec.js @@ -6,6 +6,8 @@ const { ERROR_MESSAGE, ERROR_TYPE, ERROR_STACK } = require('../../dd-trace/src/c const { expectedSchema, rawExpectedSchema } = require('./naming') +const ddpv = require('mocha/package.json').version + describe('Plugin', () => { let mysql let tracer @@ -45,6 +47,7 @@ describe('Plugin', () => { tracer.scope().activate(span, () => { const span = tracer.scope().active() + // eslint-disable-next-line n/handle-callback-err connection.query('SELECT 1 + 1 AS solution', (err, results, fields) => { expect(results).to.not.be.null expect(fields).to.not.be.null @@ -303,7 +306,7 @@ describe('Plugin', () => { }) beforeEach(() => { - const plugin = tracer._pluginManager._pluginsByName['mysql'] + const plugin = tracer._pluginManager._pluginsByName.mysql computeStub = sinon.stub(plugin._tracerConfig, 'spanComputePeerService') remapStub = sinon.stub(plugin._tracerConfig, 'peerServiceMapping') }) @@ -319,7 +322,8 @@ describe('Plugin', () => { connection.query('SELECT 1 + 1 AS solution', () => { try { expect(connection._protocol._queue[0].sql).to.equal( - `/*dddbs='serviced',dde='tester',ddps='test',ddpv='8.4.0'*/ SELECT 1 + 1 AS solution`) + '/*dddb=\'db\',dddbs=\'serviced\',dde=\'tester\',ddh=\'127.0.0.1\',ddps=\'test\'' + + `,ddpv='${ddpv}'*/ SELECT 1 + 1 AS solution`) } catch (e) { done(e) } @@ -333,7 +337,8 @@ describe('Plugin', () => { connection.query('SELECT 1 + 1 AS solution', () => { try { expect(connection._protocol._queue[0].sql).to.equal( - `/*dddbs='db',dde='tester',ddps='test',ddpv='8.4.0'*/ SELECT 1 + 1 AS solution`) + '/*dddb=\'db\',dddbs=\'db\',dde=\'tester\',ddh=\'127.0.0.1\',ddps=\'test\'' + + `,ddpv='${ddpv}'*/ SELECT 1 + 1 AS solution`) } catch (e) { done(e) } @@ -347,7 +352,8 @@ describe('Plugin', () => { connection.query('SELECT 1 + 1 AS solution', () => { try { expect(connection._protocol._queue[0].sql).to.equal( - `/*dddbs='remappedDB',dde='tester',ddps='test',ddpv='8.4.0'*/ SELECT 1 + 1 AS solution`) + '/*dddb=\'db\',dddbs=\'remappedDB\',dde=\'tester\',ddh=\'127.0.0.1\',' + + `ddps='test',ddpv='${ddpv}'*/ SELECT 1 + 1 AS solution`) } catch (e) { done(e) } @@ -375,7 +381,8 @@ describe('Plugin', () => { connection.query('SELECT 1 + 1 AS solution', () => { try { expect(connection._protocol._queue[0].sql).to.equal( - `/*dddbs='serviced',dde='tester',ddps='test',ddpv='8.4.0'*/ SELECT 1 + 1 AS solution`) + '/*dddb=\'db\',dddbs=\'serviced\',dde=\'tester\',ddh=\'127.0.0.1\',ddps=\'test\',' + + `ddpv='${ddpv}'*/ SELECT 1 + 1 AS solution`) } catch (e) { done(e) } @@ -421,8 +428,8 @@ describe('Plugin', () => { connection.query('SELECT 1 + 1 AS solution', () => { try { expect(connection._protocol._queue[0].sql).to.equal( - `/*dddbs='~!%40%23%24%25%5E%26*()_%2B%7C%3F%3F%2F%3C%3E',dde='tester',` + - `ddps='test',ddpv='8.4.0'*/ SELECT 1 + 1 AS solution`) + '/*dddb=\'db\',dddbs=\'~!%40%23%24%25%5E%26*()_%2B%7C%3F%3F%2F%3C%3E\',dde=\'tester\',' + + `ddh='127.0.0.1',ddps='test',ddpv='${ddpv}'*/ SELECT 1 + 1 AS solution`) done() } catch (e) { done(e) @@ -459,7 +466,7 @@ describe('Plugin', () => { const spanId = traces[0][0].span_id.toString(16).padStart(16, '0') expect(queryText).to.equal( - `/*dddbs='post',dde='tester',ddps='test',ddpv='8.4.0',` + + `/*dddb='db',dddbs='post',dde='tester',ddh='127.0.0.1',ddps='test',ddpv='${ddpv}',` + `traceparent='00-${traceId}-${spanId}-00'*/ SELECT 1 + 1 AS solution`) }).then(done, done) const clock = sinon.useFakeTimers(new Date()) @@ -468,6 +475,7 @@ describe('Plugin', () => { queryText = connection._protocol._queue[0].sql }) }) + it('query should inject _dd.dbm_trace_injected into span', done => { agent.use(traces => { expect(traces[0][0].meta).to.have.property('_dd.dbm_trace_injected', 'true') @@ -502,7 +510,8 @@ describe('Plugin', () => { pool.query('SELECT 1 + 1 AS solution', () => { try { expect(pool._allConnections[0]._protocol._queue[0].sql).to.equal( - `/*dddbs='post',dde='tester',ddps='test',ddpv='8.4.0'*/ SELECT 1 + 1 AS solution`) + '/*dddb=\'db\',dddbs=\'post\',dde=\'tester\',ddh=\'127.0.0.1\',' + + `ddps='test',ddpv='${ddpv}'*/ SELECT 1 + 1 AS solution`) } catch (e) { done(e) } @@ -539,7 +548,7 @@ describe('Plugin', () => { const spanId = traces[0][0].span_id.toString(16).padStart(16, '0') expect(queryText).to.equal( - `/*dddbs='post',dde='tester',ddps='test',ddpv='8.4.0',` + + `/*dddb='db',dddbs='post',dde='tester',ddh='127.0.0.1',ddps='test',ddpv='${ddpv}',` + `traceparent='00-${traceId}-${spanId}-00'*/ SELECT 1 + 1 AS solution`) }).then(done, done) const clock = sinon.useFakeTimers(new Date()) @@ -548,6 +557,7 @@ describe('Plugin', () => { queryText = pool._allConnections[0]._protocol._queue[0].sql }) }) + it('query should inject _dd.dbm_trace_injected into span', done => { agent.use(traces => { expect(traces[0][0].meta).to.have.property('_dd.dbm_trace_injected', 'true') diff --git a/packages/datadog-plugin-mysql/test/integration-test/client.spec.js b/packages/datadog-plugin-mysql/test/integration-test/client.spec.js index 9fb59247676..621e8fcd07c 100644 --- a/packages/datadog-plugin-mysql/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-mysql/test/integration-test/client.spec.js @@ -20,7 +20,7 @@ describe('esm', () => { before(async function () { this.timeout(20000) sandbox = await createSandbox([`'mysql@${version}'`], false, [ - `./packages/datadog-plugin-mysql/test/integration-test/*`]) + './packages/datadog-plugin-mysql/test/integration-test/*']) }) after(async () => { diff --git a/packages/datadog-plugin-mysql/test/leak.js b/packages/datadog-plugin-mysql/test/leak.js deleted file mode 100644 index 665e8aae508..00000000000 --- a/packages/datadog-plugin-mysql/test/leak.js +++ /dev/null @@ -1,27 +0,0 @@ -'use strict' - -require('../../dd-trace') - .init({ plugins: false, sampleRate: 0 }) - .use('mysql') - -const test = require('tape') -const mysql = require('../../../versions/mysql').get() -const profile = require('../../dd-trace/test/profile') - -test('mysql plugin should not leak', t => { - const connection = mysql.createConnection({ - host: 'localhost', - user: 'root', - database: 'db' - }) - - connection.connect(err => { - if (err) return t.fail(err) - - profile(t, operation).then(() => connection.end()) - }) - - function operation (done) { - connection.query('SELECT 1 + 1 AS solution', done) - } -}) diff --git a/packages/datadog-plugin-mysql/test/naming.js b/packages/datadog-plugin-mysql/test/naming.js index aa9f1c95555..d9f2342d5d1 100644 --- a/packages/datadog-plugin-mysql/test/naming.js +++ b/packages/datadog-plugin-mysql/test/naming.js @@ -14,6 +14,6 @@ const rawExpectedSchema = { } module.exports = { - rawExpectedSchema: rawExpectedSchema, + rawExpectedSchema, expectedSchema: resolveNaming(rawExpectedSchema) } diff --git a/packages/datadog-plugin-mysql2/test/index.spec.js b/packages/datadog-plugin-mysql2/test/index.spec.js index 38d6c487eaa..20efeb20454 100644 --- a/packages/datadog-plugin-mysql2/test/index.spec.js +++ b/packages/datadog-plugin-mysql2/test/index.spec.js @@ -6,6 +6,8 @@ const { ERROR_MESSAGE, ERROR_TYPE, ERROR_STACK } = require('../../dd-trace/src/c const { expectedSchema, rawExpectedSchema } = require('./naming') +const ddpv = require('mocha/package.json').version + describe('Plugin', () => { let mysql2 let tracer @@ -55,6 +57,7 @@ describe('Plugin', () => { tracer.scope().activate(span, () => { const span = tracer.scope().active() + // eslint-disable-next-line n/handle-callback-err connection.query('SELECT 1 + 1 AS solution', (err, results, fields) => { try { expect(results).to.not.be.null @@ -365,14 +368,15 @@ describe('Plugin', () => { it('should contain comment in query text', done => { const connect = connection.query('SELECT 1 + 1 AS solution', (...args) => { try { - expect(connect.sql).to.equal(`/*dddbs='serviced',dde='tester',` + - `ddps='test',ddpv='8.4.0'*/ SELECT 1 + 1 AS solution`) + expect(connect.sql).to.equal('/*dddb=\'db\',dddbs=\'serviced\',dde=\'tester\',ddh=\'127.0.0.1\',' + + `ddps='test',ddpv='${ddpv}'*/ SELECT 1 + 1 AS solution`) } catch (e) { done(e) } done() }) }) + it('trace query resource should not be changed when propagation is enabled', done => { agent.use(traces => { expect(traces[0][0]).to.have.property('resource', 'SELECT 1 + 1 AS solution') @@ -411,8 +415,8 @@ describe('Plugin', () => { const connect = connection.query('SELECT 1 + 1 AS solution', () => { try { expect(connect.sql).to.equal( - `/*dddbs='~!%40%23%24%25%5E%26*()_%2B%7C%3F%3F%2F%3C%3E',dde='tester',` + - `ddps='test',ddpv='8.4.0'*/ SELECT 1 + 1 AS solution`) + '/*dddb=\'db\',dddbs=\'~!%40%23%24%25%5E%26*()_%2B%7C%3F%3F%2F%3C%3E\',dde=\'tester\',' + + `ddh='127.0.0.1',ddps='test',ddpv='${ddpv}'*/ SELECT 1 + 1 AS solution`) done() } catch (e) { done(e) @@ -449,7 +453,7 @@ describe('Plugin', () => { const spanId = traces[0][0].span_id.toString(16).padStart(16, '0') expect(queryText).to.equal( - `/*dddbs='post',dde='tester',ddps='test',ddpv='8.4.0',` + + `/*dddb='db',dddbs='post',dde='tester',ddh='127.0.0.1',ddps='test',ddpv='${ddpv}',` + `traceparent='00-${traceId}-${spanId}-00'*/ SELECT 1 + 1 AS solution`) }).then(done, done) const clock = sinon.useFakeTimers(new Date()) @@ -458,6 +462,7 @@ describe('Plugin', () => { queryText = connect.sql }) }) + it('query should inject _dd.dbm_trace_injected into span', done => { agent.use(traces => { expect(traces[0][0].meta).to.have.property('_dd.dbm_trace_injected', 'true') @@ -492,7 +497,8 @@ describe('Plugin', () => { const queryPool = pool.query('SELECT 1 + 1 AS solution', () => { try { expect(queryPool.sql).to.equal( - `/*dddbs='post',dde='tester',ddps='test',ddpv='8.4.0'*/ SELECT 1 + 1 AS solution`) + '/*dddb=\'db\',dddbs=\'post\',dde=\'tester\',ddh=\'127.0.0.1\',' + + `ddps='test',ddpv='${ddpv}'*/ SELECT 1 + 1 AS solution`) } catch (e) { done(e) } @@ -529,7 +535,7 @@ describe('Plugin', () => { const spanId = traces[0][0].span_id.toString(16).padStart(16, '0') expect(queryText).to.equal( - `/*dddbs='post',dde='tester',ddps='test',ddpv='8.4.0',` + + `/*dddb='db',dddbs='post',dde='tester',ddh='127.0.0.1',ddps='test',ddpv='${ddpv}',` + `traceparent='00-${traceId}-${spanId}-00'*/ SELECT 1 + 1 AS solution`) }).then(done, done) const clock = sinon.useFakeTimers(new Date()) @@ -538,6 +544,7 @@ describe('Plugin', () => { queryText = queryPool.sql }) }) + it('query should inject _dd.dbm_trace_injected into span', done => { agent.use(traces => { expect(traces[0][0].meta).to.have.property('_dd.dbm_trace_injected', 'true') diff --git a/packages/datadog-plugin-mysql2/test/integration-test/client.spec.js b/packages/datadog-plugin-mysql2/test/integration-test/client.spec.js index 8f511647f8c..6be107f34e0 100644 --- a/packages/datadog-plugin-mysql2/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-mysql2/test/integration-test/client.spec.js @@ -21,7 +21,7 @@ describe('esm', () => { before(async function () { this.timeout(20000) sandbox = await createSandbox([`'mysql2@${version}'`], false, [ - `./packages/datadog-plugin-mysql2/test/integration-test/*`]) + './packages/datadog-plugin-mysql2/test/integration-test/*']) }) after(async () => { diff --git a/packages/datadog-plugin-mysql2/test/naming.js b/packages/datadog-plugin-mysql2/test/naming.js index aa9f1c95555..d9f2342d5d1 100644 --- a/packages/datadog-plugin-mysql2/test/naming.js +++ b/packages/datadog-plugin-mysql2/test/naming.js @@ -14,6 +14,6 @@ const rawExpectedSchema = { } module.exports = { - rawExpectedSchema: rawExpectedSchema, + rawExpectedSchema, expectedSchema: resolveNaming(rawExpectedSchema) } diff --git a/packages/datadog-plugin-net/test/index.spec.js b/packages/datadog-plugin-net/test/index.spec.js index 7f1cf5d3e33..adcf175e405 100644 --- a/packages/datadog-plugin-net/test/index.spec.js +++ b/packages/datadog-plugin-net/test/index.spec.js @@ -1,6 +1,5 @@ 'use strict' -const getPort = require('get-port') const dns = require('dns') const agent = require('../../dd-trace/test/plugins/agent') const { expectSomeSpan } = require('../../dd-trace/test/plugins/helpers') @@ -15,274 +14,295 @@ describe('Plugin', () => { let tracer let parent - describe('net', () => { - afterEach(() => { - return agent.close() - }) - - afterEach(() => { - tcp.close() - }) + ['net', 'node:net'].forEach(pluginToBeLoaded => { + describe(pluginToBeLoaded, () => { + afterEach(() => { + return agent.close() + }) - afterEach(() => { - ipc.close() - }) + afterEach(() => { + tcp.close() + }) - beforeEach(() => { - return agent.load('net') - .then(() => { - net = require(`net`) - tracer = require('../../dd-trace') - parent = tracer.startSpan('parent') - parent.finish() + afterEach(() => { + ipc.close() + }) - return getPort() - }).then(_port => { - port = _port + beforeEach(() => { + return agent.load(['net', 'dns']) + .then(() => { + net = require(pluginToBeLoaded) + tracer = require('../../dd-trace') + parent = tracer.startSpan('parent') + parent.finish() + }).then(_port => { + return new Promise(resolve => setImmediate(resolve)) + }) + }) - return new Promise(resolve => setImmediate(resolve)) + beforeEach(done => { + tcp = new net.Server(socket => { + socket.write('') + }) + tcp.listen(0, () => { + port = tcp.address().port + done() }) - }) - - beforeEach(done => { - tcp = new net.Server(socket => { - socket.write('') }) - tcp.listen(port, () => done()) - }) - beforeEach(done => { - ipc = new net.Server(socket => { - socket.write('') + beforeEach(done => { + ipc = new net.Server(socket => { + socket.write('') + }) + ipc.listen('/tmp/dd-trace.sock', () => done()) }) - ipc.listen('/tmp/dd-trace.sock', () => done()) - }) - it('should instrument connect with a path', done => { - expectSomeSpan(agent, { - name: 'ipc.connect', - service: 'test', - resource: '/tmp/dd-trace.sock', - meta: { - 'span.kind': 'client', - 'ipc.path': '/tmp/dd-trace.sock' - }, - parent_id: new Int64BE(parent.context()._spanId._buffer) - }).then(done).catch(done) - - tracer.scope().activate(parent, () => { - net.connect('/tmp/dd-trace.sock') - }) - }) + it('should instrument connect with a path', done => { + expectSomeSpan(agent, { + name: 'ipc.connect', + service: 'test', + resource: '/tmp/dd-trace.sock', + meta: { + 'span.kind': 'client', + 'ipc.path': '/tmp/dd-trace.sock' + }, + parent_id: new Int64BE(parent.context()._spanId._buffer) + }).then(done).catch(done) - withPeerService( - () => tracer, - 'net', - () => { - const socket = new net.Socket() - socket.connect(port, 'localhost') - }, - 'localhost', - 'out.host' - ) - - it('should instrument connect with a port', done => { - const socket = new net.Socket() - tracer.scope().activate(parent, () => { - socket.connect(port, 'localhost') - socket.on('connect', () => { - expectSomeSpan(agent, { - name: 'tcp.connect', - service: 'test', - resource: `localhost:${port}`, - meta: { - 'component': 'net', - 'span.kind': 'client', - 'tcp.family': 'IPv4', - 'tcp.remote.host': 'localhost', - 'tcp.local.address': socket.localAddress, - 'out.host': 'localhost' - }, - metrics: { - 'network.destination.port': port, - 'tcp.remote.port': port, - 'tcp.local.port': socket.localPort - }, - parent_id: new Int64BE(parent.context()._spanId._buffer) - }, 2000).then(done).catch(done) + tracer.scope().activate(parent, () => { + net.connect('/tmp/dd-trace.sock') }) }) - }) - it('should instrument connect with TCP options', done => { - const socket = new net.Socket() - tracer.scope().activate(parent, () => { - socket.connect({ - port, - host: 'localhost' - }) - socket.on('connect', () => { - expectSomeSpan(agent, { - name: 'tcp.connect', - service: 'test', - resource: `localhost:${port}`, - meta: { - 'component': 'net', - 'span.kind': 'client', - 'tcp.family': 'IPv4', - 'tcp.remote.host': 'localhost', - 'tcp.local.address': socket.localAddress, - 'out.host': 'localhost' - }, - metrics: { - 'network.destination.port': port, - 'tcp.remote.port': port, - 'tcp.local.port': socket.localPort - }, - parent_id: new Int64BE(parent.context()._spanId._buffer) - }).then(done).catch(done) + it('should instrument dns', done => { + const socket = new net.Socket() + tracer.scope().activate(parent, () => { + socket.connect(port, 'localhost') + socket.on('connect', () => { + expectSomeSpan(agent, { + name: 'dns.lookup', + service: 'test', + resource: 'localhost' + }, 2000).then(done).catch(done) + }) }) }) - }) - it('should instrument connect with IPC options', done => { - expectSomeSpan(agent, { - name: 'ipc.connect', - service: 'test', - resource: '/tmp/dd-trace.sock', - meta: { - 'component': 'net', - 'span.kind': 'client', - 'ipc.path': '/tmp/dd-trace.sock' + withPeerService( + () => tracer, + 'net', + () => { + const socket = new net.Socket() + socket.connect(port, 'localhost') }, - parent_id: new Int64BE(parent.context()._spanId._buffer) - }).then(done).catch(done) + 'localhost', + 'out.host' + ) - tracer.scope().activate(parent, () => { - net.connect({ - path: '/tmp/dd-trace.sock' + it('should instrument connect with a port', done => { + const socket = new net.Socket() + tracer.scope().activate(parent, () => { + socket.connect(port, 'localhost') + socket.on('connect', () => { + expectSomeSpan(agent, { + name: 'tcp.connect', + service: 'test', + resource: `localhost:${port}`, + meta: { + component: 'net', + 'span.kind': 'client', + 'tcp.family': 'IPv4', + 'tcp.remote.host': 'localhost', + 'tcp.local.address': socket.localAddress, + 'out.host': 'localhost' + }, + metrics: { + 'network.destination.port': port, + 'tcp.remote.port': port, + 'tcp.local.port': socket.localPort + }, + parent_id: new Int64BE(parent.context()._spanId._buffer) + }, 2000).then(done).catch(done) + }) }) }) - }) - - it('should instrument error', done => { - const socket = new net.Socket() - - let error = null - agent - .use(traces => { - expect(traces[0][0]).to.deep.include({ - name: 'tcp.connect', - service: 'test', - resource: `localhost:${port}` - }) - expect(traces[0][0].meta).to.deep.include({ - 'component': 'net', - 'span.kind': 'client', - 'tcp.family': 'IPv4', - 'tcp.remote.host': 'localhost', - 'out.host': 'localhost', - [ERROR_TYPE]: error.name, - [ERROR_MESSAGE]: error.message || error.code, - [ERROR_STACK]: error.stack + it('should instrument connect with TCP options', done => { + const socket = new net.Socket() + tracer.scope().activate(parent, () => { + socket.connect({ + port, + host: 'localhost' }) - expect(traces[0][0].metrics).to.deep.include({ - 'network.destination.port': port, - 'tcp.remote.port': port + socket.on('connect', () => { + expectSomeSpan(agent, { + name: 'tcp.connect', + service: 'test', + resource: `localhost:${port}`, + meta: { + component: 'net', + 'span.kind': 'client', + 'tcp.family': 'IPv4', + 'tcp.remote.host': 'localhost', + 'tcp.local.address': socket.localAddress, + 'out.host': 'localhost' + }, + metrics: { + 'network.destination.port': port, + 'tcp.remote.port': port, + 'tcp.local.port': socket.localPort + }, + parent_id: new Int64BE(parent.context()._spanId._buffer) + }).then(done).catch(done) }) - expect(traces[0][0].parent_id.toString()).to.equal(parent.context().toSpanId()) }) - .then(done) - .catch(done) + }) - tracer.scope().activate(parent, () => { - tcp.close() - socket.connect({ port }) - socket.once('error', (err) => { - error = err + it('should instrument connect with IPC options', done => { + expectSomeSpan(agent, { + name: 'ipc.connect', + service: 'test', + resource: '/tmp/dd-trace.sock', + meta: { + component: 'net', + 'span.kind': 'client', + 'ipc.path': '/tmp/dd-trace.sock' + }, + parent_id: new Int64BE(parent.context()._spanId._buffer) + }).then(done).catch(done) + + tracer.scope().activate(parent, () => { + net.connect({ + path: '/tmp/dd-trace.sock' + }) }) }) - }) - it('should cleanup event listeners when the socket changes state', done => { - const socket = new net.Socket() - - tracer.scope().activate(parent, () => { - const events = ['connect', 'error', 'close', 'timeout'] - - socket.connect({ port }) - socket.destroy() + it('should instrument error', done => { + const socket = new net.Socket() - socket.once('close', () => { - expect(socket.eventNames()).to.not.include.members(events) - done() + let error = null + + agent + .use(traces => { + expect(traces[0][0]).to.deep.include({ + name: 'tcp.connect', + service: 'test', + resource: `localhost:${port}` + }) + expect(traces[0][0].meta).to.deep.include({ + component: 'net', + 'span.kind': 'client', + 'tcp.family': 'IPv4', + 'tcp.remote.host': 'localhost', + 'out.host': 'localhost', + [ERROR_TYPE]: error.name, + [ERROR_MESSAGE]: error.message || error.code, + [ERROR_STACK]: error.stack + }) + expect(traces[0][0].metrics).to.deep.include({ + 'network.destination.port': port, + 'tcp.remote.port': port + }) + expect(traces[0][0].parent_id.toString()).to.equal(parent.context().toSpanId()) + }) + .then(done) + .catch(done) + + tracer.scope().activate(parent, () => { + tcp.close() + socket.connect({ port }) + socket.once('error', (err) => { + error = err + }) }) }) - }) - it('should run event listeners in the correct scope', () => { - return tracer.scope().activate(parent, () => { + it('should cleanup event listeners when the socket changes state', done => { const socket = new net.Socket() - const promises = Array(5).fill(0).map(() => { - let res - let rej - const p = new Promise((resolve, reject) => { - res = resolve - rej = reject + tracer.scope().activate(parent, () => { + const events = ['connect', 'error', 'close', 'timeout'] + + socket.connect({ port }) + socket.destroy() + + socket.once('close', () => { + setImmediate(() => { + // Node.js 21.2 broke this function. We'll have to do the more manual way for now. + // expect(socket.eventNames()).to.not.include.members(events) + for (const event of events) { + expect(socket.listeners(event)).to.have.lengthOf(0) + } + done() + }) }) - p.resolve = res - p.reject = rej - return p }) + }) - socket.on('connect', () => { - expect(tracer.scope().active()).to.equal(parent) - promises[0].resolve() - }) + it('should run event listeners in the correct scope', () => { + return tracer.scope().activate(parent, () => { + const socket = new net.Socket() + + const promises = Array(5).fill(0).map(() => { + let res + let rej + const p = new Promise((resolve, reject) => { + res = resolve + rej = reject + }) + p.resolve = res + p.reject = rej + return p + }) - socket.on('ready', () => { - expect(tracer.scope().active()).to.equal(parent) - socket.destroy() - promises[1].resolve() - }) + socket.on('connect', () => { + expect(tracer.scope().active()).to.equal(parent) + promises[0].resolve() + }) - socket.on('close', () => { - expect(tracer.scope().active()).to.not.be.null - expect(tracer.scope().active().context()._name).to.equal('tcp.connect') - promises[2].resolve() - }) + socket.on('ready', () => { + expect(tracer.scope().active()).to.equal(parent) + socket.destroy() + promises[1].resolve() + }) - socket.on('lookup', () => { - expect(tracer.scope().active()).to.not.be.null - expect(tracer.scope().active().context()._name).to.equal('tcp.connect') - promises[3].resolve() - }) + socket.on('close', () => { + expect(tracer.scope().active()).to.not.be.null + expect(tracer.scope().active().context()._name).to.equal('tcp.connect') + promises[2].resolve() + }) - socket.connect({ - port, - lookup: (...args) => { + socket.on('lookup', () => { expect(tracer.scope().active()).to.not.be.null expect(tracer.scope().active().context()._name).to.equal('tcp.connect') - promises[4].resolve() - dns.lookup(...args) - } - }) + promises[3].resolve() + }) - return Promise.all(promises) + socket.connect({ + port, + lookup: (...args) => { + expect(tracer.scope().active()).to.not.be.null + expect(tracer.scope().active().context()._name).to.equal('tcp.connect') + promises[4].resolve() + dns.lookup(...args) + } + }) + + return Promise.all(promises) + }) }) - }) - it('should run the connection callback in the correct scope', done => { - const socket = new net.Socket() + it('should run the connection callback in the correct scope', done => { + const socket = new net.Socket() - tracer.scope().activate(parent, () => { - socket.connect({ port }, function () { - expect(this).to.equal(socket) - expect(tracer.scope().active()).to.equal(parent) - socket.destroy() - done() + tracer.scope().activate(parent, () => { + socket.connect({ port }, function () { + expect(this).to.equal(socket) + expect(tracer.scope().active()).to.equal(parent) + socket.destroy() + done() + }) }) }) }) diff --git a/packages/datadog-plugin-net/test/integration-test/client.spec.js b/packages/datadog-plugin-net/test/integration-test/client.spec.js index 587b3298cf2..c0810e36383 100644 --- a/packages/datadog-plugin-net/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-net/test/integration-test/client.spec.js @@ -20,7 +20,7 @@ describe('esm', () => { before(async function () { this.timeout(20000) sandbox = await createSandbox(['net', 'get-port'], false, [ - `./packages/datadog-plugin-net/test/integration-test/*`]) + './packages/datadog-plugin-net/test/integration-test/*']) }) after(async () => { diff --git a/packages/datadog-plugin-next/src/index.js b/packages/datadog-plugin-next/src/index.js index 4bd1c21f984..1dff5bec4e9 100644 --- a/packages/datadog-plugin-next/src/index.js +++ b/packages/datadog-plugin-next/src/index.js @@ -6,6 +6,8 @@ const analyticsSampler = require('../../dd-trace/src/analytics_sampler') const { COMPONENT } = require('../../dd-trace/src/constants') const web = require('../../dd-trace/src/plugins/util/web') +const errorPages = ['/404', '/500', '/_error', '/_not-found', '/_not-found/page'] + class NextPlugin extends ServerPlugin { static get id () { return 'next' @@ -40,6 +42,13 @@ class NextPlugin extends ServerPlugin { } error ({ span, error }) { + if (!span) { + const store = storage.getStore() + if (!store) return + + span = store.span + } + this.addError(error, span) } @@ -49,11 +58,21 @@ class NextPlugin extends ServerPlugin { if (!store) return const span = store.span - const error = span.context()._tags['error'] - - if (!this.config.validateStatus(res.statusCode) && !error) { - span.setTag('error', req.error || nextRequest.error || true) - web.addError(req, req.error || nextRequest.error || true) + const error = span.context()._tags.error + const requestError = req.error || nextRequest.error + + if (requestError) { + // prioritize user-set errors from API routes + span.setTag('error', requestError) + web.addError(req, requestError) + } else if (error) { + // general error handling + span.setTag('error', error) + web.addError(req, requestError || error) + } else if (!this.config.validateStatus(res.statusCode)) { + // where there's no error, we still need to validate status + span.setTag('error', true) + web.addError(req, true) } span.addTags({ @@ -73,14 +92,21 @@ class NextPlugin extends ServerPlugin { const span = store.span const req = this._requests.get(span) + // safeguard against missing req in complicated timeout scenarios + if (!req) return + // Only use error page names if there's not already a name const current = span.context()._tags['next.page'] - if (current && ['/404', '/500', '/_error', '/_not-found'].includes(page)) { + const isErrorPage = errorPages.includes(page) + + if (current && isErrorPage) { return } // remove ending /route or /page for appDir projects - if (isAppPath) page = page.substring(0, page.lastIndexOf('/')) + // need to check if not an error page too, as those are marked as app directory + // in newer versions + if (isAppPath && !isErrorPage) page = page.substring(0, page.lastIndexOf('/')) // handle static resource if (isStatic) { @@ -94,7 +120,6 @@ class NextPlugin extends ServerPlugin { 'resource.name': `${req.method} ${page}`.trim(), 'next.page': page }) - web.setRoute(req, page) } diff --git a/packages/datadog-plugin-next/test/datadog.js b/packages/datadog-plugin-next/test/datadog.js index 1e1008cf5d4..565fe15db99 100644 --- a/packages/datadog-plugin-next/test/datadog.js +++ b/packages/datadog-plugin-next/test/datadog.js @@ -1,8 +1,4 @@ -module.exports = require('../../..').init({ - service: 'test', - flushInterval: 0, - plugins: false -}).use('next', process.env.WITH_CONFIG ? { +const config = { validateStatus: code => false, hooks: { request: (span, req) => { @@ -15,4 +11,10 @@ module.exports = require('../../..').init({ span.setTag('foo', 'bar') } } -} : true).use('http') +} + +module.exports = require('../../..').init({ + service: 'test', + flushInterval: 0, + plugins: false +}).use('next', process.env.WITH_CONFIG ? config : true).use('http') diff --git a/packages/datadog-plugin-next/test/index.spec.js b/packages/datadog-plugin-next/test/index.spec.js index d03668bcb2a..caec28e3b1a 100644 --- a/packages/datadog-plugin-next/test/index.spec.js +++ b/packages/datadog-plugin-next/test/index.spec.js @@ -2,15 +2,22 @@ /* eslint import/no-extraneous-dependencies: ["error", {"packageDir": ['./']}] */ +const path = require('path') const axios = require('axios') const getPort = require('get-port') const { execSync, spawn } = require('child_process') const agent = require('../../dd-trace/test/plugins/agent') const { writeFileSync, readdirSync } = require('fs') const { satisfies } = require('semver') -const { DD_MAJOR } = require('../../../version') +const { DD_MAJOR, NODE_MAJOR } = require('../../../version') const { rawExpectedSchema } = require('./naming') +const BUILD_COMMAND = NODE_MAJOR < 18 + ? 'yarn exec next build' + : 'NODE_OPTIONS=--openssl-legacy-provider yarn exec next build' +let VERSIONS_TO_TEST = NODE_MAJOR < 18 ? '>=11.1 <13.2' : '>=11.1' +VERSIONS_TO_TEST = DD_MAJOR >= 4 ? VERSIONS_TO_TEST : '>=9.5 <11.1' + describe('Plugin', function () { let server let port @@ -19,8 +26,8 @@ describe('Plugin', function () { const satisfiesStandalone = version => satisfies(version, '>=12.0.0') // TODO: Figure out why 10.x tests are failing. - withVersions('next', 'next', DD_MAJOR >= 4 && '>=11', version => { - const pkg = require(`${__dirname}/../../../versions/next@${version}/node_modules/next/package.json`) + withVersions('next', 'next', VERSIONS_TO_TEST, version => { + const pkg = require(`../../../versions/next@${version}/node_modules/next/package.json`) const startServer = ({ withConfig, standalone }, schemaVersion = 'v0', defaultToGlobalService = false) => { before(async () => { @@ -30,9 +37,9 @@ describe('Plugin', function () { }) before(function (done) { - this.timeout(40000) + this.timeout(120000) const cwd = standalone - ? `${__dirname}/.next/standalone` + ? path.join(__dirname, '.next/standalone') : __dirname server = spawn('node', ['server'], { @@ -45,6 +52,7 @@ describe('Plugin', function () { WITH_CONFIG: withConfig, DD_TRACE_SPAN_ATTRIBUTE_SCHEMA: schemaVersion, DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED: defaultToGlobalService, + // eslint-disable-next-line n/no-path-concat NODE_OPTIONS: `--require ${__dirname}/datadog.js`, HOSTNAME: '127.0.0.1', TIMES_HOOK_CALLED: 0 @@ -52,15 +60,16 @@ describe('Plugin', function () { }) server.once('error', done) - server.stdout.once('data', () => { - // first log outputted isn't always the server started log - // https://github.com/vercel/next.js/blob/v10.2.0/packages/next/next-server/server/config-utils.ts#L39 - // these are webpack related logs that run during execution time and not build - - // additionally, next.js sets timeouts in 10.x when displaying extra logs - // https://github.com/vercel/next.js/blob/v10.2.0/packages/next/server/next.ts#L132-L133 - setTimeout(done, 700) // relatively high timeout chosen to be safe - }) + + function waitUntilServerStarted (chunk) { + const chunkString = chunk.toString() + if (chunkString?.includes(port) || chunkString?.includes('Ready ')) { + server.stdout.off('data', waitUntilServerStarted) + done() + } + } + server.stdout.on('data', waitUntilServerStarted) + server.stderr.on('data', chunk => process.stderr.write(chunk)) server.stdout.on('data', chunk => process.stdout.write(chunk)) }) @@ -76,11 +85,11 @@ describe('Plugin', function () { } before(async function () { - this.timeout(120 * 1000) // Webpack is very slow and builds on every test run + this.timeout(240 * 1000) // Webpack is very slow and builds on every test run const cwd = __dirname - const pkg = require(`${__dirname}/../../../versions/next@${version}/package.json`) - const realVersion = require(`${__dirname}/../../../versions/next@${version}`).version() + const pkg = require(`../../../versions/next@${version}/package.json`) + const realVersion = require(`../../../versions/next@${version}`).version() delete pkg.workspaces @@ -90,14 +99,18 @@ describe('Plugin', function () { // https://nextjs.org/blog/next-9-5#webpack-5-support-beta if (realVersion.startsWith('9')) pkg.resolutions = { webpack: '^5.0.0' } - writeFileSync(`${__dirname}/package.json`, JSON.stringify(pkg, null, 2)) + writeFileSync(path.join(__dirname, 'package.json'), JSON.stringify(pkg, null, 2)) // installing here for standalone purposes, copying `nodules` above was not generating the server file properly // if there is a way to re-use nodules from somewhere in the versions folder, this `execSync` will be reverted - execSync('yarn install', { cwd }) + try { + execSync('yarn install', { cwd }) + } catch (e) { // retry in case of error from registry + execSync('yarn install', { cwd }) + } // building in-process makes tests fail for an unknown reason - execSync('yarn exec next build', { + execSync(BUILD_COMMAND, { cwd, env: { ...process.env, @@ -108,8 +121,8 @@ describe('Plugin', function () { if (satisfiesStandalone(realVersion)) { // copy public and static files to the `standalone` folder - const publicOrigin = `${__dirname}/public` - const publicDestination = `${__dirname}/.next/standalone/public` + const publicOrigin = path.join(__dirname, 'public') + const publicDestination = path.join(__dirname, '.next/standalone/public') execSync(`mkdir ${publicDestination}`) execSync(`cp ${publicOrigin}/test.txt ${publicDestination}/test.txt`) } @@ -123,7 +136,7 @@ describe('Plugin', function () { '.next', 'yarn.lock' ] - const paths = files.map(file => `${__dirname}/${file}`) + const paths = files.map(file => path.join(__dirname, file)) execSync(`rm -rf ${paths.join(' ')}`) }) @@ -344,6 +357,25 @@ describe('Plugin', function () { .get(`http://127.0.0.1:${port}/hello/world`) .catch(done) }) + + it('should attach errors by default', done => { + agent + .use(traces => { + const spans = traces[0] + + expect(spans[1]).to.have.property('name', 'next.request') + expect(spans[1]).to.have.property('error', 1) + + expect(spans[1].meta).to.have.property('http.status_code', '500') + expect(spans[1].meta).to.have.property('error.message', 'fail') + expect(spans[1].meta).to.have.property('error.type', 'Error') + expect(spans[1].meta['error.stack']).to.exist + }) + .then(done) + .catch(done) + + axios.get(`http://127.0.0.1:${port}/error/get_server_side_props`) + }) }) describe('for static files', () => { @@ -371,7 +403,7 @@ describe('Plugin', function () { it('should do automatic instrumentation for static chunks', done => { // get first static chunk file programatically - const file = readdirSync(`${__dirname}/.next/static/chunks`)[0] + const file = readdirSync(path.join(__dirname, '.next/static/chunks'))[0] agent .use(traces => { @@ -433,7 +465,7 @@ describe('Plugin', function () { .use(traces => { const spans = traces[0] - expect(spans[1]).to.have.property('resource', `GET /api/appDir/[name]`) + expect(spans[1]).to.have.property('resource', 'GET /api/appDir/[name]') }) .then(done) .catch(done) @@ -448,7 +480,7 @@ describe('Plugin', function () { .use(traces => { const spans = traces[0] - expect(spans[1]).to.have.property('resource', `GET /appDir/[name]`) + expect(spans[1]).to.have.property('resource', 'GET /appDir/[name]') expect(spans[1].meta).to.have.property('http.status_code', '200') }) .then(done) diff --git a/packages/datadog-plugin-next/test/integration-test/client.spec.js b/packages/datadog-plugin-next/test/integration-test/client.spec.js index 1ab099ba7bd..054e2fc6357 100644 --- a/packages/datadog-plugin-next/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-next/test/integration-test/client.spec.js @@ -8,20 +8,31 @@ const { spawnPluginIntegrationTestProc } = require('../../../../integration-tests/helpers') const { assert } = require('chai') +const { NODE_MAJOR } = require('../../../../version') const hookFile = 'dd-trace/loader-hook.mjs' +const BUILD_COMMAND = NODE_MAJOR < 18 + ? 'yarn exec next build' + : 'NODE_OPTIONS=--openssl-legacy-provider yarn exec next build' +const NODE_OPTIONS = NODE_MAJOR < 18 + ? `--loader=${hookFile} --require dd-trace/init` + : `--loader=${hookFile} --require dd-trace/init --openssl-legacy-provider` + +const VERSIONS_TO_TEST = NODE_MAJOR < 18 ? '>=11.1 <13.2' : '>=11.1' + describe('esm', () => { let agent let proc let sandbox // match versions tested with unit tests - withVersions('next', 'next', '>=11', version => { + withVersions('next', 'next', VERSIONS_TO_TEST, version => { before(async function () { // next builds slower in the CI, match timeout with unit tests this.timeout(120 * 1000) sandbox = await createSandbox([`'next@${version}'`, 'react', 'react-dom'], - false, ['./packages/datadog-plugin-next/test/integration-test/*'], 'yarn exec next build') + false, ['./packages/datadog-plugin-next/test/integration-test/*'], + BUILD_COMMAND) }) after(async () => { @@ -39,7 +50,7 @@ describe('esm', () => { it('is instrumented', async () => { proc = await spawnPluginIntegrationTestProc(sandbox.folder, 'server.mjs', agent.port, undefined, { - NODE_OPTIONS: `--loader=${hookFile} --require dd-trace/init` + NODE_OPTIONS }) return curlAndAssertMessage(agent, proc, ({ headers, payload }) => { assert.propertyVal(headers, 'host', `127.0.0.1:${agent.port}`) diff --git a/packages/datadog-plugin-next/test/server.js b/packages/datadog-plugin-next/test/server.js index e77c478b8a6..cc69d2833b9 100644 --- a/packages/datadog-plugin-next/test/server.js +++ b/packages/datadog-plugin-next/test/server.js @@ -3,6 +3,7 @@ const { PORT, HOSTNAME } = process.env const { createServer } = require('http') +// eslint-disable-next-line n/no-deprecated-api const { parse } = require('url') const next = require('next') // eslint-disable-line import/no-extraneous-dependencies diff --git a/packages/datadog-plugin-nyc/src/index.js b/packages/datadog-plugin-nyc/src/index.js new file mode 100644 index 00000000000..c407b55221c --- /dev/null +++ b/packages/datadog-plugin-nyc/src/index.js @@ -0,0 +1,35 @@ +const CiPlugin = require('../../dd-trace/src/plugins/ci_plugin') + +class NycPlugin extends CiPlugin { + static get id () { + return 'nyc' + } + + constructor (...args) { + super(...args) + + this.addSub('ci:nyc:wrap', (nyc) => { + if (nyc?.config?.all) { + this.nyc = nyc + } + }) + + this.addSub('ci:nyc:get-coverage', ({ onDone }) => { + if (this.nyc?.getCoverageMapFromAllCoverageFiles) { + this.nyc.getCoverageMapFromAllCoverageFiles() + .then((untestedCoverageMap) => { + this.nyc = null + onDone(untestedCoverageMap) + }).catch((e) => { + this.nyc = null + onDone() + }) + } else { + this.nyc = null + onDone() + } + }) + } +} + +module.exports = NycPlugin diff --git a/packages/datadog-plugin-openai/src/index.js b/packages/datadog-plugin-openai/src/index.js index 71ed800dfd5..f96b44543d2 100644 --- a/packages/datadog-plugin-openai/src/index.js +++ b/packages/datadog-plugin-openai/src/index.js @@ -7,6 +7,7 @@ const { storage } = require('../../datadog-core') const services = require('./services') const Sampler = require('../../dd-trace/src/sampler') const { MEASURED } = require('../../../ext/tags') +const { estimateTokens } = require('./token-estimator') // String#replaceAll unavailable on Node.js@v14 (dd-trace@<=v3) const RE_NEWLINE = /\n/g @@ -15,10 +16,24 @@ const RE_TAB = /\t/g // TODO: In the future we should refactor config.js to make it requirable let MAX_TEXT_LEN = 128 +function safeRequire (path) { + try { + // eslint-disable-next-line import/no-extraneous-dependencies + return require(path) + } catch { + return null + } +} + +const encodingForModel = safeRequire('tiktoken')?.encoding_for_model + class OpenApiPlugin extends TracingPlugin { static get id () { return 'openai' } static get operation () { return 'request' } static get system () { return 'openai' } + static get prefix () { + return 'tracing:apm:openai:request' + } constructor (...args) { super(...args) @@ -43,8 +58,10 @@ class OpenApiPlugin extends TracingPlugin { super.configure(config) } - start ({ methodName, args, basePath, apiKey }) { + bindStart (ctx) { + const { methodName, args, basePath, apiKey } = ctx const payload = normalizeRequestPayload(methodName, args) + const store = storage.getStore() || {} const span = this.startSpan('openai.request', { service: this.config.service, @@ -75,21 +92,19 @@ class OpenApiPlugin extends TracingPlugin { 'openai.request.user': payload.user, 'openai.request.file_id': payload.file_id // deleteFile, retrieveFile, downloadFile } - }) + }, false) - const fullStore = storage.getStore() || {} // certain request body fields are later used for logs - const store = Object.create(null) - fullStore.openai = store // namespacing these fields + const openaiStore = Object.create(null) const tags = {} // The remaining tags are added one at a time // createChatCompletion, createCompletion, createImage, createImageEdit, createTranscription, createTranslation - if ('prompt' in payload) { + if (payload.prompt) { const prompt = payload.prompt - store.prompt = prompt + openaiStore.prompt = prompt if (typeof prompt === 'string' || (Array.isArray(prompt) && typeof prompt[0] === 'number')) { // This is a single prompt, either String or [Number] - tags[`openai.request.prompt`] = normalizeStringOrTokenArray(prompt, true) + tags['openai.request.prompt'] = normalizeStringOrTokenArray(prompt, true) } else if (Array.isArray(prompt)) { // This is multiple prompts, either [String] or [[Number]] for (let i = 0; i < prompt.length; i++) { @@ -99,202 +114,350 @@ class OpenApiPlugin extends TracingPlugin { } // createEdit, createEmbedding, createModeration - if ('input' in payload) { + if (payload.input) { const normalized = normalizeStringOrTokenArray(payload.input, false) - tags[`openai.request.input`] = truncateText(normalized) - store.input = normalized + tags['openai.request.input'] = truncateText(normalized) + openaiStore.input = normalized } // createChatCompletion, createCompletion - if (typeof payload.logit_bias === 'object' && payload.logit_bias) { + if (payload.logit_bias !== null && typeof payload.logit_bias === 'object') { for (const [tokenId, bias] of Object.entries(payload.logit_bias)) { tags[`openai.request.logit_bias.${tokenId}`] = bias } } + if (payload.stream) { + tags['openai.request.stream'] = payload.stream + } + switch (methodName) { case 'createFineTune': + case 'fine_tuning.jobs.create': + case 'fine-tune.create': createFineTuneRequestExtraction(tags, payload) break case 'createImage': + case 'images.generate': case 'createImageEdit': + case 'images.edit': case 'createImageVariation': - commonCreateImageRequestExtraction(tags, payload, store) + case 'images.createVariation': + commonCreateImageRequestExtraction(tags, payload, openaiStore) break case 'createChatCompletion': - createChatCompletionRequestExtraction(tags, payload, store) + case 'chat.completions.create': + createChatCompletionRequestExtraction(tags, payload, openaiStore) break case 'createFile': + case 'files.create': case 'retrieveFile': + case 'files.retrieve': commonFileRequestExtraction(tags, payload) break case 'createTranscription': + case 'audio.transcriptions.create': case 'createTranslation': - commonCreateAudioRequestExtraction(tags, payload, store) + case 'audio.translations.create': + commonCreateAudioRequestExtraction(tags, payload, openaiStore) break case 'retrieveModel': + case 'models.retrieve': retrieveModelRequestExtraction(tags, payload) break case 'listFineTuneEvents': + case 'fine_tuning.jobs.listEvents': + case 'fine-tune.listEvents': case 'retrieveFineTune': + case 'fine_tuning.jobs.retrieve': + case 'fine-tune.retrieve': case 'deleteModel': + case 'models.del': case 'cancelFineTune': + case 'fine_tuning.jobs.cancel': + case 'fine-tune.cancel': commonLookupFineTuneRequestExtraction(tags, payload) break case 'createEdit': - createEditRequestExtraction(tags, payload, store) + case 'edits.create': + createEditRequestExtraction(tags, payload, openaiStore) break } span.addTags(tags) + + ctx.currentStore = { ...store, span, openai: openaiStore } + + return ctx.currentStore } - finish ({ headers, body, method, path }) { - const span = this.activeSpan + asyncEnd (ctx) { + const { result } = ctx + const store = ctx.currentStore + + const span = store?.span + if (!span) return + + const error = !!span.context()._tags.error + + let headers, body, method, path + if (!error) { + headers = result.headers + body = result.data + method = result.request.method + path = result.request.path + } + + if (!error && headers?.constructor.name === 'Headers') { + headers = Object.fromEntries(headers) + } const methodName = span._spanContext._tags['resource.name'] body = coerceResponseBody(body, methodName) - const fullStore = storage.getStore() - const store = fullStore.openai + const openaiStore = store.openai + if (!error && (path?.startsWith('https://') || path?.startsWith('http://'))) { + // basic checking for if the path was set as a full URL + // not using a full regex as it will likely be "https://api.openai.com/..." + path = new URL(path).pathname + } const endpoint = lookupOperationEndpoint(methodName, path) - const tags = { - 'openai.request.endpoint': endpoint, - 'openai.request.method': method, + const tags = error + ? {} + : { + 'openai.request.endpoint': endpoint, + 'openai.request.method': method.toUpperCase(), - 'openai.organization.id': body.organization_id, // only available in fine-tunes endpoints - 'openai.organization.name': headers['openai-organization'], + 'openai.organization.id': body.organization_id, // only available in fine-tunes endpoints + 'openai.organization.name': headers['openai-organization'], - 'openai.response.model': headers['openai-model'] || body.model, // specific model, often undefined - 'openai.response.id': body.id, // common creation value, numeric epoch - 'openai.response.deleted': body.deleted, // common boolean field in delete responses + 'openai.response.model': headers['openai-model'] || body.model, // specific model, often undefined + 'openai.response.id': body.id, // common creation value, numeric epoch + 'openai.response.deleted': body.deleted, // common boolean field in delete responses - // The OpenAI API appears to use both created and created_at in different places - // Here we're conciously choosing to surface this inconsistency instead of normalizing - 'openai.response.created': body.created, - 'openai.response.created_at': body.created_at - } + // The OpenAI API appears to use both created and created_at in different places + // Here we're conciously choosing to surface this inconsistency instead of normalizing + 'openai.response.created': body.created, + 'openai.response.created_at': body.created_at + } - responseDataExtractionByMethod(methodName, tags, body, store) + responseDataExtractionByMethod(methodName, tags, body, openaiStore) span.addTags(tags) - super.finish() - this.sendLog(methodName, span, tags, store, false) - this.sendMetrics(headers, body, endpoint, span._duration) + span.finish() + this.sendLog(methodName, span, tags, openaiStore, error) + this.sendMetrics(headers, body, endpoint, span._duration, error, tags) } - error (...args) { - super.error(...args) - - const span = this.activeSpan - const methodName = span._spanContext._tags['resource.name'] + sendMetrics (headers, body, endpoint, duration, error, spanTags) { + const tags = [`error:${Number(!!error)}`] + if (error) { + this.metrics.increment('openai.request.error', 1, tags) + } else { + tags.push(`org:${headers['openai-organization']}`) + tags.push(`endpoint:${endpoint}`) // just "/v1/models", no method + tags.push(`model:${headers['openai-model'] || body.model}`) + } - const fullStore = storage.getStore() - const store = fullStore.openai + this.metrics.distribution('openai.request.duration', duration * 1000, tags) - // We don't know most information about the request when it fails + const promptTokens = spanTags['openai.response.usage.prompt_tokens'] + const promptTokensEstimated = spanTags['openai.response.usage.prompt_tokens_estimated'] - const tags = [`error:1`] - this.metrics.distribution('openai.request.duration', span._duration * 1000, tags) - this.metrics.increment('openai.request.error', 1, tags) + const completionTokens = spanTags['openai.response.usage.completion_tokens'] + const completionTokensEstimated = spanTags['openai.response.usage.completion_tokens_estimated'] - this.sendLog(methodName, span, {}, store, true) - } + const totalTokens = spanTags['openai.response.usage.total_tokens'] - sendMetrics (headers, body, endpoint, duration) { - const tags = [ - `org:${headers['openai-organization']}`, - `endpoint:${endpoint}`, // just "/v1/models", no method - `model:${headers['openai-model']}`, - `error:0` - ] + if (!error) { + if (promptTokens != null) { + if (promptTokensEstimated) { + this.metrics.distribution( + 'openai.tokens.prompt', promptTokens, [...tags, 'openai.estimated:true']) + } else { + this.metrics.distribution('openai.tokens.prompt', promptTokens, tags) + } + } - this.metrics.distribution('openai.request.duration', duration * 1000, tags) + if (completionTokens != null) { + if (completionTokensEstimated) { + this.metrics.distribution( + 'openai.tokens.completion', completionTokens, [...tags, 'openai.estimated:true']) + } else { + this.metrics.distribution('openai.tokens.completion', completionTokens, tags) + } + } - if (body && ('usage' in body)) { - const promptTokens = body.usage.prompt_tokens - const completionTokens = body.usage.completion_tokens - this.metrics.distribution('openai.tokens.prompt', promptTokens, tags) - this.metrics.distribution('openai.tokens.completion', completionTokens, tags) - this.metrics.distribution('openai.tokens.total', promptTokens + completionTokens, tags) + if (totalTokens != null) { + if (promptTokensEstimated || completionTokensEstimated) { + this.metrics.distribution( + 'openai.tokens.total', totalTokens, [...tags, 'openai.estimated:true']) + } else { + this.metrics.distribution('openai.tokens.total', totalTokens, tags) + } + } } - if ('x-ratelimit-limit-requests' in headers) { - this.metrics.gauge('openai.ratelimit.requests', Number(headers['x-ratelimit-limit-requests']), tags) - } + if (headers) { + if (headers['x-ratelimit-limit-requests']) { + this.metrics.gauge('openai.ratelimit.requests', Number(headers['x-ratelimit-limit-requests']), tags) + } - if ('x-ratelimit-remaining-requests' in headers) { - this.metrics.gauge('openai.ratelimit.remaining.requests', Number(headers['x-ratelimit-remaining-requests']), tags) - } + if (headers['x-ratelimit-remaining-requests']) { + this.metrics.gauge( + 'openai.ratelimit.remaining.requests', Number(headers['x-ratelimit-remaining-requests']), tags + ) + } - if ('x-ratelimit-limit-tokens' in headers) { - this.metrics.gauge('openai.ratelimit.tokens', Number(headers['x-ratelimit-limit-tokens']), tags) - } + if (headers['x-ratelimit-limit-tokens']) { + this.metrics.gauge('openai.ratelimit.tokens', Number(headers['x-ratelimit-limit-tokens']), tags) + } - if ('x-ratelimit-remaining-tokens' in headers) { - this.metrics.gauge('openai.ratelimit.remaining.tokens', Number(headers['x-ratelimit-remaining-tokens']), tags) + if (headers['x-ratelimit-remaining-tokens']) { + this.metrics.gauge('openai.ratelimit.remaining.tokens', Number(headers['x-ratelimit-remaining-tokens']), tags) + } } } - sendLog (methodName, span, tags, store, error) { - if (!Object.keys(store).length) return + sendLog (methodName, span, tags, openaiStore, error) { + if (!openaiStore) return + if (!Object.keys(openaiStore).length) return if (!this.sampler.isSampled()) return const log = { status: error ? 'error' : 'info', message: `sampled ${methodName}`, - ...store + ...openaiStore } this.logger.log(log, span, tags) } } -function createEditRequestExtraction (tags, payload, store) { +function countPromptTokens (methodName, payload, model) { + let promptTokens = 0 + let promptEstimated = false + if (methodName === 'chat.completions.create') { + const messages = payload.messages + for (const message of messages) { + const content = message.content + if (typeof content === 'string') { + const { tokens, estimated } = countTokens(content, model) + promptTokens += tokens + promptEstimated = estimated + } else if (Array.isArray(content)) { + for (const c of content) { + if (c.type === 'text') { + const { tokens, estimated } = countTokens(c.text, model) + promptTokens += tokens + promptEstimated = estimated + } + // unsupported token computation for image_url + // as even though URL is a string, its true token count + // is based on the image itself, something onerous to do client-side + } + } + } + } else if (methodName === 'completions.create') { + let prompt = payload.prompt + if (!Array.isArray(prompt)) prompt = [prompt] + + for (const p of prompt) { + const { tokens, estimated } = countTokens(p, model) + promptTokens += tokens + promptEstimated = estimated + } + } + + return { promptTokens, promptEstimated } +} + +function countCompletionTokens (body, model) { + let completionTokens = 0 + let completionEstimated = false + if (body?.choices) { + for (const choice of body.choices) { + const message = choice.message || choice.delta // delta for streamed responses + const text = choice.text + const content = text || message?.content + + const { tokens, estimated } = countTokens(content, model) + completionTokens += tokens + completionEstimated = estimated + } + } + + return { completionTokens, completionEstimated } +} + +function countTokens (content, model) { + if (encodingForModel) { + try { + // try using tiktoken if it was available + const encoder = encodingForModel(model) + const tokens = encoder.encode(content).length + encoder.free() + return { tokens, estimated: false } + } catch { + // possible errors from tiktoken: + // * model not available for token counts + // * issue encoding content + } + } + + return { + tokens: estimateTokens(content), + estimated: true + } +} + +function createEditRequestExtraction (tags, payload, openaiStore) { const instruction = payload.instruction tags['openai.request.instruction'] = instruction - store.instruction = instruction + openaiStore.instruction = instruction } function retrieveModelRequestExtraction (tags, payload) { tags['openai.request.id'] = payload.id } -function createChatCompletionRequestExtraction (tags, payload, store) { - if (!defensiveArrayLength(payload.messages)) return +function createChatCompletionRequestExtraction (tags, payload, openaiStore) { + const messages = payload.messages + if (!defensiveArrayLength(messages)) return - store.messages = payload.messages + openaiStore.messages = payload.messages for (let i = 0; i < payload.messages.length; i++) { const message = payload.messages[i] - tags[`openai.request.${i}.content`] = truncateText(message.content) - tags[`openai.request.${i}.role`] = message.role - tags[`openai.request.${i}.name`] = message.name - tags[`openai.request.${i}.finish_reason`] = message.finish_reason + tagChatCompletionRequestContent(message.content, i, tags) + tags[`openai.request.messages.${i}.role`] = message.role + tags[`openai.request.messages.${i}.name`] = message.name + tags[`openai.request.messages.${i}.finish_reason`] = message.finish_reason } } -function commonCreateImageRequestExtraction (tags, payload, store) { +function commonCreateImageRequestExtraction (tags, payload, openaiStore) { // createImageEdit, createImageVariation - if (payload.file && typeof payload.file === 'object' && payload.file.path) { - const file = path.basename(payload.file.path) + const img = payload.file || payload.image + if (img !== null && typeof img === 'object' && img.path) { + const file = path.basename(img.path) tags['openai.request.image'] = file - store.file = file + openaiStore.file = file } // createImageEdit - if (payload.mask && typeof payload.mask === 'object' && payload.mask.path) { + if (payload.mask !== null && typeof payload.mask === 'object' && payload.mask.path) { const mask = path.basename(payload.mask.path) tags['openai.request.mask'] = mask - store.mask = mask + openaiStore.mask = mask } tags['openai.request.size'] = payload.size @@ -302,63 +465,91 @@ function commonCreateImageRequestExtraction (tags, payload, store) { tags['openai.request.language'] = payload.language } -function responseDataExtractionByMethod (methodName, tags, body, store) { +function responseDataExtractionByMethod (methodName, tags, body, openaiStore) { switch (methodName) { case 'createModeration': + case 'moderations.create': createModerationResponseExtraction(tags, body) break case 'createCompletion': + case 'completions.create': case 'createChatCompletion': + case 'chat.completions.create': case 'createEdit': - commonCreateResponseExtraction(tags, body, store) + case 'edits.create': + commonCreateResponseExtraction(tags, body, openaiStore, methodName) break case 'listFiles': + case 'files.list': case 'listFineTunes': + case 'fine_tuning.jobs.list': + case 'fine-tune.list': case 'listFineTuneEvents': + case 'fine_tuning.jobs.listEvents': + case 'fine-tune.listEvents': commonListCountResponseExtraction(tags, body) break case 'createEmbedding': - createEmbeddingResponseExtraction(tags, body) + case 'embeddings.create': + createEmbeddingResponseExtraction(tags, body, openaiStore) break case 'createFile': + case 'files.create': case 'retrieveFile': + case 'files.retrieve': createRetrieveFileResponseExtraction(tags, body) break case 'deleteFile': + case 'files.del': deleteFileResponseExtraction(tags, body) break case 'downloadFile': + case 'files.retrieveContent': + case 'files.content': downloadFileResponseExtraction(tags, body) break case 'createFineTune': + case 'fine_tuning.jobs.create': + case 'fine-tune.create': case 'retrieveFineTune': + case 'fine_tuning.jobs.retrieve': + case 'fine-tune.retrieve': case 'cancelFineTune': + case 'fine_tuning.jobs.cancel': + case 'fine-tune.cancel': commonFineTuneResponseExtraction(tags, body) break case 'createTranscription': + case 'audio.transcriptions.create': case 'createTranslation': + case 'audio.translations.create': createAudioResponseExtraction(tags, body) break case 'createImage': + case 'images.generate': case 'createImageEdit': + case 'images.edit': case 'createImageVariation': + case 'images.createVariation': commonImageResponseExtraction(tags, body) break case 'listModels': + case 'models.list': listModelsResponseExtraction(tags, body) break case 'retrieveModel': + case 'models.retrieve': retrieveModelResponseExtraction(tags, body) break } @@ -431,15 +622,19 @@ function createFineTuneRequestExtraction (tags, body) { function commonFineTuneResponseExtraction (tags, body) { tags['openai.response.events_count'] = defensiveArrayLength(body.events) tags['openai.response.fine_tuned_model'] = body.fine_tuned_model - if (body.hyperparams) { - tags['openai.response.hyperparams.n_epochs'] = body.hyperparams.n_epochs - tags['openai.response.hyperparams.batch_size'] = body.hyperparams.batch_size - tags['openai.response.hyperparams.prompt_loss_weight'] = body.hyperparams.prompt_loss_weight - tags['openai.response.hyperparams.learning_rate_multiplier'] = body.hyperparams.learning_rate_multiplier + + const hyperparams = body.hyperparams || body.hyperparameters + const hyperparamsKey = body.hyperparams ? 'hyperparams' : 'hyperparameters' + + if (hyperparams) { + tags[`openai.response.${hyperparamsKey}.n_epochs`] = hyperparams.n_epochs + tags[`openai.response.${hyperparamsKey}.batch_size`] = hyperparams.batch_size + tags[`openai.response.${hyperparamsKey}.prompt_loss_weight`] = hyperparams.prompt_loss_weight + tags[`openai.response.${hyperparamsKey}.learning_rate_multiplier`] = hyperparams.learning_rate_multiplier } - tags['openai.response.training_files_count'] = defensiveArrayLength(body.training_files) + tags['openai.response.training_files_count'] = defensiveArrayLength(body.training_files || body.training_file) tags['openai.response.result_files_count'] = defensiveArrayLength(body.result_files) - tags['openai.response.validation_files_count'] = defensiveArrayLength(body.validation_files) + tags['openai.response.validation_files_count'] = defensiveArrayLength(body.validation_files || body.validation_file) tags['openai.response.updated_at'] = body.updated_at tags['openai.response.status'] = body.status } @@ -454,14 +649,14 @@ function deleteFileResponseExtraction (tags, body) { tags['openai.response.id'] = body.id } -function commonCreateAudioRequestExtraction (tags, body, store) { +function commonCreateAudioRequestExtraction (tags, body, openaiStore) { tags['openai.request.response_format'] = body.response_format tags['openai.request.language'] = body.language - if (body.file && typeof body.file === 'object' && body.file.path) { + if (body.file !== null && typeof body.file === 'object' && body.file.path) { const filename = path.basename(body.file.path) tags['openai.request.filename'] = filename - store.file = filename + openaiStore.file = filename } } @@ -471,7 +666,7 @@ function commonFileRequestExtraction (tags, body) { // User can provider either exact file contents or a file read stream // With the stream we extract the filepath // This is a best effort attempt to extract the filename during the request - if (body.file && typeof body.file === 'object' && body.file.path) { + if (body.file !== null && typeof body.file === 'object' && body.file.path) { tags['openai.request.filename'] = path.basename(body.file.path) } } @@ -484,8 +679,8 @@ function createRetrieveFileResponseExtraction (tags, body) { tags['openai.response.status_details'] = body.status_details } -function createEmbeddingResponseExtraction (tags, body) { - usageExtraction(tags, body) +function createEmbeddingResponseExtraction (tags, body, openaiStore) { + usageExtraction(tags, body, openaiStore) if (!body.data) return @@ -519,37 +714,81 @@ function createModerationResponseExtraction (tags, body) { } // createCompletion, createChatCompletion, createEdit -function commonCreateResponseExtraction (tags, body, store) { - usageExtraction(tags, body) +function commonCreateResponseExtraction (tags, body, openaiStore, methodName) { + usageExtraction(tags, body, methodName, openaiStore) if (!body.choices) return tags['openai.response.choices_count'] = body.choices.length - store.choices = body.choices + openaiStore.choices = body.choices + + for (let choiceIdx = 0; choiceIdx < body.choices.length; choiceIdx++) { + const choice = body.choices[choiceIdx] - for (let i = 0; i < body.choices.length; i++) { - const choice = body.choices[i] - tags[`openai.response.choices.${i}.finish_reason`] = choice.finish_reason - tags[`openai.response.choices.${i}.logprobs`] = ('logprobs' in choice) ? 'returned' : undefined - tags[`openai.response.choices.${i}.text`] = truncateText(choice.text) + // logprobs can be null and we still want to tag it as 'returned' even when set to 'null' + const specifiesLogProb = Object.keys(choice).indexOf('logprobs') !== -1 + + tags[`openai.response.choices.${choiceIdx}.finish_reason`] = choice.finish_reason + tags[`openai.response.choices.${choiceIdx}.logprobs`] = specifiesLogProb ? 'returned' : undefined + tags[`openai.response.choices.${choiceIdx}.text`] = truncateText(choice.text) // createChatCompletion only - if ('message' in choice) { - const message = choice.message - tags[`openai.response.choices.${i}.message.role`] = message.role - tags[`openai.response.choices.${i}.message.content`] = truncateText(message.content) - tags[`openai.response.choices.${i}.message.name`] = truncateText(message.name) + const message = choice.message || choice.delta // delta for streamed responses + if (message) { + tags[`openai.response.choices.${choiceIdx}.message.role`] = message.role + tags[`openai.response.choices.${choiceIdx}.message.content`] = truncateText(message.content) + tags[`openai.response.choices.${choiceIdx}.message.name`] = truncateText(message.name) + if (message.tool_calls) { + const toolCalls = message.tool_calls + for (let toolIdx = 0; toolIdx < toolCalls.length; toolIdx++) { + tags[`openai.response.choices.${choiceIdx}.message.tool_calls.${toolIdx}.function.name`] = + toolCalls[toolIdx].function.name + tags[`openai.response.choices.${choiceIdx}.message.tool_calls.${toolIdx}.function.arguments`] = + toolCalls[toolIdx].function.arguments + tags[`openai.response.choices.${choiceIdx}.message.tool_calls.${toolIdx}.id`] = + toolCalls[toolIdx].id + } + } } } } // createCompletion, createChatCompletion, createEdit, createEmbedding -function usageExtraction (tags, body) { - if (typeof body.usage !== 'object' || !body.usage) return - tags['openai.response.usage.prompt_tokens'] = body.usage.prompt_tokens - tags['openai.response.usage.completion_tokens'] = body.usage.completion_tokens - tags['openai.response.usage.total_tokens'] = body.usage.total_tokens +function usageExtraction (tags, body, methodName, openaiStore) { + let promptTokens = 0 + let completionTokens = 0 + let totalTokens = 0 + if (body && body.usage) { + promptTokens = body.usage.prompt_tokens + completionTokens = body.usage.completion_tokens + totalTokens = body.usage.total_tokens + } else if (body.model && ['chat.completions.create', 'completions.create'].includes(methodName)) { + // estimate tokens based on method name for completions and chat completions + const { model } = body + let promptEstimated = false + let completionEstimated = false + + // prompt tokens + const payload = openaiStore + const promptTokensCount = countPromptTokens(methodName, payload, model) + promptTokens = promptTokensCount.promptTokens + promptEstimated = promptTokensCount.promptEstimated + + // completion tokens + const completionTokensCount = countCompletionTokens(body, model) + completionTokens = completionTokensCount.completionTokens + completionEstimated = completionTokensCount.completionEstimated + + // total tokens + totalTokens = promptTokens + completionTokens + if (promptEstimated) tags['openai.response.usage.prompt_tokens_estimated'] = true + if (completionEstimated) tags['openai.response.usage.completion_tokens_estimated'] = true + } + + if (promptTokens != null) tags['openai.response.usage.prompt_tokens'] = promptTokens + if (completionTokens != null) tags['openai.response.usage.completion_tokens'] = completionTokens + if (totalTokens != null) tags['openai.response.usage.total_tokens'] = totalTokens } function truncateApiKey (apiKey) { @@ -561,6 +800,7 @@ function truncateApiKey (apiKey) { */ function truncateText (text) { if (!text) return + if (typeof text !== 'string' || !text || (typeof text === 'string' && text.length === 0)) return text = text .replace(RE_NEWLINE, '\\n') @@ -573,38 +813,88 @@ function truncateText (text) { return text } +function tagChatCompletionRequestContent (contents, messageIdx, tags) { + if (typeof contents === 'string') { + tags[`openai.request.messages.${messageIdx}.content`] = contents + } else if (Array.isArray(contents)) { + // content can also be an array of objects + // which represent text input or image url + for (const contentIdx in contents) { + const content = contents[contentIdx] + const type = content.type + tags[`openai.request.messages.${messageIdx}.content.${contentIdx}.type`] = content.type + if (type === 'text') { + tags[`openai.request.messages.${messageIdx}.content.${contentIdx}.text`] = truncateText(content.text) + } else if (type === 'image_url') { + tags[`openai.request.messages.${messageIdx}.content.${contentIdx}.image_url.url`] = + truncateText(content.image_url.url) + } + // unsupported type otherwise, won't be tagged + } + } + // unsupported type otherwise, won't be tagged +} + // The server almost always responds with JSON function coerceResponseBody (body, methodName) { switch (methodName) { case 'downloadFile': + case 'files.retrieveContent': + case 'files.content': return { file: body } } - return typeof body === 'object' ? body : {} + const type = typeof body + if (type === 'string') { + try { + return JSON.parse(body) + } catch { + return body + } + } else if (type === 'object') { + return body + } else { + return {} + } } // This method is used to replace a dynamic URL segment with an asterisk function lookupOperationEndpoint (operationId, url) { switch (operationId) { case 'deleteModel': + case 'models.del': case 'retrieveModel': + case 'models.retrieve': return '/v1/models/*' case 'deleteFile': + case 'files.del': case 'retrieveFile': + case 'files.retrieve': return '/v1/files/*' case 'downloadFile': + case 'files.retrieveContent': + case 'files.content': return '/v1/files/*/content' case 'retrieveFineTune': + case 'fine-tune.retrieve': return '/v1/fine-tunes/*' + case 'fine_tuning.jobs.retrieve': + return '/v1/fine_tuning/jobs/*' case 'listFineTuneEvents': + case 'fine-tune.listEvents': return '/v1/fine-tunes/*/events' + case 'fine_tuning.jobs.listEvents': + return '/v1/fine_tuning/jobs/*/events' case 'cancelFineTune': + case 'fine-tune.cancel': return '/v1/fine-tunes/*/cancel' + case 'fine_tuning.jobs.cancel': + return '/v1/fine_tuning/jobs/*/cancel' } return url @@ -618,12 +908,17 @@ function lookupOperationEndpoint (operationId, url) { function normalizeRequestPayload (methodName, args) { switch (methodName) { case 'listModels': + case 'models.list': case 'listFiles': + case 'files.list': case 'listFineTunes': + case 'fine_tuning.jobs.list': + case 'fine-tune.list': // no argument return {} case 'retrieveModel': + case 'models.retrieve': return { id: args[0] } case 'createFile': @@ -633,19 +928,30 @@ function normalizeRequestPayload (methodName, args) { } case 'deleteFile': + case 'files.del': case 'retrieveFile': + case 'files.retrieve': case 'downloadFile': + case 'files.retrieveContent': + case 'files.content': return { file_id: args[0] } case 'listFineTuneEvents': + case 'fine_tuning.jobs.listEvents': + case 'fine-tune.listEvents': return { fine_tune_id: args[0], stream: args[1] // undocumented } case 'retrieveFineTune': + case 'fine_tuning.jobs.retrieve': + case 'fine-tune.retrieve': case 'deleteModel': + case 'models.del': case 'cancelFineTune': + case 'fine_tuning.jobs.cancel': + case 'fine-tune.cancel': return { fine_tune_id: args[0] } case 'createImageEdit': @@ -702,7 +1008,16 @@ function normalizeStringOrTokenArray (input, truncate) { } function defensiveArrayLength (maybeArray) { - return Array.isArray(maybeArray) ? maybeArray.length : undefined + if (maybeArray) { + if (Array.isArray(maybeArray)) { + return maybeArray.length + } else { + // case of a singular item (ie body.training_file vs body.training_files) + return 1 + } + } + + return undefined } module.exports = OpenApiPlugin diff --git a/packages/datadog-plugin-openai/src/services.js b/packages/datadog-plugin-openai/src/services.js index 58662f59846..6b0cd7db642 100644 --- a/packages/datadog-plugin-openai/src/services.js +++ b/packages/datadog-plugin-openai/src/services.js @@ -1,6 +1,7 @@ 'use strict' -const { DogStatsDClient, NoopDogStatsDClient } = require('../../dd-trace/src/dogstatsd') +const { DogStatsDClient } = require('../../dd-trace/src/dogstatsd') +const NoopDogStatsDClient = require('../../dd-trace/src/noop/dogstatsd') const { ExternalLogger, NoopExternalLogger } = require('../../dd-trace/src/external-logger/src') const FLUSH_INTERVAL = 10 * 1000 diff --git a/packages/datadog-plugin-openai/src/token-estimator.js b/packages/datadog-plugin-openai/src/token-estimator.js new file mode 100644 index 00000000000..46595f0c2a5 --- /dev/null +++ b/packages/datadog-plugin-openai/src/token-estimator.js @@ -0,0 +1,20 @@ +'use strict' + +// If model is unavailable or tiktoken is not imported, then provide a very rough estimate of the number of tokens +// Approximate using the following assumptions: +// * English text +// * 1 token ~= 4 chars +// * 1 token ~= ¾ words +module.exports.estimateTokens = function (content) { + let estimatedTokens = 0 + if (typeof content === 'string') { + const estimation1 = content.length / 4 + + const matches = content.match(/[\w']+|[.,!?;~@#$%^&*()+/-]/g) + const estimation2 = matches ? matches.length * 0.75 : 0 // in the case of an empty string + estimatedTokens = Math.round((1.5 * estimation1 + 0.5 * estimation2) / 2) + } else if (Array.isArray(content) && typeof content[0] === 'number') { + estimatedTokens = content.length + } + return estimatedTokens +} diff --git a/packages/datadog-plugin-openai/test/index.spec.js b/packages/datadog-plugin-openai/test/index.spec.js index ccf586645c0..8df38a11650 100644 --- a/packages/datadog-plugin-openai/test/index.spec.js +++ b/packages/datadog-plugin-openai/test/index.spec.js @@ -20,13 +20,15 @@ describe('Plugin', () => { let clock let metricStub let externalLoggerStub + let realVersion + let tracer describe('openai', () => { withVersions('openai', 'openai', version => { const moduleRequirePath = `../../../versions/openai@${version}` beforeEach(() => { - require(tracerRequirePath) + tracer = require(tracerRequirePath) }) before(() => { @@ -39,13 +41,26 @@ describe('Plugin', () => { beforeEach(() => { clock = sinon.useFakeTimers() - const { Configuration, OpenAIApi } = require(moduleRequirePath).get() - const configuration = new Configuration({ - apiKey: 'sk-DATADOG-ACCEPTANCE-TESTS' - }) + const requiredModule = require(moduleRequirePath) + const module = requiredModule.get() + realVersion = requiredModule.version() + + if (semver.satisfies(realVersion, '>=4.0.0')) { + const OpenAI = module + + openai = new OpenAI({ + apiKey: 'sk-DATADOG-ACCEPTANCE-TESTS' + }) + } else { + const { Configuration, OpenAIApi } = module - openai = new OpenAIApi(configuration) + const configuration = new Configuration({ + apiKey: 'sk-DATADOG-ACCEPTANCE-TESTS' + }) + + openai = new OpenAIApi(configuration) + } metricStub = sinon.stub(DogStatsDClient.prototype, '_add') externalLoggerStub = sinon.stub(NoopExternalLogger.prototype, 'log') @@ -71,29 +86,160 @@ describe('Plugin', () => { }) }) - describe('createCompletion()', () => { + describe('with error', () => { let scope - after(() => { + beforeEach(() => { + scope = nock('https://api.openai.com:443') + .get('/v1/models') + .reply(400, { + error: { + message: 'fake message', + type: 'fake type', + param: 'fake param', + code: null + } + }) + }) + + afterEach(() => { nock.removeInterceptor(scope) scope.done() }) + it('should attach the error to the span', async () => { + const checkTraces = agent + .use(traces => { + expect(traces[0][0]).to.have.property('error', 1) + // the message content differs on OpenAI version, even between patches + expect(traces[0][0].meta['error.message']).to.exist + expect(traces[0][0].meta).to.have.property('error.type', 'Error') + expect(traces[0][0].meta['error.stack']).to.exist + }) + + try { + if (semver.satisfies(realVersion, '>=4.0.0')) { + await openai.models.list() + } else { + await openai.listModels() + } + } catch { + // ignore, we expect an error + } + + await checkTraces + + clock.tick(10 * 1000) + + const expectedTags = ['error:1'] + + expect(metricStub).to.have.been.calledWith('openai.request.error', 1, 'c', expectedTags) + expect(metricStub).to.have.been.calledWith('openai.request.duration') // timing value not guaranteed + + expect(metricStub).to.not.have.been.calledWith('openai.tokens.prompt') + expect(metricStub).to.not.have.been.calledWith('openai.tokens.completion') + expect(metricStub).to.not.have.been.calledWith('openai.tokens.total') + expect(metricStub).to.not.have.been.calledWith('openai.ratelimit.requests') + expect(metricStub).to.not.have.been.calledWith('openai.ratelimit.tokens') + expect(metricStub).to.not.have.been.calledWith('openai.ratelimit.remaining.requests') + expect(metricStub).to.not.have.been.calledWith('openai.ratelimit.remaining.tokens') + }) + }) + + describe('maintains context', () => { + afterEach(() => { + nock.cleanAll() + }) + + it('should maintain the context with a non-streamed call', async () => { + nock('https://api.openai.com:443') + .post('/v1/completions') + .reply(200, { + id: 'cmpl-7GWDlQbOrAYGmeFZtoRdOEjDXDexM', + object: 'text_completion', + created: 1684171461, + model: 'text-davinci-002', + choices: [{ + text: 'FOO BAR BAZ', + index: 0, + logprobs: null, + finish_reason: 'length' + }], + usage: { prompt_tokens: 3, completion_tokens: 16, total_tokens: 19 } + }) + + await tracer.trace('outer', async (outerSpan) => { + const params = { + model: 'text-davinci-002', + prompt: 'Hello, \n\nFriend\t\tHi' + } + + if (semver.satisfies(realVersion, '>=4.0.0')) { + const result = await openai.completions.create(params) + expect(result.id).to.eql('cmpl-7GWDlQbOrAYGmeFZtoRdOEjDXDexM') + } else { + const result = await openai.createCompletion(params) + expect(result.data.id).to.eql('cmpl-7GWDlQbOrAYGmeFZtoRdOEjDXDexM') + } + + tracer.trace('child of outer', innerSpan => { + expect(innerSpan.context()._parentId).to.equal(outerSpan.context()._spanId) + }) + }) + }) + + if (semver.intersects('>4.1.0', version)) { + it('should maintain the context with a streamed call', async () => { + nock('https://api.openai.com:443') + .post('/v1/chat/completions') + .reply(200, function () { + return fs.createReadStream(Path.join(__dirname, 'streamed-responses/chat.completions.simple.txt')) + }, { + 'Content-Type': 'text/plain', + 'openai-organization': 'kill-9' + }) + + await tracer.trace('outer', async (outerSpan) => { + const stream = await openai.chat.completions.create({ + model: 'gpt-4o', + messages: [{ role: 'user', content: 'Hello, OpenAI!', name: 'hunter2' }], + temperature: 0.5, + stream: true + }) + + for await (const part of stream) { + expect(part).to.have.property('choices') + expect(part.choices[0]).to.have.property('delta') + } + + tracer.trace('child of outer', innerSpan => { + expect(innerSpan.context()._parentId).to.equal(outerSpan.context()._spanId) + }) + }) + }) + } + }) + + describe('create completion', () => { + afterEach(() => { + nock.cleanAll() + }) + it('makes a successful call', async () => { - scope = nock('https://api.openai.com:443') + nock('https://api.openai.com:443') .post('/v1/completions') .reply(200, { - 'id': 'cmpl-7GWDlQbOrAYGmeFZtoRdOEjDXDexM', - 'object': 'text_completion', - 'created': 1684171461, - 'model': 'text-davinci-002', - 'choices': [{ - 'text': 'FOO BAR BAZ', - 'index': 0, - 'logprobs': null, - 'finish_reason': 'length' + id: 'cmpl-7GWDlQbOrAYGmeFZtoRdOEjDXDexM', + object: 'text_completion', + created: 1684171461, + model: 'text-davinci-002', + choices: [{ + text: 'FOO BAR BAZ', + index: 0, + logprobs: null, + finish_reason: 'length' }], - 'usage': { 'prompt_tokens': 3, 'completion_tokens': 16, 'total_tokens': 19 } + usage: { prompt_tokens: 3, completion_tokens: 16, total_tokens: 19 } }, [ 'Date', 'Mon, 15 May 2023 17:24:22 GMT', 'Content-Type', 'application/json', @@ -116,7 +262,11 @@ describe('Plugin', () => { .use(traces => { expect(traces[0][0]).to.have.property('name', 'openai.request') expect(traces[0][0]).to.have.property('type', 'openai') - expect(traces[0][0]).to.have.property('resource', 'createCompletion') + if (semver.satisfies(realVersion, '>=4.0.0')) { + expect(traces[0][0]).to.have.property('resource', 'completions.create') + } else { + expect(traces[0][0]).to.have.property('resource', 'createCompletion') + } expect(traces[0][0]).to.have.property('error', 0) expect(traces[0][0].meta).to.have.property('openai.request.method', 'POST') expect(traces[0][0].meta).to.have.property('openai.request.endpoint', '/v1/completions') @@ -149,7 +299,7 @@ describe('Plugin', () => { expect(traces[0][0].metrics).to.have.property('openai.response.usage.total_tokens', 19) }) - const result = await openai.createCompletion({ + const params = { model: 'text-davinci-002', prompt: 'Hello, \n\nFriend\t\tHi', suffix: 'foo', @@ -164,21 +314,27 @@ describe('Plugin', () => { presence_penalty: -0.1, frequency_penalty: 0.11, best_of: 2, - logit_bias: { '50256': 30 }, + logit_bias: { 50256: 30 }, user: 'hunter2' - }) + } - expect(result.data.id).to.eql('cmpl-7GWDlQbOrAYGmeFZtoRdOEjDXDexM') + if (semver.satisfies(realVersion, '>=4.0.0')) { + const result = await openai.completions.create(params) + expect(result.id).to.eql('cmpl-7GWDlQbOrAYGmeFZtoRdOEjDXDexM') + } else { + const result = await openai.createCompletion(params) + expect(result.data.id).to.eql('cmpl-7GWDlQbOrAYGmeFZtoRdOEjDXDexM') + } await checkTraces clock.tick(10 * 1000) const expectedTags = [ + 'error:0', 'org:kill-9', 'endpoint:/v1/completions', - 'model:text-davinci-002', - 'error:0' + 'model:text-davinci-002' ] expect(metricStub).to.have.been.calledWith('openai.request.duration') // timing value not guaranteed @@ -193,7 +349,9 @@ describe('Plugin', () => { expect(externalLoggerStub).to.have.been.calledWith({ status: 'info', - message: 'sampled createCompletion', + message: semver.satisfies(realVersion, '>=4.0.0') + ? 'sampled completions.create' + : 'sampled createCompletion', prompt: 'Hello, \n\nFriend\t\tHi', choices: [ { @@ -207,7 +365,7 @@ describe('Plugin', () => { }) it('should not throw with empty response body', async () => { - scope = nock('https://api.openai.com:443') + nock('https://api.openai.com:443') .post('/v1/completions') .reply(200, {}, [ 'Date', 'Mon, 15 May 2023 17:24:22 GMT', @@ -232,12 +390,17 @@ describe('Plugin', () => { expect(traces[0][0]).to.have.property('name', 'openai.request') }) - await openai.createCompletion({ + const params = { model: 'text-davinci-002', prompt: 'Hello, ', - suffix: 'foo', - stream: true - }) + suffix: 'foo' + } + + if (semver.satisfies(realVersion, '>=4.0.0')) { + await openai.completions.create(params) + } else { + await openai.createCompletion(params) + } await checkTraces @@ -245,23 +408,25 @@ describe('Plugin', () => { }) }) - describe('createEmbedding()', () => { - let scope + describe('create embedding with stream:true', () => { + after(() => { + nock.cleanAll() + }) - before(() => { - scope = nock('https://api.openai.com:443') + it('makes a successful call', async () => { + nock('https://api.openai.com:443') .post('/v1/embeddings') .reply(200, { - 'object': 'list', - 'data': [{ - 'object': 'embedding', - 'index': 0, - 'embedding': [-0.0034387498, -0.026400521] + object: 'list', + data: [{ + object: 'embedding', + index: 0, + embedding: [-0.0034387498, -0.026400521] }], - 'model': 'text-embedding-ada-002-v2', - 'usage': { - 'prompt_tokens': 2, - 'total_tokens': 2 + model: 'text-embedding-ada-002-v2', + usage: { + prompt_tokens: 2, + total_tokens: 2 } }, [ 'Date', 'Mon, 15 May 2023 20:49:06 GMT', @@ -272,19 +437,16 @@ describe('Plugin', () => { 'openai-processing-ms', '344', 'openai-version', '2020-10-01' ]) - }) - - after(() => { - nock.removeInterceptor(scope) - scope.done() - }) - it('makes a successful call', async () => { const checkTraces = agent .use(traces => { expect(traces[0][0]).to.have.property('name', 'openai.request') expect(traces[0][0]).to.have.property('type', 'openai') - expect(traces[0][0]).to.have.property('resource', 'createEmbedding') + if (semver.satisfies(realVersion, '>=4.0.0')) { + expect(traces[0][0]).to.have.property('resource', 'embeddings.create') + } else { + expect(traces[0][0]).to.have.property('resource', 'createEmbedding') + } expect(traces[0][0]).to.have.property('error', 0) expect(traces[0][0].meta).to.have.property('openai.request.endpoint', '/v1/embeddings') expect(traces[0][0].meta).to.have.property('openai.request.method', 'POST') @@ -300,76 +462,207 @@ describe('Plugin', () => { expect(traces[0][0].metrics).to.have.property('openai.response.usage.total_tokens', 2) }) - const result = await openai.createEmbedding({ + const params = { model: 'text-embedding-ada-002', input: 'Cat?', user: 'hunter2' + } + + if (semver.satisfies(realVersion, '>=4.0.0')) { + const result = await openai.embeddings.create(params) + expect(result.model).to.eql('text-embedding-ada-002-v2') + } else { + const result = await openai.createEmbedding(params) + expect(result.data.model).to.eql('text-embedding-ada-002-v2') + } + + await checkTraces + + expect(externalLoggerStub).to.have.been.calledWith({ + status: 'info', + message: semver.satisfies(realVersion, '>=4.0.0') ? 'sampled embeddings.create' : 'sampled createEmbedding', + input: 'Cat?' }) + }) + + it('makes a successful call with stream true', async () => { + // Testing that adding stream:true to the params doesn't break the instrumentation + nock('https://api.openai.com:443') + .post('/v1/embeddings') + .reply(200, { + object: 'list', + data: [{ + object: 'embedding', + index: 0, + embedding: [-0.0034387498, -0.026400521] + }], + model: 'text-embedding-ada-002-v2', + usage: { + prompt_tokens: 2, + total_tokens: 2 + } + }, [ + 'Date', 'Mon, 15 May 2023 20:49:06 GMT', + 'Content-Type', 'application/json', + 'Content-Length', '75', + 'access-control-allow-origin', '*', + 'openai-organization', 'kill-9', + 'openai-processing-ms', '344', + 'openai-version', '2020-10-01' + ]) + + const checkTraces = agent + .use(traces => { + expect(traces[0][0]).to.have.property('name', 'openai.request') + expect(traces[0][0]).to.have.property('type', 'openai') + if (semver.satisfies(realVersion, '>=4.0.0')) { + expect(traces[0][0]).to.have.property('resource', 'embeddings.create') + } else { + expect(traces[0][0]).to.have.property('resource', 'createEmbedding') + } + expect(traces[0][0]).to.have.property('error', 0) + expect(traces[0][0].meta).to.have.property('openai.request.endpoint', '/v1/embeddings') + expect(traces[0][0].meta).to.have.property('openai.request.method', 'POST') - expect(result.data.model).to.eql('text-embedding-ada-002-v2') + expect(traces[0][0].meta).to.have.property('openai.organization.name', 'kill-9') + expect(traces[0][0].meta).to.have.property('openai.request.input', 'Cat?') + expect(traces[0][0].meta).to.have.property('openai.request.model', 'text-embedding-ada-002') + expect(traces[0][0].meta).to.have.property('openai.request.user', 'hunter2') + expect(traces[0][0].meta).to.have.property('openai.response.model', 'text-embedding-ada-002-v2') + expect(traces[0][0].metrics).to.have.property('openai.response.embeddings_count', 1) + expect(traces[0][0].metrics).to.have.property('openai.response.embedding.0.embedding_length', 2) + expect(traces[0][0].metrics).to.have.property('openai.response.usage.prompt_tokens', 2) + expect(traces[0][0].metrics).to.have.property('openai.response.usage.total_tokens', 2) + }) + + const params = { + model: 'text-embedding-ada-002', + input: 'Cat?', + user: 'hunter2', + stream: true + } + + if (semver.satisfies(realVersion, '>=4.0.0')) { + const result = await openai.embeddings.create(params) + expect(result.model).to.eql('text-embedding-ada-002-v2') + } else { + const result = await openai.createEmbedding(params) + expect(result.data.model).to.eql('text-embedding-ada-002-v2') + } + + await checkTraces expect(externalLoggerStub).to.have.been.calledWith({ status: 'info', - message: 'sampled createEmbedding', + message: semver.satisfies(realVersion, '>=4.0.0') ? 'sampled embeddings.create' : 'sampled createEmbedding', input: 'Cat?' }) + }) + }) + + describe('embedding with missing usages', () => { + afterEach(() => { + nock.cleanAll() + }) + + it('makes a successful call', async () => { + nock('https://api.openai.com:443') + .post('/v1/embeddings') + .reply(200, { + object: 'list', + data: [{ + object: 'embedding', + index: 0, + embedding: [-0.0034387498, -0.026400521] + }], + model: 'text-embedding-ada-002-v2', + usage: { + prompt_tokens: 0 + } + }, []) + + const checkTraces = agent + .use(traces => { + expect(traces[0][0].metrics).to.have.property('openai.response.usage.prompt_tokens', 0) + expect(traces[0][0].metrics).to.not.have.property('openai.response.usage.completion_tokens') + expect(traces[0][0].metrics).to.not.have.property('openai.response.usage.total_tokens') + }) + + const params = { + model: 'text-embedding-ada-002', + input: '', + user: 'hunter2' + } + + if (semver.satisfies(realVersion, '>=4.0.0')) { + const result = await openai.embeddings.create(params) + expect(result.model).to.eql('text-embedding-ada-002-v2') + } else { + const result = await openai.createEmbedding(params) + expect(result.data.model).to.eql('text-embedding-ada-002-v2') + } await checkTraces + + expect(metricStub).to.have.been.calledWith('openai.request.duration') // timing value not guaranteed + expect(metricStub).to.have.been.calledWith('openai.tokens.prompt') + expect(metricStub).to.not.have.been.calledWith('openai.tokens.completion') + expect(metricStub).to.not.have.been.calledWith('openai.tokens.total') }) }) - describe('listModels()', () => { + describe('list models', () => { let scope before(() => { scope = nock('https://api.openai.com:443') .get('/v1/models') .reply(200, { - 'object': 'list', - 'data': [ + object: 'list', + data: [ { - 'id': 'whisper-1', - 'object': 'model', - 'created': 1677532384, - 'owned_by': 'openai-internal', - 'permission': [{ - 'id': 'modelperm-KlsZlfft3Gma8pI6A8rTnyjs', - 'object': 'model_permission', - 'created': 1683912666, - 'allow_create_engine': false, - 'allow_sampling': true, - 'allow_logprobs': true, - 'allow_search_indices': false, - 'allow_view': true, - 'allow_fine_tuning': false, - 'organization': '*', - 'group': null, - 'is_blocking': false + id: 'whisper-1', + object: 'model', + created: 1677532384, + owned_by: 'openai-internal', + permission: [{ + id: 'modelperm-KlsZlfft3Gma8pI6A8rTnyjs', + object: 'model_permission', + created: 1683912666, + allow_create_engine: false, + allow_sampling: true, + allow_logprobs: true, + allow_search_indices: false, + allow_view: true, + allow_fine_tuning: false, + organization: '*', + group: null, + is_blocking: false }], - 'root': 'whisper-1', - 'parent': null + root: 'whisper-1', + parent: null }, { - 'id': 'babbage', - 'object': 'model', - 'created': 1649358449, - 'owned_by': 'openai', - 'permission': [{ - 'id': 'modelperm-49FUp5v084tBB49tC4z8LPH5', - 'object': 'model_permission', - 'created': 1669085501, - 'allow_create_engine': false, - 'allow_sampling': true, - 'allow_logprobs': true, - 'allow_search_indices': false, - 'allow_view': true, - 'allow_fine_tuning': false, - 'organization': '*', - 'group': null, - 'is_blocking': false + id: 'babbage', + object: 'model', + created: 1649358449, + owned_by: 'openai', + permission: [{ + id: 'modelperm-49FUp5v084tBB49tC4z8LPH5', + object: 'model_permission', + created: 1669085501, + allow_create_engine: false, + allow_sampling: true, + allow_logprobs: true, + allow_search_indices: false, + allow_view: true, + allow_fine_tuning: false, + organization: '*', + group: null, + is_blocking: false }], - 'root': 'babbage', - 'parent': null + root: 'babbage', + parent: null } ] }, [ @@ -392,7 +685,11 @@ describe('Plugin', () => { .use(traces => { expect(traces[0][0]).to.have.property('name', 'openai.request') expect(traces[0][0]).to.have.property('type', 'openai') - expect(traces[0][0]).to.have.property('resource', 'listModels') + if (semver.satisfies(realVersion, '>=4.0.0')) { + expect(traces[0][0]).to.have.property('resource', 'models.list') + } else { + expect(traces[0][0]).to.have.property('resource', 'listModels') + } expect(traces[0][0]).to.have.property('error', 0) expect(traces[0][0].meta).to.have.property('openai.request.method', 'GET') expect(traces[0][0].meta).to.have.property('openai.request.endpoint', '/v1/models') @@ -401,42 +698,47 @@ describe('Plugin', () => { // Note that node doesn't accept a user value }) - const result = await openai.listModels() - - expect(result.data.object).to.eql('list') - expect(result.data.data.length).to.eql(2) + if (semver.satisfies(realVersion, '>=4.0.0')) { + const result = await openai.models.list() + expect(result.object).to.eql('list') + expect(result.data.length).to.eql(2) + } else { + const result = await openai.listModels() + expect(result.data.object).to.eql('list') + expect(result.data.data.length).to.eql(2) + } await checkTraces }) }) - describe('retrieveModel()', () => { + describe('retrieve model', () => { let scope before(() => { scope = nock('https://api.openai.com:443') .get('/v1/models/gpt-4') .reply(200, { - 'id': 'gpt-4', - 'object': 'model', - 'created': 1678604602, - 'owned_by': 'openai', - 'permission': [{ - 'id': 'modelperm-ffiDrbtOGIZuczdJcFuOo2Mi', - 'object': 'model_permission', - 'created': 1684185078, - 'allow_create_engine': false, - 'allow_sampling': false, - 'allow_logprobs': false, - 'allow_search_indices': false, - 'allow_view': false, - 'allow_fine_tuning': false, - 'organization': '*', - 'group': null, - 'is_blocking': false + id: 'gpt-4', + object: 'model', + created: 1678604602, + owned_by: 'openai', + permission: [{ + id: 'modelperm-ffiDrbtOGIZuczdJcFuOo2Mi', + object: 'model_permission', + created: 1684185078, + allow_create_engine: false, + allow_sampling: false, + allow_logprobs: false, + allow_search_indices: false, + allow_view: false, + allow_fine_tuning: false, + organization: '*', + group: null, + is_blocking: false }], - 'root': 'gpt-4', - 'parent': 'stevebob' + root: 'gpt-4', + parent: 'stevebob' }, [ 'Date', 'Mon, 15 May 2023 23:41:40 GMT', 'Content-Type', 'application/json', @@ -457,7 +759,11 @@ describe('Plugin', () => { .use(traces => { expect(traces[0][0]).to.have.property('name', 'openai.request') expect(traces[0][0]).to.have.property('type', 'openai') - expect(traces[0][0]).to.have.property('resource', 'retrieveModel') + if (semver.satisfies(realVersion, '>=4.0.0')) { + expect(traces[0][0]).to.have.property('resource', 'models.retrieve') + } else { + expect(traces[0][0]).to.have.property('resource', 'retrieveModel') + } expect(traces[0][0]).to.have.property('error', 0) expect(traces[0][0].meta).to.have.property('openai.request.method', 'GET') expect(traces[0][0].meta).to.have.property('openai.request.endpoint', '/v1/models/*') @@ -480,31 +786,37 @@ describe('Plugin', () => { expect(traces[0][0].metrics).to.have.property('openai.response.permission.is_blocking', 0) }) - const result = await openai.retrieveModel('gpt-4') + if (semver.satisfies(realVersion, '>=4.0.0')) { + const result = await openai.models.retrieve('gpt-4') - expect(result.data.id).to.eql('gpt-4') + expect(result.id).to.eql('gpt-4') + } else { + const result = await openai.retrieveModel('gpt-4') + + expect(result.data.id).to.eql('gpt-4') + } await checkTraces }) }) - describe('createEdit()', () => { + describe('create edit', () => { let scope before(() => { scope = nock('https://api.openai.com:443') .post('/v1/edits') .reply(200, { - 'object': 'edit', - 'created': 1684267309, - 'choices': [{ - 'text': 'What day of the week is it, Bob?\n', - 'index': 0 + object: 'edit', + created: 1684267309, + choices: [{ + text: 'What day of the week is it, Bob?\n', + index: 0 }], - 'usage': { - 'prompt_tokens': 25, - 'completion_tokens': 28, - 'total_tokens': 53 + usage: { + prompt_tokens: 25, + completion_tokens: 28, + total_tokens: 53 } }, [ 'Date', 'Tue, 16 May 2023 20:01:49 GMT', @@ -528,98 +840,101 @@ describe('Plugin', () => { scope.done() }) - it('makes a successful call', async () => { - const checkTraces = agent - .use(traces => { - expect(traces[0][0]).to.have.property('name', 'openai.request') - expect(traces[0][0]).to.have.property('type', 'openai') - expect(traces[0][0]).to.have.property('resource', 'createEdit') - expect(traces[0][0]).to.have.property('error', 0) - expect(traces[0][0].meta).to.have.property('openai.organization.name', 'kill-9') - expect(traces[0][0].meta).to.have.property('openai.request.method', 'POST') - expect(traces[0][0].meta).to.have.property('openai.request.endpoint', '/v1/edits') + if (semver.satisfies(realVersion, '<4.0.0')) { + // `edits.create` was deprecated and removed after 4.0.0 + it('makes a successful call', async () => { + const checkTraces = agent + .use(traces => { + expect(traces[0][0]).to.have.property('name', 'openai.request') + expect(traces[0][0]).to.have.property('type', 'openai') + expect(traces[0][0]).to.have.property('resource', 'createEdit') + expect(traces[0][0]).to.have.property('error', 0) + expect(traces[0][0].meta).to.have.property('openai.organization.name', 'kill-9') + expect(traces[0][0].meta).to.have.property('openai.request.method', 'POST') + expect(traces[0][0].meta).to.have.property('openai.request.endpoint', '/v1/edits') - expect(traces[0][0].meta).to.have.property('openai.request.input', 'What day of the wek is it?') - expect(traces[0][0].meta).to.have.property('openai.request.instruction', 'Fix the spelling mistakes') - expect(traces[0][0].meta).to.have.property('openai.request.model', 'text-davinci-edit-001') - expect(traces[0][0].meta).to.have.property('openai.request.user', 'hunter2') - expect(traces[0][0].meta).to.have.property('openai.response.choices.0.text', - 'What day of the week is it, Bob?\\n') - expect(traces[0][0].metrics).to.have.property('openai.request.n', 1) - expect(traces[0][0].metrics).to.have.property('openai.request.temperature', 1.00001) - expect(traces[0][0].metrics).to.have.property('openai.request.top_p', 0.999) - expect(traces[0][0].metrics).to.have.property('openai.response.choices_count', 1) - expect(traces[0][0].metrics).to.have.property('openai.response.created', 1684267309) - expect(traces[0][0].metrics).to.have.property('openai.response.usage.completion_tokens', 28) - expect(traces[0][0].metrics).to.have.property('openai.response.usage.prompt_tokens', 25) - expect(traces[0][0].metrics).to.have.property('openai.response.usage.total_tokens', 53) - }) + expect(traces[0][0].meta).to.have.property('openai.request.input', 'What day of the wek is it?') + expect(traces[0][0].meta).to.have.property('openai.request.instruction', 'Fix the spelling mistakes') + expect(traces[0][0].meta).to.have.property('openai.request.model', 'text-davinci-edit-001') + expect(traces[0][0].meta).to.have.property('openai.request.user', 'hunter2') + expect(traces[0][0].meta).to.have.property('openai.response.choices.0.text', + 'What day of the week is it, Bob?\\n') + expect(traces[0][0].metrics).to.have.property('openai.request.n', 1) + expect(traces[0][0].metrics).to.have.property('openai.request.temperature', 1.00001) + expect(traces[0][0].metrics).to.have.property('openai.request.top_p', 0.999) + expect(traces[0][0].metrics).to.have.property('openai.response.choices_count', 1) + expect(traces[0][0].metrics).to.have.property('openai.response.created', 1684267309) + expect(traces[0][0].metrics).to.have.property('openai.response.usage.completion_tokens', 28) + expect(traces[0][0].metrics).to.have.property('openai.response.usage.prompt_tokens', 25) + expect(traces[0][0].metrics).to.have.property('openai.response.usage.total_tokens', 53) + }) - const result = await openai.createEdit({ - 'model': 'text-davinci-edit-001', - 'input': 'What day of the wek is it?', - 'instruction': 'Fix the spelling mistakes', - 'n': 1, - 'temperature': 1.00001, - 'top_p': 0.999, - 'user': 'hunter2' - }) + const result = await openai.createEdit({ + model: 'text-davinci-edit-001', + input: 'What day of the wek is it?', + instruction: 'Fix the spelling mistakes', + n: 1, + temperature: 1.00001, + top_p: 0.999, + user: 'hunter2' + }) - expect(result.data.choices[0].text).to.eql('What day of the week is it, Bob?\n') + expect(result.data.choices[0].text).to.eql('What day of the week is it, Bob?\n') - clock.tick(10 * 1000) + clock.tick(10 * 1000) - await checkTraces + await checkTraces - const expectedTags = [ - 'org:kill-9', - 'endpoint:/v1/edits', - 'model:text-davinci-edit:001', - 'error:0' - ] + const expectedTags = [ + 'error:0', + 'org:kill-9', + 'endpoint:/v1/edits', + 'model:text-davinci-edit:001' + ] - expect(metricStub).to.be.calledWith('openai.ratelimit.requests', 20, 'g', expectedTags) - expect(metricStub).to.be.calledWith('openai.ratelimit.remaining.requests', 19, 'g', expectedTags) + expect(metricStub).to.be.calledWith('openai.ratelimit.requests', 20, 'g', expectedTags) + expect(metricStub).to.be.calledWith('openai.ratelimit.remaining.requests', 19, 'g', expectedTags) - expect(externalLoggerStub).to.have.been.calledWith({ - status: 'info', - message: 'sampled createEdit', - input: 'What day of the wek is it?', - instruction: 'Fix the spelling mistakes', - choices: [{ - text: 'What day of the week is it, Bob?\n', - index: 0 - }] + expect(externalLoggerStub).to.have.been.calledWith({ + status: 'info', + message: semver.satisfies(realVersion, '>=4.0.0') ? 'sampled edits.create' : 'sampled createEdit', + input: 'What day of the wek is it?', + instruction: 'Fix the spelling mistakes', + choices: [{ + text: 'What day of the week is it, Bob?\n', + index: 0 + }] + }) }) - }) + } }) - describe('listFiles()', () => { + describe('list files', () => { let scope before(() => { scope = nock('https://api.openai.com:443') .get('/v1/files') .reply(200, { - 'object': 'list', - 'data': [{ - 'object': 'file', - 'id': 'file-foofoofoo', - 'purpose': 'fine-tune-results', - 'filename': 'compiled_results.csv', - 'bytes': 3460, - 'created_at': 1684000162, - 'status': 'processed', - 'status_details': null + object: 'list', + data: [{ + object: 'file', + id: 'file-foofoofoo', + purpose: 'fine-tune-results', + filename: 'compiled_results.csv', + bytes: 3460, + created_at: 1684000162, + status: 'processed', + status_details: null }, { - 'object': 'file', - 'id': 'file-barbarbar', - 'purpose': 'fine-tune-results', - 'filename': 'compiled_results.csv', - 'bytes': 13595, - 'created_at': 1684000508, - 'status': 'processed', - 'status_details': null + object: 'file', + id: 'file-barbarbar', + purpose: 'fine-tune-results', + filename: 'compiled_results.csv', + bytes: 13595, + created_at: 1684000508, + status: 'processed', + status_details: null }] }, [ 'Date', 'Wed, 17 May 2023 21:34:04 GMT', @@ -642,7 +957,11 @@ describe('Plugin', () => { .use(traces => { expect(traces[0][0]).to.have.property('name', 'openai.request') expect(traces[0][0]).to.have.property('type', 'openai') - expect(traces[0][0]).to.have.property('resource', 'listFiles') + if (semver.satisfies(realVersion, '>=4.0.0')) { + expect(traces[0][0]).to.have.property('resource', 'files.list') + } else { + expect(traces[0][0]).to.have.property('resource', 'listFiles') + } expect(traces[0][0]).to.have.property('error', 0) expect(traces[0][0].meta).to.have.property('openai.organization.name', 'kill-9') @@ -651,30 +970,37 @@ describe('Plugin', () => { expect(traces[0][0].metrics).to.have.property('openai.response.count', 2) }) - const result = await openai.listFiles() + if (semver.satisfies(realVersion, '>=4.0.0')) { + const result = await openai.files.list() + + expect(result.data.length).to.eql(2) + expect(result.data[0].id).to.eql('file-foofoofoo') + } else { + const result = await openai.listFiles() - expect(result.data.data.length).to.eql(2) - expect(result.data.data[0].id).to.eql('file-foofoofoo') + expect(result.data.data.length).to.eql(2) + expect(result.data.data[0].id).to.eql('file-foofoofoo') + } await checkTraces }) }) - describe('createFile()', () => { + describe('create file', () => { let scope before(() => { scope = nock('https://api.openai.com:443') .post('/v1/files') .reply(200, { - 'object': 'file', - 'id': 'file-268aYWYhvxWwHb4nIzP9FHM6', - 'purpose': 'fine-tune', - 'filename': 'dave-hal.jsonl', - 'bytes': 356, - 'created_at': 1684362764, - 'status': 'uploaded', - 'status_details': 'foo' // dummy value for testing + object: 'file', + id: 'file-268aYWYhvxWwHb4nIzP9FHM6', + purpose: 'fine-tune', + filename: 'dave-hal.jsonl', + bytes: 356, + created_at: 1684362764, + status: 'uploaded', + status_details: 'foo' // dummy value for testing }, [ 'Date', 'Wed, 17 May 2023 22:32:44 GMT', 'Content-Type', 'application/json', @@ -696,7 +1022,11 @@ describe('Plugin', () => { .use(traces => { expect(traces[0][0]).to.have.property('name', 'openai.request') expect(traces[0][0]).to.have.property('type', 'openai') - expect(traces[0][0]).to.have.property('resource', 'createFile') + if (semver.satisfies(realVersion, '>=4.0.0')) { + expect(traces[0][0]).to.have.property('resource', 'files.create') + } else { + expect(traces[0][0]).to.have.property('resource', 'createFile') + } expect(traces[0][0]).to.have.property('error', 0) expect(traces[0][0].meta).to.have.property('openai.organization.name', 'kill-9') expect(traces[0][0].meta).to.have.property('openai.request.endpoint', '/v1/files') @@ -713,25 +1043,34 @@ describe('Plugin', () => { expect(traces[0][0].metrics).to.have.property('openai.response.created_at', 1684362764) }) - const result = await openai.createFile(fs.createReadStream( - Path.join(__dirname, 'dave-hal.jsonl')), 'fine-tune') + if (semver.satisfies(realVersion, '>=4.0.0')) { + const result = await openai.files.create({ + file: fs.createReadStream(Path.join(__dirname, 'dave-hal.jsonl')), + purpose: 'fine-tune' + }) + + expect(result.filename).to.eql('dave-hal.jsonl') + } else { + const result = await openai.createFile(fs.createReadStream( + Path.join(__dirname, 'dave-hal.jsonl')), 'fine-tune') - expect(result.data.filename).to.eql('dave-hal.jsonl') + expect(result.data.filename).to.eql('dave-hal.jsonl') + } await checkTraces }) }) - describe('deleteFile()', () => { + describe('delete file', () => { let scope before(() => { scope = nock('https://api.openai.com:443') .delete('/v1/files/file-268aYWYhvxWwHb4nIzP9FHM6') .reply(200, { - 'object': 'file', - 'id': 'file-268aYWYhvxWwHb4nIzP9FHM6', - 'deleted': true + object: 'file', + id: 'file-268aYWYhvxWwHb4nIzP9FHM6', + deleted: true }, [ 'Date', 'Wed, 17 May 2023 23:03:54 GMT', 'Content-Type', 'application/json', @@ -752,7 +1091,11 @@ describe('Plugin', () => { .use(traces => { expect(traces[0][0]).to.have.property('name', 'openai.request') expect(traces[0][0]).to.have.property('type', 'openai') - expect(traces[0][0]).to.have.property('resource', 'deleteFile') + if (semver.satisfies(realVersion, '>=4.0.0')) { + expect(traces[0][0]).to.have.property('resource', 'files.del') + } else { + expect(traces[0][0]).to.have.property('resource', 'deleteFile') + } expect(traces[0][0]).to.have.property('error', 0) expect(traces[0][0].meta).to.have.property('openai.organization.name', 'kill-9') expect(traces[0][0].meta).to.have.property('openai.request.method', 'DELETE') @@ -763,29 +1106,35 @@ describe('Plugin', () => { expect(traces[0][0].metrics).to.have.property('openai.response.deleted', 1) }) - const result = await openai.deleteFile('file-268aYWYhvxWwHb4nIzP9FHM6') + if (semver.satisfies(realVersion, '>=4.0.0')) { + const result = await openai.files.del('file-268aYWYhvxWwHb4nIzP9FHM6') - expect(result.data.deleted).to.eql(true) + expect(result.deleted).to.eql(true) + } else { + const result = await openai.deleteFile('file-268aYWYhvxWwHb4nIzP9FHM6') + + expect(result.data.deleted).to.eql(true) + } await checkTraces }) }) - describe('retrieveFile()', () => { + describe('retrieve file', () => { let scope before(() => { scope = nock('https://api.openai.com:443') .get('/v1/files/file-fIkEUgQPWnVXNKPJsr4pEWiz') .reply(200, { - 'object': 'file', - 'id': 'file-fIkEUgQPWnVXNKPJsr4pEWiz', - 'purpose': 'fine-tune', - 'filename': 'dave-hal.jsonl', - 'bytes': 356, - 'created_at': 1684362764, - 'status': 'uploaded', - 'status_details': 'foo' // dummy value for testing + object: 'file', + id: 'file-fIkEUgQPWnVXNKPJsr4pEWiz', + purpose: 'fine-tune', + filename: 'dave-hal.jsonl', + bytes: 356, + created_at: 1684362764, + status: 'uploaded', + status_details: 'foo' // dummy value for testing }, [ 'Date', 'Wed, 17 May 2023 23:14:02 GMT', 'Content-Type', 'application/json', @@ -807,7 +1156,11 @@ describe('Plugin', () => { .use(traces => { expect(traces[0][0]).to.have.property('name', 'openai.request') expect(traces[0][0]).to.have.property('type', 'openai') - expect(traces[0][0]).to.have.property('resource', 'retrieveFile') + if (semver.satisfies(realVersion, '>=4.0.0')) { + expect(traces[0][0]).to.have.property('resource', 'files.retrieve') + } else { + expect(traces[0][0]).to.have.property('resource', 'retrieveFile') + } expect(traces[0][0]).to.have.property('error', 0) expect(traces[0][0].meta).to.have.property('openai.organization.name', 'kill-9') expect(traces[0][0].meta).to.have.property('openai.request.method', 'GET') @@ -823,15 +1176,21 @@ describe('Plugin', () => { expect(traces[0][0].metrics).to.have.property('openai.response.created_at', 1684362764) }) - const result = await openai.retrieveFile('file-fIkEUgQPWnVXNKPJsr4pEWiz') + if (semver.satisfies(realVersion, '>=4.0.0')) { + const result = await openai.files.retrieve('file-fIkEUgQPWnVXNKPJsr4pEWiz') - expect(result.data.filename).to.eql('dave-hal.jsonl') + expect(result.filename).to.eql('dave-hal.jsonl') + } else { + const result = await openai.retrieveFile('file-fIkEUgQPWnVXNKPJsr4pEWiz') + + expect(result.data.filename).to.eql('dave-hal.jsonl') + } await checkTraces }) }) - describe('downloadFile()', () => { + describe('download file', () => { let scope before(() => { @@ -854,75 +1213,111 @@ describe('Plugin', () => { scope.done() }) + // TODO: issues with content being async arraybuffer, how to compute byteLength before promise resolves? it('makes a successful call', async () => { const checkTraces = agent .use(traces => { expect(traces[0][0]).to.have.property('name', 'openai.request') expect(traces[0][0]).to.have.property('type', 'openai') - expect(traces[0][0]).to.have.property('resource', 'downloadFile') + if (semver.satisfies(realVersion, '>=4.0.0 <4.17.1')) { + expect(traces[0][0]).to.have.property('resource', 'files.retrieveContent') + } else if (semver.satisfies(realVersion, '>=4.17.1')) { + expect(traces[0][0]).to.have.property('resource', 'files.content') + } else { + expect(traces[0][0]).to.have.property('resource', 'downloadFile') + } expect(traces[0][0]).to.have.property('error', 0) expect(traces[0][0].meta).to.have.property('openai.organization.name', 'kill-9') expect(traces[0][0].meta).to.have.property('openai.request.method', 'GET') expect(traces[0][0].meta).to.have.property('openai.request.endpoint', '/v1/files/*/content') expect(traces[0][0].meta).to.have.property('openai.request.file_id', 'file-t3k1gVSQDHrfZnPckzftlZ4A') - expect(traces[0][0].metrics).to.have.property('openai.response.total_bytes', 88) + if (semver.satisfies(realVersion, '<4.17.0')) { + expect(traces[0][0].metrics).to.have.property('openai.response.total_bytes', 88) + } }) - const result = await openai.downloadFile('file-t3k1gVSQDHrfZnPckzftlZ4A') + if (semver.satisfies(realVersion, '>=4.0.0 < 4.17.1')) { + const result = await openai.files.retrieveContent('file-t3k1gVSQDHrfZnPckzftlZ4A') + + expect(result) + .to.eql('{"prompt": "foo?", "completion": "bar."}\n{"prompt": "foofoo?", "completion": "barbar."}\n') + } else if (semver.satisfies(realVersion, '>=4.17.1')) { + const result = await openai.files.content('file-t3k1gVSQDHrfZnPckzftlZ4A') - /** - * TODO: Seems like an OpenAI library bug? - * downloading single line JSONL file results in the JSON being converted into an object. - * downloading multi-line JSONL file then provides a basic string. - * This suggests the library is doing `try { return JSON.parse(x) } catch { return x }` - */ - expect(result.data[0]).to.eql('{') // raw JSONL file + expect(result.constructor.name).to.eql('Response') + } else { + const result = await openai.downloadFile('file-t3k1gVSQDHrfZnPckzftlZ4A') + + /** + * TODO: Seems like an OpenAI library bug? + * downloading single line JSONL file results in the JSON being converted into an object. + * downloading multi-line JSONL file then provides a basic string. + * This suggests the library is doing `try { return JSON.parse(x) } catch { return x }` + */ + expect(result.data[0]).to.eql('{') // raw JSONL file + } await checkTraces }) }) - describe('createFineTune()', () => { + describe('create finetune', () => { let scope beforeEach(() => { + const response = { + id: 'ft-10RCfqSvgyEcauomw7VpiYco', + created_at: 1684442489, + updated_at: 1684442489, + organization_id: 'org-COOLORG', + model: 'curie', + fine_tuned_model: 'huh', + status: 'pending', + result_files: [] + } + if (semver.satisfies(realVersion, '>=4.1.0')) { + response.object = 'fine_tuning.job' + response.hyperparameters = { + n_epochs: 5, + batch_size: 3, + prompt_loss_weight: 0.01, + learning_rate_multiplier: 0.1 + } + response.validation_file = null + response.training_file = 'file-t3k1gVSQDHrfZnPckzftlZ4A' + } else { + response.object = 'fine-tunes' + response.hyperparams = { + n_epochs: 5, + batch_size: 3, + prompt_loss_weight: 0.01, + learning_rate_multiplier: 0.1 + } + response.training_files = [{ + object: 'file', + id: 'file-t3k1gVSQDHrfZnPckzftlZ4A', + purpose: 'fine-tune', + filename: 'dave-hal.jsonl', + bytes: 356, + created_at: 1684365950, + status: 'processed', + status_details: null + }] + response.validation_files = [] + response.events = [{ + object: 'fine-tune-event', + level: 'info', + message: 'Created fine-tune: ft-10RCfqSvgyEcauomw7VpiYco', + created_at: 1684442489 + }] + } + scope = nock('https://api.openai.com:443') - .post('/v1/fine-tunes') - .reply(200, { - 'object': 'fine-tune', - 'id': 'ft-10RCfqSvgyEcauomw7VpiYco', - 'hyperparams': { - 'n_epochs': 5, - 'batch_size': 3, - 'prompt_loss_weight': 0.01, - 'learning_rate_multiplier': 0.1 - }, - 'organization_id': 'org-COOLORG', - 'model': 'curie', - 'training_files': [{ - 'object': 'file', - 'id': 'file-t3k1gVSQDHrfZnPckzftlZ4A', - 'purpose': 'fine-tune', - 'filename': 'dave-hal.jsonl', - 'bytes': 356, - 'created_at': 1684365950, - 'status': 'processed', - 'status_details': null - }], - 'validation_files': [], - 'result_files': [], - 'created_at': 1684442489, - 'updated_at': 1684442489, - 'status': 'pending', - 'fine_tuned_model': 'huh', - 'events': [{ - 'object': 'fine-tune-event', - 'level': 'info', - 'message': 'Created fine-tune: ft-10RCfqSvgyEcauomw7VpiYco', - 'created_at': 1684442489 - }] - }, [ + .post( + semver.satisfies(realVersion, '>=4.1.0') ? '/v1/fine_tuning/jobs' : '/v1/fine-tunes' + ) + .reply(200, response, [ 'Date', 'Thu, 18 May 2023 20:41:30 GMT', 'Content-Type', 'application/json', 'Content-Length', '898', @@ -942,11 +1337,21 @@ describe('Plugin', () => { .use(traces => { expect(traces[0][0]).to.have.property('name', 'openai.request') expect(traces[0][0]).to.have.property('type', 'openai') - expect(traces[0][0]).to.have.property('resource', 'createFineTune') + if (semver.satisfies(realVersion, '>=4.1.0')) { + expect(traces[0][0]).to.have.property('resource', 'fine_tuning.jobs.create') + } else if (semver.satisfies(realVersion, '>=4.0.0')) { + expect(traces[0][0]).to.have.property('resource', 'fine-tune.create') + } else { + expect(traces[0][0]).to.have.property('resource', 'createFineTune') + } expect(traces[0][0]).to.have.property('error', 0) expect(traces[0][0].meta).to.have.property('openai.organization.id', 'org-COOLORG') // no name just id expect(traces[0][0].meta).to.have.property('openai.request.method', 'POST') - expect(traces[0][0].meta).to.have.property('openai.request.endpoint', '/v1/fine-tunes') + if (semver.satisfies(realVersion, '>=4.1.0')) { + expect(traces[0][0].meta).to.have.property('openai.request.endpoint', '/v1/fine_tuning/jobs') + } else { + expect(traces[0][0].meta).to.have.property('openai.request.endpoint', '/v1/fine-tunes') + } expect(traces[0][0].meta).to.have.property('openai.request.classification_positive_class', 'wat') expect(traces[0][0].meta).to.have.property('openai.request.model', 'curie') @@ -966,19 +1371,27 @@ describe('Plugin', () => { expect(traces[0][0].metrics).to.have.property('openai.request.n_epochs', 4) expect(traces[0][0].metrics).to.have.property('openai.request.prompt_loss_weight', 0.01) expect(traces[0][0].metrics).to.have.property('openai.response.created_at', 1684442489) - expect(traces[0][0].metrics).to.have.property('openai.response.events_count', 1) - expect(traces[0][0].metrics).to.have.property('openai.response.hyperparams.batch_size', 3) - expect(traces[0][0].metrics).to.have.property('openai.response.hyperparams.learning_rate_multiplier', 0.1) - expect(traces[0][0].metrics).to.have.property('openai.response.hyperparams.n_epochs', 5) - expect(traces[0][0].metrics).to.have.property('openai.response.hyperparams.prompt_loss_weight', 0.01) + + const hyperparams = semver.satisfies(realVersion, '>=4.1.0') ? 'hyperparameters' : 'hyperparams' + + if (semver.satisfies(realVersion, '<4.1.0')) { + expect(traces[0][0].metrics).to.have.property('openai.response.events_count', 1) + } + expect(traces[0][0].metrics).to.have.property(`openai.response.${hyperparams}.batch_size`, 3) + expect(traces[0][0].metrics) + .to.have.property(`openai.response.${hyperparams}.learning_rate_multiplier`, 0.1) + expect(traces[0][0].metrics).to.have.property(`openai.response.${hyperparams}.n_epochs`, 5) + expect(traces[0][0].metrics).to.have.property(`openai.response.${hyperparams}.prompt_loss_weight`, 0.01) expect(traces[0][0].metrics).to.have.property('openai.response.result_files_count', 0) expect(traces[0][0].metrics).to.have.property('openai.response.training_files_count', 1) expect(traces[0][0].metrics).to.have.property('openai.response.updated_at', 1684442489) - expect(traces[0][0].metrics).to.have.property('openai.response.validation_files_count', 0) + if (semver.satisfies(realVersion, '<4.1.0')) { + expect(traces[0][0].metrics).to.have.property('openai.response.validation_files_count', 0) + } }) // only certain request parameter combinations are allowed, leaving unused ones commented for now - const result = await openai.createFineTune({ + const params = { training_file: 'file-t3k1gVSQDHrfZnPckzftlZ4A', validation_file: 'file-foobar', model: 'curie', @@ -992,9 +1405,21 @@ describe('Plugin', () => { classification_positive_class: 'wat', classification_betas: [0.1, 0.2, 0.3] // validation_file: '', - }) + } - expect(result.data.id).to.eql('ft-10RCfqSvgyEcauomw7VpiYco') + if (semver.satisfies(realVersion, '>=4.1.0')) { + const result = await openai.fineTuning.jobs.create(params) + + expect(result.id).to.eql('ft-10RCfqSvgyEcauomw7VpiYco') + } else if (semver.satisfies(realVersion, '>=4.0.0')) { + const result = await openai.fineTunes.create(params) + + expect(result.id).to.eql('ft-10RCfqSvgyEcauomw7VpiYco') + } else { + const result = await openai.createFineTune(params) + + expect(result.data.id).to.eql('ft-10RCfqSvgyEcauomw7VpiYco') + } await checkTraces }) @@ -1005,125 +1430,159 @@ describe('Plugin', () => { expect(traces[0][0]).to.have.property('name', 'openai.request') }) - await openai.createFineTune({ - classification_betas: null - }) + if (semver.satisfies(realVersion, '>=4.1.0')) { + await openai.fineTuning.jobs.create({ + classification_betas: null + }) + } else if (semver.satisfies(realVersion, '>=4.0.0')) { + await openai.fineTunes.create({ + classification_betas: null + }) + } else { + await openai.createFineTune({ + classification_betas: null + }) + } await checkTraces }) }) - describe('retrieveFineTune()', () => { + describe('retrieve finetune', () => { let scope - before(() => { - scope = nock('https://api.openai.com:443') - .get('/v1/fine-tunes/ft-10RCfqSvgyEcauomw7VpiYco') - .reply(200, { - 'object': 'fine-tune', - 'id': 'ft-10RCfqSvgyEcauomw7VpiYco', - 'hyperparams': { - 'n_epochs': 4, - 'batch_size': 3, - 'prompt_loss_weight': 0.01, - 'learning_rate_multiplier': 0.1 + beforeEach(() => { + const response = { + id: 'ft-10RCfqSvgyEcauomw7VpiYco', + organization_id: 'org-COOLORG', + model: 'curie', + created_at: 1684442489, + updated_at: 1684442697, + status: 'succeeded', + fine_tuned_model: 'curie:ft-foo:deleteme-2023-05-18-20-44-56' + } + + if (semver.satisfies(realVersion, '>=4.1.0')) { + response.object = 'fine_tuning.job' + response.hyperparameters = { + n_epochs: 4, + batch_size: 3, + prompt_loss_weight: 0.01, + learning_rate_multiplier: 0.1 + } + response.result_files = [ + 'file-bJyf8TM0jeSZueBo4jpodZVQ' + ] + response.validation_file = null + response.training_file = 'file-t3k1gVSQDHrfZnPckzftlZ4A' + } else { + response.object = 'fine-tune' + response.hyperparams = { + n_epochs: 4, + batch_size: 3, + prompt_loss_weight: 0.01, + learning_rate_multiplier: 0.1 + } + response.result_files = [ + { + object: 'file', + id: 'file-bJyf8TM0jeSZueBo4jpodZVQ', + purpose: 'fine-tune-results', + filename: 'compiled_results.csv', + bytes: 410, + created_at: 1684442697, + status: 'processed', + status_details: null + } + ] + response.validation_files = [] + response.training_files = [{ + object: 'file', + id: 'file-t3k1gVSQDHrfZnPckzftlZ4A', + purpose: 'fine-tune', + filename: 'dave-hal.jsonl', + bytes: 356, + created_at: 1684365950, + status: 'processed', + status_details: null + }] + response.events = [ + { + object: 'fine-tune-event', + level: 'info', + message: 'Created fine-tune: ft-10RCfqSvgyEcauomw7VpiYco', + created_at: 1684442489 }, - 'organization_id': 'org-COOLORG', - 'model': 'curie', - 'training_files': [{ - 'object': 'file', - 'id': 'file-t3k1gVSQDHrfZnPckzftlZ4A', - 'purpose': 'fine-tune', - 'filename': 'dave-hal.jsonl', - 'bytes': 356, - 'created_at': 1684365950, - 'status': 'processed', - 'status_details': null - }], - 'validation_files': [], - 'result_files': [{ - 'object': 'file', - 'id': 'file-bJyf8TM0jeSZueBo4jpodZVQ', - 'purpose': 'fine-tune-results', - 'filename': 'compiled_results.csv', - 'bytes': 410, - 'created_at': 1684442697, - 'status': 'processed', - 'status_details': null - }], - 'created_at': 1684442489, - 'updated_at': 1684442697, - 'status': 'succeeded', - 'fine_tuned_model': 'curie:ft-foo:deleteme-2023-05-18-20-44-56', - 'events': [ - { - 'object': 'fine-tune-event', - 'level': 'info', - 'message': 'Created fine-tune: ft-10RCfqSvgyEcauomw7VpiYco', - 'created_at': 1684442489 - }, - { - 'object': 'fine-tune-event', - 'level': 'info', - 'message': 'Fine-tune costs $0.00', - 'created_at': 1684442612 - }, - { - 'object': 'fine-tune-event', - 'level': 'info', - 'message': 'Fine-tune enqueued. Queue number: 0', - 'created_at': 1684442612 - }, - { - 'object': 'fine-tune-event', - 'level': 'info', - 'message': 'Fine-tune started', - 'created_at': 1684442614 - }, - { - 'object': 'fine-tune-event', - 'level': 'info', - 'message': 'Completed epoch 1/4', - 'created_at': 1684442677 - }, - { - 'object': 'fine-tune-event', - 'level': 'info', - 'message': 'Completed epoch 2/4', - 'created_at': 1684442677 - }, - { - 'object': 'fine-tune-event', - 'level': 'info', - 'message': 'Completed epoch 3/4', - 'created_at': 1684442678 - }, - { - 'object': 'fine-tune-event', - 'level': 'info', - 'message': 'Completed epoch 4/4', - 'created_at': 1684442679 - }, - { - 'object': 'fine-tune-event', - 'level': 'info', - 'message': 'Uploaded model: curie:ft-foo:deleteme-2023-05-18-20-44-56', - 'created_at': 1684442696 - }, - { - 'object': 'fine-tune-event', - 'level': 'info', - 'message': 'Uploaded result file: file-bJyf8TM0jeSZueBo4jpodZVQ', - 'created_at': 1684442697 - }, - { - 'object': 'fine-tune-event', - 'level': 'info', - 'message': 'Fine-tune succeeded', - 'created_at': 1684442697 - } - ] - }, [ + { + object: 'fine-tune-event', + level: 'info', + message: 'Fine-tune costs $0.00', + created_at: 1684442612 + }, + { + object: 'fine-tune-event', + level: 'info', + message: 'Fine-tune enqueued. Queue number: 0', + created_at: 1684442612 + }, + { + object: 'fine-tune-event', + level: 'info', + message: 'Fine-tune started', + created_at: 1684442614 + }, + { + object: 'fine-tune-event', + level: 'info', + message: 'Completed epoch 1/4', + created_at: 1684442677 + }, + { + object: 'fine-tune-event', + level: 'info', + message: 'Completed epoch 2/4', + created_at: 1684442677 + }, + { + object: 'fine-tune-event', + level: 'info', + message: 'Completed epoch 3/4', + created_at: 1684442678 + }, + { + object: 'fine-tune-event', + level: 'info', + message: 'Completed epoch 4/4', + created_at: 1684442679 + }, + { + object: 'fine-tune-event', + level: 'info', + message: 'Uploaded model: curie:ft-foo:deleteme-2023-05-18-20-44-56', + created_at: 1684442696 + }, + { + object: 'fine-tune-event', + level: 'info', + message: 'Uploaded result file: file-bJyf8TM0jeSZueBo4jpodZVQ', + created_at: 1684442697 + }, + { + object: 'fine-tune-event', + level: 'info', + message: 'Fine-tune succeeded', + created_at: 1684442697 + } + ] + } + + scope = nock('https://api.openai.com:443') + .get( + semver.satisfies(realVersion, '>=4.1.0') + ? '/v1/fine_tuning/jobs/ft-10RCfqSvgyEcauomw7VpiYco' + : '/v1/fine-tunes/ft-10RCfqSvgyEcauomw7VpiYco' + ) + .reply(200, response, [ 'Date', 'Thu, 18 May 2023 22:11:53 GMT', 'Content-Type', 'application/json', 'Content-Length', '2727', @@ -1143,85 +1602,143 @@ describe('Plugin', () => { .use(traces => { expect(traces[0][0]).to.have.property('name', 'openai.request') expect(traces[0][0]).to.have.property('type', 'openai') - expect(traces[0][0]).to.have.property('resource', 'retrieveFineTune') + if (semver.satisfies(realVersion, '>=4.1.0')) { + expect(traces[0][0]).to.have.property('resource', 'fine_tuning.jobs.retrieve') + } else if (semver.satisfies(realVersion, '>=4.0.0')) { + expect(traces[0][0]).to.have.property('resource', 'fine-tune.retrieve') + } else { + expect(traces[0][0]).to.have.property('resource', 'retrieveFineTune') + } expect(traces[0][0]).to.have.property('error', 0) expect(traces[0][0].meta).to.have.property('openai.organization.id', 'org-COOLORG') // no name just id expect(traces[0][0].meta).to.have.property('openai.request.method', 'GET') - expect(traces[0][0].meta).to.have.property('openai.request.endpoint', '/v1/fine-tunes/*') + if (semver.satisfies(realVersion, '>=4.1.0')) { + expect(traces[0][0].meta).to.have.property('openai.request.endpoint', '/v1/fine_tuning/jobs/*') + } else { + expect(traces[0][0].meta).to.have.property('openai.request.endpoint', '/v1/fine-tunes/*') + } expect(traces[0][0].meta).to.have.property('openai.request.fine_tune_id', 'ft-10RCfqSvgyEcauomw7VpiYco') expect(traces[0][0].meta).to.have.property('openai.response.id', 'ft-10RCfqSvgyEcauomw7VpiYco') expect(traces[0][0].meta).to.have.property('openai.response.model', 'curie') expect(traces[0][0].meta).to.have.property('openai.response.status', 'succeeded') expect(traces[0][0].metrics).to.have.property('openai.response.created_at', 1684442489) - expect(traces[0][0].metrics).to.have.property('openai.response.events_count', 11) - expect(traces[0][0].metrics).to.have.property('openai.response.hyperparams.batch_size', 3) - expect(traces[0][0].metrics).to.have.property('openai.response.hyperparams.learning_rate_multiplier', 0.1) - expect(traces[0][0].metrics).to.have.property('openai.response.hyperparams.n_epochs', 4) - expect(traces[0][0].metrics).to.have.property('openai.response.hyperparams.prompt_loss_weight', 0.01) + if (semver.satisfies(realVersion, '<4.1.0')) { + expect(traces[0][0].metrics).to.have.property('openai.response.events_count', 11) + } + + const hyperparamsKey = semver.satisfies(realVersion, '>=4.1.0') ? 'hyperparameters' : 'hyperparams' + + expect(traces[0][0].metrics).to.have.property(`openai.response.${hyperparamsKey}.batch_size`, 3) + expect(traces[0][0].metrics) + .to.have.property(`openai.response.${hyperparamsKey}.learning_rate_multiplier`, 0.1) + expect(traces[0][0].metrics).to.have.property(`openai.response.${hyperparamsKey}.n_epochs`, 4) + expect(traces[0][0].metrics) + .to.have.property(`openai.response.${hyperparamsKey}.prompt_loss_weight`, 0.01) expect(traces[0][0].metrics).to.have.property('openai.response.result_files_count', 1) expect(traces[0][0].metrics).to.have.property('openai.response.training_files_count', 1) expect(traces[0][0].metrics).to.have.property('openai.response.updated_at', 1684442697) - expect(traces[0][0].metrics).to.have.property('openai.response.validation_files_count', 0) + if (semver.satisfies(realVersion, '<4.1.0')) { + expect(traces[0][0].metrics).to.have.property('openai.response.validation_files_count', 0) + } }) - const result = await openai.retrieveFineTune('ft-10RCfqSvgyEcauomw7VpiYco') + if (semver.satisfies(realVersion, '>=4.1.0')) { + const result = await openai.fineTuning.jobs.retrieve('ft-10RCfqSvgyEcauomw7VpiYco') - expect(result.data.id).to.eql('ft-10RCfqSvgyEcauomw7VpiYco') + expect(result.id).to.eql('ft-10RCfqSvgyEcauomw7VpiYco') + } else if (semver.satisfies(realVersion, '>=4.0.0')) { + const result = await openai.fineTunes.retrieve('ft-10RCfqSvgyEcauomw7VpiYco') + + expect(result.id).to.eql('ft-10RCfqSvgyEcauomw7VpiYco') + } else { + const result = await openai.retrieveFineTune('ft-10RCfqSvgyEcauomw7VpiYco') + + expect(result.data.id).to.eql('ft-10RCfqSvgyEcauomw7VpiYco') + } await checkTraces }) }) - describe('listFineTunes()', () => { + describe('list finetunes', () => { let scope - before(() => { + beforeEach(() => { + const response = { + object: 'list' + } + + if (semver.satisfies(realVersion, '>=4.1.0')) { + response.data = [{ + object: 'fine-tuning.jobs', + id: 'ft-10RCfqSvgyEcauomw7VpiYco', + hyperparameters: { + n_epochs: 4, + batch_size: 3, + prompt_loss_weight: 0.01, + learning_rate_multiplier: 0.1 + }, + created_at: 1684442489, + updated_at: 1684442697, + organization_id: 'org-COOLORG', + model: 'curie', + fine_tuned_model: 'curie:ft-foo:deleteme-2023-05-18-20-44-56', + result_files: [], + status: 'succeeded', + validation_file: null, + training_file: 'file-t3k1gVSQDHrfZnPckzftlZ4A' + }] + } else { + response.data = [{ + object: 'fine-tune', + id: 'ft-10RCfqSvgyEcauomw7VpiYco', + hyperparams: { + n_epochs: 4, + batch_size: 3, + prompt_loss_weight: 0.01, + learning_rate_multiplier: 0.1 + }, + organization_id: 'org-COOLORG', + model: 'curie', + training_files: [{ + object: 'file', + id: 'file-t3k1gVSQDHrfZnPckzftlZ4A', + purpose: 'fine-tune', + filename: 'dave-hal.jsonl', + bytes: 356, + created_at: 1684365950, + status: 'processed', + status_details: null + }], + validation_files: [], + result_files: [{ + object: 'file', + id: 'file-bJyf8TM0jeSZueBo4jpodZVQ', + purpose: 'fine-tune-results', + filename: 'compiled_results.csv', + bytes: 410, + created_at: 1684442697, + status: 'processed', + status_details: null + }], + created_at: 1684442489, + updated_at: 1684442697, + status: 'succeeded', + fine_tuned_model: 'curie:ft-foo:deleteme-2023-05-18-20-44-56' + }] + } + scope = nock('https://api.openai.com:443') - .get('/v1/fine-tunes') - .reply(200, { - 'object': 'list', - 'data': [{ - 'object': 'fine-tune', - 'id': 'ft-10RCfqSvgyEcauomw7VpiYco', - 'hyperparams': { - 'n_epochs': 4, - 'batch_size': 3, - 'prompt_loss_weight': 0.01, - 'learning_rate_multiplier': 0.1 - }, - 'organization_id': 'org-COOLORG', - 'model': 'curie', - 'training_files': [{ - 'object': 'file', - 'id': 'file-t3k1gVSQDHrfZnPckzftlZ4A', - 'purpose': 'fine-tune', - 'filename': 'dave-hal.jsonl', - 'bytes': 356, - 'created_at': 1684365950, - 'status': 'processed', - 'status_details': null - }], - 'validation_files': [], - 'result_files': [{ - 'object': 'file', - 'id': 'file-bJyf8TM0jeSZueBo4jpodZVQ', - 'purpose': 'fine-tune-results', - 'filename': 'compiled_results.csv', - 'bytes': 410, - 'created_at': 1684442697, - 'status': 'processed', - 'status_details': null - }], - 'created_at': 1684442489, - 'updated_at': 1684442697, - 'status': 'succeeded', - 'fine_tuned_model': 'curie:ft-foo:deleteme-2023-05-18-20-44-56' - }] - }) + .get( + semver.satisfies(realVersion, '>=4.1.0') + ? '/v1/fine_tuning/jobs' + : '/v1/fine-tunes' + ) + .reply(200, response) }) - after(() => { + afterEach(() => { nock.removeInterceptor(scope) scope.done() }) @@ -1231,99 +1748,122 @@ describe('Plugin', () => { .use(traces => { expect(traces[0][0]).to.have.property('name', 'openai.request') expect(traces[0][0]).to.have.property('type', 'openai') - expect(traces[0][0]).to.have.property('resource', 'listFineTunes') + if (semver.satisfies(realVersion, '>=4.1.0')) { + expect(traces[0][0]).to.have.property('resource', 'fine_tuning.jobs.list') + } else if (semver.satisfies(realVersion, '>=4.0.0')) { + expect(traces[0][0]).to.have.property('resource', 'fine-tune.list') + } else { + expect(traces[0][0]).to.have.property('resource', 'listFineTunes') + } expect(traces[0][0]).to.have.property('error', 0) expect(traces[0][0].meta).to.have.property('openai.request.method', 'GET') - expect(traces[0][0].meta).to.have.property('openai.request.endpoint', '/v1/fine-tunes') + if (semver.satisfies(realVersion, '>=4.1.0')) { + expect(traces[0][0].meta).to.have.property('openai.request.endpoint', '/v1/fine_tuning/jobs') + } else { + expect(traces[0][0].meta).to.have.property('openai.request.endpoint', '/v1/fine-tunes') + } expect(traces[0][0].metrics).to.have.property('openai.response.count', 1) }) - const result = await openai.listFineTunes() + if (semver.satisfies(realVersion, '>=4.1.0')) { + const result = await openai.fineTuning.jobs.list() + + expect(result.body.object).to.eql('list') + } else if (semver.satisfies(realVersion, '>=4.0.0')) { + const result = await openai.fineTunes.list() + + expect(result.object).to.eql('list') + } else { + const result = await openai.listFineTunes() - expect(result.data.object).to.eql('list') + expect(result.data.object).to.eql('list') + } await checkTraces }) }) - describe('listFineTuneEvents()', () => { + describe('list finetune events', () => { let scope - before(() => { + beforeEach(() => { // beforeEach allows realVersion to be set first before nocking the call + const response = { + object: 'list', + data: [ + { + level: 'info', + message: 'Created fine-tune: ft-10RCfqSvgyEcauomw7VpiYco', + created_at: 1684442489 + }, + { + level: 'info', + message: 'Fine-tune costs $0.00', + created_at: 1684442612 + }, + { + level: 'info', + message: 'Fine-tune enqueued. Queue number: 0', + created_at: 1684442612 + }, + { + level: 'info', + message: 'Fine-tune started', + created_at: 1684442614 + }, + { + level: 'info', + message: 'Completed epoch 1/4', + created_at: 1684442677 + }, + { + level: 'info', + message: 'Completed epoch 2/4', + created_at: 1684442677 + }, + { + level: 'info', + message: 'Completed epoch 3/4', + created_at: 1684442678 + }, + { + level: 'info', + message: 'Completed epoch 4/4', + created_at: 1684442679 + }, + { + level: 'info', + message: 'Uploaded model: curie:ft-foo:deleteme-2023-05-18-20-44-56', + created_at: 1684442696 + }, + { + level: 'info', + message: 'Uploaded result file: file-bJyf8TM0jeSZueBo4jpodZVQ', + created_at: 1684442697 + }, + { + level: 'info', + message: 'Fine-tune succeeded', + created_at: 1684442697 + } + ] + } + + for (const event of response.data) { + if (semver.satisfies(realVersion, '>=4.1.0')) { + event.object = 'fine_tuning.job.event' + } else { + event.object = 'fine-tune-event' + } + } + scope = nock('https://api.openai.com:443') - .get('/v1/fine-tunes/ft-10RCfqSvgyEcauomw7VpiYco/events') - .reply(200, { - 'object': 'list', - 'data': [ - { - 'object': 'fine-tune-event', - 'level': 'info', - 'message': 'Created fine-tune: ft-10RCfqSvgyEcauomw7VpiYco', - 'created_at': 1684442489 - }, - { - 'object': 'fine-tune-event', - 'level': 'info', - 'message': 'Fine-tune costs $0.00', - 'created_at': 1684442612 - }, - { - 'object': 'fine-tune-event', - 'level': 'info', - 'message': 'Fine-tune enqueued. Queue number: 0', - 'created_at': 1684442612 - }, - { - 'object': 'fine-tune-event', - 'level': 'info', - 'message': 'Fine-tune started', - 'created_at': 1684442614 - }, - { - 'object': 'fine-tune-event', - 'level': 'info', - 'message': 'Completed epoch 1/4', - 'created_at': 1684442677 - }, - { - 'object': 'fine-tune-event', - 'level': 'info', - 'message': 'Completed epoch 2/4', - 'created_at': 1684442677 - }, - { - 'object': 'fine-tune-event', - 'level': 'info', - 'message': 'Completed epoch 3/4', - 'created_at': 1684442678 - }, - { - 'object': 'fine-tune-event', - 'level': 'info', - 'message': 'Completed epoch 4/4', - 'created_at': 1684442679 - }, - { - 'object': 'fine-tune-event', - 'level': 'info', - 'message': 'Uploaded model: curie:ft-foo:deleteme-2023-05-18-20-44-56', - 'created_at': 1684442696 - }, - { - 'object': 'fine-tune-event', - 'level': 'info', - 'message': 'Uploaded result file: file-bJyf8TM0jeSZueBo4jpodZVQ', - 'created_at': 1684442697 - }, - { - 'object': 'fine-tune-event', - 'level': 'info', - 'message': 'Fine-tune succeeded', - 'created_at': 1684442697 - } - ] - }, [ + .get( + semver.satisfies(realVersion, '>=4.1.0') + ? '/v1/fine_tuning/jobs/ft-10RCfqSvgyEcauomw7VpiYco/events' + : '/v1/fine-tunes/ft-10RCfqSvgyEcauomw7VpiYco/events' + ) + .reply(200, response, [ 'Date', 'Thu, 18 May 2023 22:47:17 GMT', 'Content-Type', 'application/json', 'Content-Length', '1718', @@ -1333,7 +1873,7 @@ describe('Plugin', () => { ]) }) - after(() => { + afterEach(() => { nock.removeInterceptor(scope) scope.done() }) @@ -1343,33 +1883,53 @@ describe('Plugin', () => { .use(traces => { expect(traces[0][0]).to.have.property('name', 'openai.request') expect(traces[0][0]).to.have.property('type', 'openai') - expect(traces[0][0]).to.have.property('resource', 'listFineTuneEvents') + if (semver.satisfies(realVersion, '>=4.1.0')) { + expect(traces[0][0]).to.have.property('resource', 'fine_tuning.jobs.listEvents') + } else if (semver.satisfies(realVersion, '>=4.0.0')) { + expect(traces[0][0]).to.have.property('resource', 'fine-tune.listEvents') + } else { + expect(traces[0][0]).to.have.property('resource', 'listFineTuneEvents') + } expect(traces[0][0]).to.have.property('error', 0) expect(traces[0][0].meta).to.have.property('openai.request.method', 'GET') - expect(traces[0][0].meta).to.have.property('openai.request.endpoint', '/v1/fine-tunes/*/events') + if (semver.satisfies(realVersion, '>=4.1.0')) { + expect(traces[0][0].meta).to.have.property('openai.request.endpoint', '/v1/fine_tuning/jobs/*/events') + } else { + expect(traces[0][0].meta).to.have.property('openai.request.endpoint', '/v1/fine-tunes/*/events') + } expect(traces[0][0].meta).to.have.property('openai.request.fine_tune_id', 'ft-10RCfqSvgyEcauomw7VpiYco') expect(traces[0][0].metrics).to.have.property('openai.response.count', 11) }) - const result = await openai.listFineTuneEvents('ft-10RCfqSvgyEcauomw7VpiYco') + if (semver.satisfies(realVersion, '>=4.1.0')) { + const result = await openai.fineTuning.jobs.listEvents('ft-10RCfqSvgyEcauomw7VpiYco') + + expect(result.body.object).to.eql('list') + } else if (semver.satisfies(realVersion, '>=4.0.0')) { + const result = await openai.fineTunes.listEvents('ft-10RCfqSvgyEcauomw7VpiYco') - expect(result.data.object).to.eql('list') + expect(result.object).to.eql('list') + } else { + const result = await openai.listFineTuneEvents('ft-10RCfqSvgyEcauomw7VpiYco') + + expect(result.data.object).to.eql('list') + } await checkTraces }) }) - describe('deleteModel()', () => { + describe('delete model', () => { let scope before(() => { scope = nock('https://api.openai.com:443') .delete('/v1/models/ft-10RCfqSvgyEcauomw7VpiYco') .reply(200, { // guessing on response format here since my key lacks permissions - 'object': 'model', - 'id': 'ft-10RCfqSvgyEcauomw7VpiYco', - 'deleted': true + object: 'model', + id: 'ft-10RCfqSvgyEcauomw7VpiYco', + deleted: true }, [ 'Date', 'Thu, 18 May 2023 22:59:08 GMT', 'Content-Type', 'application/json', @@ -1391,7 +1951,11 @@ describe('Plugin', () => { .use(traces => { expect(traces[0][0]).to.have.property('name', 'openai.request') expect(traces[0][0]).to.have.property('type', 'openai') - expect(traces[0][0]).to.have.property('resource', 'deleteModel') + if (semver.satisfies(realVersion, '>=4.0.0')) { + expect(traces[0][0]).to.have.property('resource', 'models.del') + } else { + expect(traces[0][0]).to.have.property('resource', 'deleteModel') + } expect(traces[0][0]).to.have.property('error', 0) expect(traces[0][0].meta).to.have.property('openai.request.method', 'DELETE') expect(traces[0][0].meta).to.have.property('openai.request.endpoint', '/v1/models/*') @@ -1401,62 +1965,88 @@ describe('Plugin', () => { expect(traces[0][0].meta).to.have.property('openai.response.id', 'ft-10RCfqSvgyEcauomw7VpiYco') }) - const result = await openai.deleteModel('ft-10RCfqSvgyEcauomw7VpiYco') + if (semver.satisfies(realVersion, '>=4.0.0')) { + const result = await openai.models.del('ft-10RCfqSvgyEcauomw7VpiYco') - expect(result.data.deleted).to.eql(true) + expect(result.deleted).to.eql(true) + } else { + const result = await openai.deleteModel('ft-10RCfqSvgyEcauomw7VpiYco') + + expect(result.data.deleted).to.eql(true) + } await checkTraces }) }) - describe('cancelFineTune()', () => { + describe('cancel finetune', () => { let scope - before(() => { - scope = nock('https://api.openai.com:443') - .post('/v1/fine-tunes/ft-TVpNqwlvermMegfRVqSOyPyS/cancel') - .reply(200, { - 'object': 'fine-tune', - 'id': 'ft-TVpNqwlvermMegfRVqSOyPyS', - 'hyperparams': { - 'n_epochs': 4, - 'batch_size': 3, - 'prompt_loss_weight': 0.01, - 'learning_rate_multiplier': 0.1 + beforeEach(() => { + const response = { + id: 'ft-TVpNqwlvermMegfRVqSOyPyS', + organization_id: 'org-COOLORG', + model: 'curie', + created_at: 1684452102, + updated_at: 1684452103, + status: 'cancelled', + fine_tuned_model: 'idk' + } + + if (semver.satisfies(realVersion, '>=4.1.0')) { + response.object = 'fine-tuning.job' + response.hyperparameters = { + n_epochs: 4, + batch_size: 3, + prompt_loss_weight: 0.01, + learning_rate_multiplier: 0.1 + } + response.training_files = 'file-t3k1gVSQDHrfZnPckzftlZ4A' + response.validation_file = null + response.result_files = [] + } else { + response.object = 'fine-tune' + response.hyperparams = { + n_epochs: 4, + batch_size: 3, + prompt_loss_weight: 0.01, + learning_rate_multiplier: 0.1 + } + response.training_files = [{ + object: 'file', + id: 'file-t3k1gVSQDHrfZnPckzftlZ4A', + purpose: 'fine-tune', + filename: 'dave-hal.jsonl', + bytes: 356, + created_at: 1684365950, + status: 'processed', + status_details: null + }] + response.validation_files = [] + response.result_files = [] + response.events = [ + { + object: 'fine-tune-event', + level: 'info', + message: 'Created fine-tune: ft-TVpNqwlvermMegfRVqSOyPyS', + created_at: 1684452102 }, - 'organization_id': 'org-COOLORG', - 'model': 'curie', - 'training_files': [{ - 'object': 'file', - 'id': 'file-t3k1gVSQDHrfZnPckzftlZ4A', - 'purpose': 'fine-tune', - 'filename': 'dave-hal.jsonl', - 'bytes': 356, - 'created_at': 1684365950, - 'status': 'processed', - 'status_details': null - }], - 'validation_files': [], - 'result_files': [], - 'created_at': 1684452102, - 'updated_at': 1684452103, - 'status': 'cancelled', - 'fine_tuned_model': 'idk', - 'events': [ - { - 'object': 'fine-tune-event', - 'level': 'info', - 'message': 'Created fine-tune: ft-TVpNqwlvermMegfRVqSOyPyS', - 'created_at': 1684452102 - }, - { - 'object': 'fine-tune-event', - 'level': 'info', - 'message': 'Fine-tune cancelled', - 'created_at': 1684452103 - } - ] - }, [ + { + object: 'fine-tune-event', + level: 'info', + message: 'Fine-tune cancelled', + created_at: 1684452103 + } + ] + } + + scope = nock('https://api.openai.com:443') + .post( + semver.satisfies(realVersion, '>=4.1.0') + ? '/v1/fine_tuning/jobs/ft-TVpNqwlvermMegfRVqSOyPyS/cancel' + : '/v1/fine-tunes/ft-TVpNqwlvermMegfRVqSOyPyS/cancel' + ) + .reply(200, response, [ 'Date', 'Thu, 18 May 2023 23:21:43 GMT', 'Content-Type', 'application/json', 'Content-Length', '1042', @@ -1466,7 +2056,7 @@ describe('Plugin', () => { ]) }) - after(() => { + afterEach(() => { nock.removeInterceptor(scope) scope.done() }) @@ -1476,11 +2066,21 @@ describe('Plugin', () => { .use(traces => { expect(traces[0][0]).to.have.property('name', 'openai.request') expect(traces[0][0]).to.have.property('type', 'openai') - expect(traces[0][0]).to.have.property('resource', 'cancelFineTune') + if (semver.satisfies(realVersion, '>=4.1.0')) { + expect(traces[0][0]).to.have.property('resource', 'fine_tuning.jobs.cancel') + } else if (semver.satisfies(realVersion, '>=4.0.0')) { + expect(traces[0][0]).to.have.property('resource', 'fine-tune.cancel') + } else { + expect(traces[0][0]).to.have.property('resource', 'cancelFineTune') + } expect(traces[0][0]).to.have.property('error', 0) expect(traces[0][0].meta).to.have.property('openai.organization.id', 'org-COOLORG') expect(traces[0][0].meta).to.have.property('openai.request.method', 'POST') - expect(traces[0][0].meta).to.have.property('openai.request.endpoint', '/v1/fine-tunes/*/cancel') + if (semver.satisfies(realVersion, '>=4.1.0')) { + expect(traces[0][0].meta).to.have.property('openai.request.endpoint', '/v1/fine_tuning/jobs/*/cancel') + } else { + expect(traces[0][0].meta).to.have.property('openai.request.endpoint', '/v1/fine-tunes/*/cancel') + } expect(traces[0][0].meta).to.have.property('openai.request.fine_tune_id', 'ft-TVpNqwlvermMegfRVqSOyPyS') expect(traces[0][0].meta).to.have.property('openai.response.fine_tuned_model', 'idk') @@ -1488,50 +2088,69 @@ describe('Plugin', () => { expect(traces[0][0].meta).to.have.property('openai.response.model', 'curie') expect(traces[0][0].meta).to.have.property('openai.response.status', 'cancelled') expect(traces[0][0].metrics).to.have.property('openai.response.created_at', 1684452102) - expect(traces[0][0].metrics).to.have.property('openai.response.events_count', 2) - expect(traces[0][0].metrics).to.have.property('openai.response.hyperparams.batch_size', 3) - expect(traces[0][0].metrics).to.have.property('openai.response.hyperparams.learning_rate_multiplier', 0.1) - expect(traces[0][0].metrics).to.have.property('openai.response.hyperparams.n_epochs', 4) - expect(traces[0][0].metrics).to.have.property('openai.response.hyperparams.prompt_loss_weight', 0.01) + if (semver.satisfies(realVersion, '<4.1.0')) { + expect(traces[0][0].metrics).to.have.property('openai.response.events_count', 2) + } + + const hyperparamsKey = semver.satisfies(realVersion, '>=4.1.0') ? 'hyperparameters' : 'hyperparams' + + expect(traces[0][0].metrics).to.have.property(`openai.response.${hyperparamsKey}.batch_size`, 3) + expect(traces[0][0].metrics) + .to.have.property(`openai.response.${hyperparamsKey}.learning_rate_multiplier`, 0.1) + expect(traces[0][0].metrics).to.have.property(`openai.response.${hyperparamsKey}.n_epochs`, 4) + expect(traces[0][0].metrics) + .to.have.property(`openai.response.${hyperparamsKey}.prompt_loss_weight`, 0.01) expect(traces[0][0].metrics).to.have.property('openai.response.result_files_count', 0) expect(traces[0][0].metrics).to.have.property('openai.response.training_files_count', 1) expect(traces[0][0].metrics).to.have.property('openai.response.updated_at', 1684452103) - expect(traces[0][0].metrics).to.have.property('openai.response.validation_files_count', 0) + if (semver.satisfies(realVersion, '<4.1.0')) { + expect(traces[0][0].metrics).to.have.property('openai.response.validation_files_count', 0) + } }) - const result = await openai.cancelFineTune('ft-TVpNqwlvermMegfRVqSOyPyS') + if (semver.satisfies(realVersion, '>=4.1.0')) { + const result = await openai.fineTuning.jobs.cancel('ft-TVpNqwlvermMegfRVqSOyPyS') + + expect(result.id).to.eql('ft-TVpNqwlvermMegfRVqSOyPyS') + } else if (semver.satisfies(realVersion, '>=4.0.0')) { + const result = await openai.fineTunes.cancel('ft-TVpNqwlvermMegfRVqSOyPyS') - expect(result.data.id).to.eql('ft-TVpNqwlvermMegfRVqSOyPyS') + expect(result.id).to.eql('ft-TVpNqwlvermMegfRVqSOyPyS') + } else { + const result = await openai.cancelFineTune('ft-TVpNqwlvermMegfRVqSOyPyS') + + expect(result.data.id).to.eql('ft-TVpNqwlvermMegfRVqSOyPyS') + } await checkTraces }) }) - if (semver.intersects(version, '3.0.1')) { - describe('createModeration()', () => { + if (semver.intersects(version, '>=3.0.1')) { + describe('create moderation', () => { let scope before(() => { scope = nock('https://api.openai.com:443') .post('/v1/moderations') .reply(200, { - 'id': 'modr-7HHZZZylF31ahuhmH279JrKbGTHCW', - 'model': 'text-moderation-001', - 'results': [{ - 'flagged': true, - 'categories': { - 'sexual': false, - 'hate': false, - 'violence': true, + id: 'modr-7HHZZZylF31ahuhmH279JrKbGTHCW', + model: 'text-moderation-001', + results: [{ + flagged: true, + categories: { + sexual: false, + hate: false, + violence: true, 'self-harm': false, 'sexual/minors': false, 'hate/threatening': false, 'violence/graphic': false }, - 'category_scores': { - 'sexual': 0.0018438849, - 'hate': 0.069274776, - 'violence': 0.74101615, + category_scores: { + sexual: 0.0018438849, + hate: 0.069274776, + violence: 0.74101615, 'self-harm': 0.008981651, 'sexual/minors': 0.00070737937, 'hate/threatening': 0.045174375, @@ -1559,7 +2178,11 @@ describe('Plugin', () => { .use(traces => { expect(traces[0][0]).to.have.property('name', 'openai.request') expect(traces[0][0]).to.have.property('type', 'openai') - expect(traces[0][0]).to.have.property('resource', 'createModeration') + if (semver.satisfies(realVersion, '>=4.0.0')) { + expect(traces[0][0]).to.have.property('resource', 'moderations.create') + } else { + expect(traces[0][0]).to.have.property('resource', 'createModeration') + } expect(traces[0][0]).to.have.property('error', 0) expect(traces[0][0].meta).to.have.property('openai.organization.name', 'kill-9') expect(traces[0][0].meta).to.have.property('openai.request.method', 'POST') @@ -1591,36 +2214,47 @@ describe('Plugin', () => { expect(traces[0][0].metrics).to.have.property('openai.response.flagged', 1) }) - const result = await openai.createModeration({ - input: 'I want to harm the robots', - model: 'text-moderation-stable' - }) + if (semver.satisfies(realVersion, '>=4.0.0')) { + const result = await openai.moderations.create({ + input: 'I want to harm the robots', + model: 'text-moderation-stable' + }) + + expect(result.results[0].flagged).to.eql(true) + } else { + const result = await openai.createModeration({ + input: 'I want to harm the robots', + model: 'text-moderation-stable' + }) + + expect(result.data.results[0].flagged).to.eql(true) + } - expect(result.data.results[0].flagged).to.eql(true) + await checkTraces expect(externalLoggerStub).to.have.been.calledWith({ status: 'info', - message: 'sampled createModeration', + message: semver.satisfies(realVersion, '>=4.0.0') + ? 'sampled moderations.create' + : 'sampled createModeration', input: 'I want to harm the robots' }) - - await checkTraces }) }) } - if (semver.intersects(version, '3.1')) { - describe('createImage()', () => { + if (semver.intersects(version, '>=3.1')) { + describe('create image', () => { let scope beforeEach(() => { scope = nock('https://api.openai.com:443') .post('/v1/images/generations') .reply(200, { - 'created': 1684270747, - 'data': [{ - 'url': 'https://oaidalleapiprodscus.blob.core.windows.net/private/org-COOLORG/user-FOO/img-foo.png', - 'b64_json': 'foobar===' + created: 1684270747, + data: [{ + url: 'https://oaidalleapiprodscus.blob.core.windows.net/private/org-COOLORG/user-FOO/img-foo.png', + b64_json: 'foobar===' }] }, [ 'Date', 'Tue, 16 May 2023 20:59:07 GMT', @@ -1643,7 +2277,11 @@ describe('Plugin', () => { .use(traces => { expect(traces[0][0]).to.have.property('name', 'openai.request') expect(traces[0][0]).to.have.property('type', 'openai') - expect(traces[0][0]).to.have.property('resource', 'createImage') + if (semver.satisfies(realVersion, '>=4.0.0')) { + expect(traces[0][0]).to.have.property('resource', 'images.generate') + } else { + expect(traces[0][0]).to.have.property('resource', 'createImage') + } expect(traces[0][0]).to.have.property('error', 0) expect(traces[0][0].meta).to.have.property('openai.organization.name', 'kill-9') expect(traces[0][0].meta).to.have.property('openai.request.method', 'POST') @@ -1661,23 +2299,37 @@ describe('Plugin', () => { expect(traces[0][0].metrics).to.have.property('openai.response.images_count', 1) }) - const result = await openai.createImage({ - prompt: 'A datadog wearing headphones', - n: 1, - size: '256x256', - response_format: 'url', - user: 'hunter2' - }) + if (semver.satisfies(realVersion, '>=4.0.0')) { + const result = await openai.images.generate({ + prompt: 'A datadog wearing headphones', + n: 1, + size: '256x256', + response_format: 'url', + user: 'hunter2' + }) - expect(result.data.data[0].url.startsWith('https://')).to.be.true + expect(result.data[0].url.startsWith('https://')).to.be.true + } else { + const result = await openai.createImage({ + prompt: 'A datadog wearing headphones', + n: 1, + size: '256x256', + response_format: 'url', + user: 'hunter2' + }) + + expect(result.data.data[0].url.startsWith('https://')).to.be.true + } + + await checkTraces expect(externalLoggerStub).to.have.been.calledWith({ status: 'info', - message: 'sampled createImage', + message: semver.satisfies(realVersion, '>=4.0.0') + ? 'sampled images.generate' + : 'sampled createImage', prompt: 'A datadog wearing headphones' }) - - await checkTraces }) it('makes a successful call using an array of tokens prompt', async () => { @@ -1686,23 +2338,37 @@ describe('Plugin', () => { expect(traces[0][0].meta).to.have.property('openai.request.prompt', '[999, 888, 777, 666, 555]') }) - const result = await openai.createImage({ - prompt: [999, 888, 777, 666, 555], - n: 1, - size: '256x256', - response_format: 'url', - user: 'hunter2' - }) + if (semver.satisfies(realVersion, '>=4.0.0')) { + const result = await openai.images.generate({ + prompt: [999, 888, 777, 666, 555], + n: 1, + size: '256x256', + response_format: 'url', + user: 'hunter2' + }) + + expect(result.data[0].url.startsWith('https://')).to.be.true + } else { + const result = await openai.createImage({ + prompt: [999, 888, 777, 666, 555], + n: 1, + size: '256x256', + response_format: 'url', + user: 'hunter2' + }) + + expect(result.data.data[0].url.startsWith('https://')).to.be.true + } - expect(result.data.data[0].url.startsWith('https://')).to.be.true + await checkTraces expect(externalLoggerStub).to.have.been.calledWith({ status: 'info', - message: 'sampled createImage', - prompt: [ 999, 888, 777, 666, 555 ] + message: semver.satisfies(realVersion, '>=4.0.0') + ? 'sampled images.generate' + : 'sampled createImage', + prompt: [999, 888, 777, 666, 555] }) - - await checkTraces }) it('makes a successful call using an array of string prompts', async () => { @@ -1712,23 +2378,37 @@ describe('Plugin', () => { expect(traces[0][0].meta).to.have.property('openai.request.prompt.1', 'bar') }) - const result = await openai.createImage({ - prompt: ['foo', 'bar'], - n: 1, - size: '256x256', - response_format: 'url', - user: 'hunter2' - }) + if (semver.satisfies(realVersion, '>=4.0.0')) { + const result = await openai.images.generate({ + prompt: ['foo', 'bar'], + n: 1, + size: '256x256', + response_format: 'url', + user: 'hunter2' + }) + + expect(result.data[0].url.startsWith('https://')).to.be.true + } else { + const result = await openai.createImage({ + prompt: ['foo', 'bar'], + n: 1, + size: '256x256', + response_format: 'url', + user: 'hunter2' + }) - expect(result.data.data[0].url.startsWith('https://')).to.be.true + expect(result.data.data[0].url.startsWith('https://')).to.be.true + } + + await checkTraces expect(externalLoggerStub).to.have.been.calledWith({ status: 'info', - message: 'sampled createImage', - prompt: [ 'foo', 'bar' ] + message: semver.satisfies(realVersion, '>=4.0.0') + ? 'sampled images.generate' + : 'sampled createImage', + prompt: ['foo', 'bar'] }) - - await checkTraces }) it('makes a successful call using an array of tokens prompts', async () => { @@ -1738,40 +2418,54 @@ describe('Plugin', () => { expect(traces[0][0].meta).to.have.property('openai.request.prompt.1', '[444, 555, 666]') }) - const result = await openai.createImage({ - prompt: [ - [111, 222, 333], - [444, 555, 666] - ], - n: 1, - size: '256x256', - response_format: 'url', - user: 'hunter2' - }) + if (semver.satisfies(realVersion, '>=4.0.0')) { + const result = await openai.images.generate({ + prompt: [[111, 222, 333], [444, 555, 666]], + n: 1, + size: '256x256', + response_format: 'url', + user: 'hunter2' + }) + + expect(result.data[0].url.startsWith('https://')) + } else { + const result = await openai.createImage({ + prompt: [ + [111, 222, 333], + [444, 555, 666] + ], + n: 1, + size: '256x256', + response_format: 'url', + user: 'hunter2' + }) - expect(result.data.data[0].url.startsWith('https://')).to.be.true + expect(result.data.data[0].url.startsWith('https://')).to.be.true + } + + await checkTraces expect(externalLoggerStub).to.have.been.calledWith({ status: 'info', - message: 'sampled createImage', - prompt: [ [ 111, 222, 333 ], [ 444, 555, 666 ] ] + message: semver.satisfies(realVersion, '>=4.0.0') + ? 'sampled images.generate' + : 'sampled createImage', + prompt: [[111, 222, 333], [444, 555, 666]] }) - - await checkTraces }) }) - describe('createImageEdit()', () => { + describe('create image edit', () => { let scope before(() => { scope = nock('https://api.openai.com:443') .post('/v1/images/edits') .reply(200, { - 'created': 1684850118, - 'data': [{ - 'url': 'https://oaidalleapiprodscus.blob.core.windows.net/private/org-COOLORG/user-FOO/img-bar.png', - 'b64_json': 'fOoF0f=' + created: 1684850118, + data: [{ + url: 'https://oaidalleapiprodscus.blob.core.windows.net/private/org-COOLORG/user-FOO/img-bar.png', + b64_json: 'fOoF0f=' }] }, [ 'Date', 'Tue, 23 May 2023 13:55:18 GMT', @@ -1794,7 +2488,11 @@ describe('Plugin', () => { .use(traces => { expect(traces[0][0]).to.have.property('name', 'openai.request') expect(traces[0][0]).to.have.property('type', 'openai') - expect(traces[0][0]).to.have.property('resource', 'createImageEdit') + if (semver.satisfies(realVersion, '>=4.0.0')) { + expect(traces[0][0]).to.have.property('resource', 'images.edit') + } else { + expect(traces[0][0]).to.have.property('resource', 'createImageEdit') + } expect(traces[0][0]).to.have.property('error', 0) expect(traces[0][0].meta).to.have.property('openai.organization.name', 'kill-9') expect(traces[0][0].meta).to.have.property('openai.request.method', 'POST') @@ -1814,41 +2512,57 @@ describe('Plugin', () => { expect(traces[0][0].metrics).to.have.property('openai.response.images_count', 1) }) - const result = await openai.createImageEdit( - fs.createReadStream(Path.join(__dirname, 'ntsc.png')), - 'Change all red to blue', - fs.createReadStream(Path.join(__dirname, 'ntsc.png')), - 1, - '256x256', - 'url', - 'hunter2' - ) + if (semver.satisfies(realVersion, '>=4.0.0')) { + const result = await openai.images.edit({ + image: fs.createReadStream(Path.join(__dirname, 'ntsc.png')), + prompt: 'Change all red to blue', + mask: fs.createReadStream(Path.join(__dirname, 'ntsc.png')), + n: 1, + size: '256x256', + response_format: 'url', + user: 'hunter2' + }) + + expect(result.data[0].url.startsWith('https://')).to.be.true + } else { + const result = await openai.createImageEdit( + fs.createReadStream(Path.join(__dirname, 'ntsc.png')), + 'Change all red to blue', + fs.createReadStream(Path.join(__dirname, 'ntsc.png')), + 1, + '256x256', + 'url', + 'hunter2' + ) + + expect(result.data.data[0].url.startsWith('https://')).to.be.true + } - expect(result.data.data[0].url.startsWith('https://')).to.be.true + await checkTraces expect(externalLoggerStub).to.have.been.calledWith({ status: 'info', - message: 'sampled createImageEdit', + message: semver.satisfies(realVersion, '>=4.0.0') + ? 'sampled images.edit' + : 'sampled createImageEdit', prompt: 'Change all red to blue', file: 'ntsc.png', mask: 'ntsc.png' }) - - await checkTraces }) }) - describe('createImageVariation()', () => { + describe('create image variation', () => { let scope before(() => { scope = nock('https://api.openai.com:443') .post('/v1/images/variations') .reply(200, { - 'created': 1684853320, - 'data': [{ - 'url': 'https://oaidalleapiprodscus.blob.core.windows.net/private/org-COOLORG/user-FOO/img-soup.png', - 'b64_json': 'foo=' + created: 1684853320, + data: [{ + url: 'https://oaidalleapiprodscus.blob.core.windows.net/private/org-COOLORG/user-FOO/img-soup.png', + b64_json: 'foo=' }] }, [ 'Date', 'Tue, 23 May 2023 14:48:40 GMT', @@ -1871,7 +2585,11 @@ describe('Plugin', () => { .use(traces => { expect(traces[0][0]).to.have.property('name', 'openai.request') expect(traces[0][0]).to.have.property('type', 'openai') - expect(traces[0][0]).to.have.property('resource', 'createImageVariation') + if (semver.satisfies(realVersion, '>=4.0.0')) { + expect(traces[0][0]).to.have.property('resource', 'images.createVariation') + } else { + expect(traces[0][0]).to.have.property('resource', 'createImageVariation') + } expect(traces[0][0]).to.have.property('error', 0) expect(traces[0][0].meta).to.have.property('openai.organization.name', 'kill-9') expect(traces[0][0].meta).to.have.property('openai.request.method', 'POST') @@ -1889,47 +2607,61 @@ describe('Plugin', () => { expect(traces[0][0].metrics).to.have.property('openai.response.images_count', 1) }) - const result = await openai.createImageVariation( - fs.createReadStream(Path.join(__dirname, 'ntsc.png')), 1, '256x256', 'url', 'hunter2') + if (semver.satisfies(realVersion, '>=4.0.0')) { + const result = await openai.images.createVariation({ + image: fs.createReadStream(Path.join(__dirname, 'ntsc.png')), + n: 1, + size: '256x256', + response_format: 'url', + user: 'hunter2' + }) - expect(result.data.data[0].url.startsWith('https://')).to.be.true + expect(result.data[0].url.startsWith('https://')).to.be.true + } else { + const result = await openai.createImageVariation( + fs.createReadStream(Path.join(__dirname, 'ntsc.png')), 1, '256x256', 'url', 'hunter2') - expect(externalLoggerStub).to.have.been.calledWith({ - status: 'info', - message: 'sampled createImageVariation', - file: 'ntsc.png' - }) + expect(result.data.data[0].url.startsWith('https://')).to.be.true + } await checkTraces + + expect(externalLoggerStub).to.have.been.calledWith({ + status: 'info', + message: semver.satisfies(realVersion, '>=4.0.0') + ? 'sampled images.createVariation' + : 'sampled createImageVariation', + file: 'ntsc.png' + }) }) }) } - if (semver.intersects(version, '3.2')) { - describe('createChatCompletion()', () => { + if (semver.intersects('>=3.2.0', version)) { + describe('create chat completion', () => { let scope beforeEach(() => { scope = nock('https://api.openai.com:443') .post('/v1/chat/completions') .reply(200, { - 'id': 'chatcmpl-7GaWqyMTD9BLmkmy8SxyjUGX3KSRN', - 'object': 'chat.completion', - 'created': 1684188020, - 'model': 'gpt-3.5-turbo-0301', - 'usage': { - 'prompt_tokens': 37, - 'completion_tokens': 10, - 'total_tokens': 47 + id: 'chatcmpl-7GaWqyMTD9BLmkmy8SxyjUGX3KSRN', + object: 'chat.completion', + created: 1684188020, + model: 'gpt-3.5-turbo-0301', + usage: { + prompt_tokens: 37, + completion_tokens: 10, + total_tokens: 47 }, - 'choices': [{ - 'message': { - 'role': 'assistant', - 'content': "In that case, it's best to avoid peanut", - 'name': 'hunter2' + choices: [{ + message: { + role: 'assistant', + content: "In that case, it's best to avoid peanut", + name: 'hunter2' }, - 'finish_reason': 'length', - 'index': 0 + finish_reason: 'length', + index: 0 }] }, [ 'Date', 'Mon, 15 May 2023 22:00:21 GMT', @@ -1953,20 +2685,26 @@ describe('Plugin', () => { .use(traces => { expect(traces[0][0]).to.have.property('name', 'openai.request') expect(traces[0][0]).to.have.property('type', 'openai') - expect(traces[0][0]).to.have.property('resource', 'createChatCompletion') + if (semver.satisfies(realVersion, '>=4.0.0')) { + expect(traces[0][0]).to.have.property('resource', 'chat.completions.create') + } else { + expect(traces[0][0]).to.have.property('resource', 'createChatCompletion') + } expect(traces[0][0]).to.have.property('error', 0) expect(traces[0][0].meta).to.have.property('openai.organization.name', 'kill-9') expect(traces[0][0].meta).to.have.property('openai.request.method', 'POST') expect(traces[0][0].meta).to.have.property('openai.request.endpoint', '/v1/chat/completions') - expect(traces[0][0].meta).to.have.property('openai.request.0.content', 'Peanut Butter or Jelly?') - expect(traces[0][0].meta).to.have.property('openai.request.0.name', 'hunter2') - expect(traces[0][0].meta).to.have.property('openai.request.0.role', 'user') - expect(traces[0][0].meta).to.have.property('openai.request.1.content', 'Are you allergic to peanuts?') - expect(traces[0][0].meta).to.have.property('openai.request.1.role', 'assistant') - expect(traces[0][0].meta).to.have.property('openai.request.2.content', 'Deathly allergic!') - expect(traces[0][0].meta).to.have.property('openai.request.2.role', 'user') + expect(traces[0][0].meta).to.have.property('openai.request.messages.0.content', + 'Peanut Butter or Jelly?') + expect(traces[0][0].meta).to.have.property('openai.request.messages.0.name', 'hunter2') + expect(traces[0][0].meta).to.have.property('openai.request.messages.0.role', 'user') + expect(traces[0][0].meta).to.have.property('openai.request.messages.1.content', + 'Are you allergic to peanuts?') + expect(traces[0][0].meta).to.have.property('openai.request.messages.1.role', 'assistant') + expect(traces[0][0].meta).to.have.property('openai.request.messages.2.content', 'Deathly allergic!') + expect(traces[0][0].meta).to.have.property('openai.request.messages.2.role', 'user') expect(traces[0][0].meta).to.have.property('openai.request.model', 'gpt-3.5-turbo') expect(traces[0][0].meta).to.have.property('openai.request.stop', 'time') expect(traces[0][0].meta).to.have.property('openai.request.user', 'hunter2') @@ -1987,23 +2725,23 @@ describe('Plugin', () => { expect(traces[0][0].metrics).to.have.property('openai.response.usage.total_tokens', 47) }) - const result = await openai.createChatCompletion({ + const params = { model: 'gpt-3.5-turbo', messages: [ { - 'role': 'user', - 'content': 'Peanut Butter or Jelly?', - 'name': 'hunter2' + role: 'user', + content: 'Peanut Butter or Jelly?', + name: 'hunter2' }, { - 'role': 'assistant', - 'content': 'Are you allergic to peanuts?', - 'name': 'hal' + role: 'assistant', + content: 'Are you allergic to peanuts?', + name: 'hal' }, { - 'role': 'user', - 'content': 'Deathly allergic!', - 'name': 'hunter2' + role: 'user', + content: 'Deathly allergic!', + name: 'hunter2' } ], temperature: 1.001, @@ -2012,23 +2750,40 @@ describe('Plugin', () => { presence_penalty: -0.0001, frequency_penalty: 0.0001, logit_bias: { - '1234': -1 + 1234: -1 }, top_p: 4, n: 3, stop: 'time', user: 'hunter2' - }) + } - expect(result.data.id).to.eql('chatcmpl-7GaWqyMTD9BLmkmy8SxyjUGX3KSRN') - expect(result.data.model).to.eql('gpt-3.5-turbo-0301') - expect(result.data.choices[0].message.role).to.eql('assistant') - expect(result.data.choices[0].message.content).to.eql('In that case, it\'s best to avoid peanut') - expect(result.data.choices[0].finish_reason).to.eql('length') + if (semver.satisfies(realVersion, '>=4.0.0')) { + const result = await openai.chat.completions.create(params) + + expect(result.id).to.eql('chatcmpl-7GaWqyMTD9BLmkmy8SxyjUGX3KSRN') + expect(result.model).to.eql('gpt-3.5-turbo-0301') + expect(result.choices[0].message.role).to.eql('assistant') + expect(result.choices[0].message.content).to.eql('In that case, it\'s best to avoid peanut') + expect(result.choices[0].finish_reason).to.eql('length') + } else { + const result = await openai.createChatCompletion(params) + + expect(result.data.id).to.eql('chatcmpl-7GaWqyMTD9BLmkmy8SxyjUGX3KSRN') + expect(result.data.model).to.eql('gpt-3.5-turbo-0301') + expect(result.data.choices[0].message.role).to.eql('assistant') + expect(result.data.choices[0].message.content).to.eql('In that case, it\'s best to avoid peanut') + expect(result.data.choices[0].finish_reason).to.eql('length') + } + + await checkTraces expect(externalLoggerStub).to.have.been.calledWith({ status: 'info', - message: 'sampled createChatCompletion', + message: + semver.satisfies(realVersion, '>=4.0.0') + ? 'sampled chat.completions.create' + : 'sampled createChatCompletion', messages: [ { role: 'user', @@ -2052,8 +2807,6 @@ describe('Plugin', () => { index: 0 }] }) - - await checkTraces }) it('does not error with invalid .messages or missing .logit_bias', async () => { @@ -2062,39 +2815,239 @@ describe('Plugin', () => { expect(traces[0][0]).to.have.property('name', 'openai.request') }) - await openai.createChatCompletion({ - model: 'gpt-3.5-turbo', - messages: null - }) + if (semver.satisfies(realVersion, '>=4.0.0')) { + await openai.chat.completions.create({ + model: 'gpt-3.5-turbo', + messages: null + }) + } else { + await openai.createChatCompletion({ + model: 'gpt-3.5-turbo', + messages: null + }) + } + + await checkTraces + }) + + it('should tag image_url', async () => { + const checkTraces = agent + .use(traces => { + const span = traces[0][0] + // image_url is only relevant on request/input, output has the same shape as a normal chat completion + expect(span.meta).to.have.property('openai.request.messages.0.content.0.type', 'text') + expect(span.meta).to.have.property( + 'openai.request.messages.0.content.0.text', 'I\'m allergic to peanuts. Should I avoid this food?' + ) + expect(span.meta).to.have.property('openai.request.messages.0.content.1.type', 'image_url') + expect(span.meta).to.have.property( + 'openai.request.messages.0.content.1.image_url.url', 'dummy/url/peanut_food.png' + ) + }) + + const params = { + model: 'gpt-4-visual-preview', + messages: [ + { + role: 'user', + name: 'hunter2', + content: [ + { + type: 'text', + text: 'I\'m allergic to peanuts. Should I avoid this food?' + }, + { + type: 'image_url', + image_url: { + url: 'dummy/url/peanut_food.png' + } + } + ] + } + ] + } + + if (semver.satisfies(realVersion, '>=4.0.0')) { + const result = await openai.chat.completions.create(params) + + expect(result.id).to.eql('chatcmpl-7GaWqyMTD9BLmkmy8SxyjUGX3KSRN') + expect(result.choices[0].message.role).to.eql('assistant') + expect(result.choices[0].message.content).to.eql('In that case, it\'s best to avoid peanut') + expect(result.choices[0].finish_reason).to.eql('length') + } else { + const result = await openai.createChatCompletion(params) + + expect(result.data.id).to.eql('chatcmpl-7GaWqyMTD9BLmkmy8SxyjUGX3KSRN') + expect(result.data.choices[0].message.role).to.eql('assistant') + expect(result.data.choices[0].message.content).to.eql('In that case, it\'s best to avoid peanut') + expect(result.data.choices[0].finish_reason).to.eql('length') + } await checkTraces }) }) - describe('createTranscription()', () => { + describe('create chat completion with tools', () => { + let scope + + beforeEach(() => { + scope = nock('https://api.openai.com:443') + .post('/v1/chat/completions') + .reply(200, { + id: 'chatcmpl-7GaWqyMTD9BLmkmy8SxyjUGX3KSRN', + object: 'chat.completion', + created: 1684188020, + model: 'gpt-3.5-turbo-0301', + usage: { + prompt_tokens: 37, + completion_tokens: 10, + total_tokens: 47 + }, + choices: [{ + message: { + role: 'assistant', + content: null, + name: 'hunter2', + tool_calls: [ + { + id: 'tool-1', + type: 'function', + function: { + name: 'extract_fictional_info', + arguments: '{"name":"SpongeBob","origin":"Bikini Bottom"}' + } + } + ] + }, + finish_reason: 'tool_calls', + index: 0 + }] + }, [ + 'Date', 'Mon, 15 May 2023 22:00:21 GMT', + 'Content-Type', 'application/json', + 'Content-Length', '327', + 'access-control-allow-origin', '*', + 'openai-model', 'gpt-3.5-turbo-0301', + 'openai-organization', 'kill-9', + 'openai-processing-ms', '713', + 'openai-version', '2020-10-01' + ]) + }) + + afterEach(() => { + nock.removeInterceptor(scope) + scope.done() + }) + + it('tags the tool calls successfully', async () => { + const checkTraces = agent + .use(traces => { + expect(traces[0][0].meta) + .to.have.property('openai.response.choices.0.message.tool_calls.0.function.name', + 'extract_fictional_info') + expect(traces[0][0].meta) + .to.have.property('openai.response.choices.0.message.tool_calls.0.function.arguments', + '{"name":"SpongeBob","origin":"Bikini Bottom"}') + expect(traces[0][0].meta).to.have.property('openai.response.choices.0.finish_reason', 'tool_calls') + }) + + const input = 'My name is SpongeBob and I live in Bikini Bottom.' + const tools = [ + { + name: 'extract_fictional_info', + description: 'Get the fictional information from the body of the input text', + parameters: { + type: 'object', + properties: { + name: { type: 'string', description: 'Name of the character' }, + origin: { type: 'string', description: 'Where they live' } + } + } + } + ] + + if (semver.satisfies(realVersion, '>=4.0.0')) { + const result = await openai.chat.completions.create({ + model: 'gpt-3.5-turbo', + messages: [{ role: 'user', content: input, name: 'hunter2' }], + tools: [{ type: 'function', function: tools[0] }], + tool_choice: 'auto' + }) + + expect(result.choices[0].finish_reason).to.eql('tool_calls') + } else { + const result = await openai.createChatCompletion({ + model: 'gpt-3.5-turbo', + messages: [{ role: 'user', content: input, name: 'hunter2' }], + tools: [{ type: 'function', function: tools[0] }], + tool_choice: 'auto' + }) + + expect(result.data.choices[0].finish_reason).to.eql('tool_calls') + } + + await checkTraces + + expect(externalLoggerStub).to.have.been.calledWith({ + status: 'info', + message: + semver.satisfies(realVersion, '>=4.0.0') + ? 'sampled chat.completions.create' + : 'sampled createChatCompletion', + messages: [ + { + role: 'user', + content: input, + name: 'hunter2' + } + ], + choices: [{ + message: { + role: 'assistant', + content: null, + tool_calls: [ + { + id: 'tool-1', + type: 'function', + function: { + name: 'extract_fictional_info', + arguments: '{"name":"SpongeBob","origin":"Bikini Bottom"}' + } + } + ], + name: 'hunter2' + }, + finish_reason: 'tool_calls', + index: 0 + }] + }) + }) + }) + + describe('create transcription', () => { let scope before(() => { scope = nock('https://api.openai.com:443') .post('/v1/audio/transcriptions') .reply(200, { - 'task': 'transcribe', - 'language': 'english', - 'duration': 2.19, - 'segments': [{ - 'id': 0, - 'seek': 0, - 'start': 0, - 'end': 2, - 'text': ' Hello, friend.', - 'tokens': [50364, 2425, 11, 1277, 13, 50464], - 'temperature': 0.5, - 'avg_logprob': -0.7777707236153739, - 'compression_ratio': 0.6363636363636364, - 'no_speech_prob': 0.043891049921512604, - 'transient': false + task: 'transcribe', + language: 'english', + duration: 2.19, + segments: [{ + id: 0, + seek: 0, + start: 0, + end: 2, + text: ' Hello, friend.', + tokens: [50364, 2425, 11, 1277, 13, 50464], + temperature: 0.5, + avg_logprob: -0.7777707236153739, + compression_ratio: 0.6363636363636364, + no_speech_prob: 0.043891049921512604, + transient: false }], - 'text': 'Hello, friend.' + text: 'Hello, friend.' }, [ 'Date', 'Fri, 19 May 2023 03:19:49 GMT', 'Content-Type', 'text/plain; charset=utf-8', @@ -2117,7 +3070,11 @@ describe('Plugin', () => { .use(traces => { expect(traces[0][0]).to.have.property('name', 'openai.request') expect(traces[0][0]).to.have.property('type', 'openai') - expect(traces[0][0]).to.have.property('resource', 'createTranscription') + if (semver.satisfies(realVersion, '>=4.0.0')) { + expect(traces[0][0]).to.have.property('resource', 'audio.transcriptions.create') + } else { + expect(traces[0][0]).to.have.property('resource', 'createTranscription') + } expect(traces[0][0]).to.have.property('error', 0) expect(traces[0][0].meta).to.have.property('openai.organization.name', 'kill-9') @@ -2136,52 +3093,68 @@ describe('Plugin', () => { }) // TODO: Should test each of 'json, text, srt, verbose_json, vtt' since response formats differ - const result = await openai.createTranscription( - fs.createReadStream(Path.join(__dirname, '/hello-friend.m4a')), - 'whisper-1', - 'what does this say', - 'verbose_json', - 0.5, - 'en' - ) + if (semver.satisfies(realVersion, '>=4.0.0')) { + const result = await openai.audio.transcriptions.create({ + file: fs.createReadStream(Path.join(__dirname, '/hello-friend.m4a')), + model: 'whisper-1', + prompt: 'what does this say', + response_format: 'verbose_json', + temperature: 0.5, + language: 'en' + }) + + // for OpenAI v4, the result is a stringified version of the JSON response + expect(typeof result).to.eql('string') + } else { + const result = await openai.createTranscription( + fs.createReadStream(Path.join(__dirname, '/hello-friend.m4a')), + 'whisper-1', + 'what does this say', + 'verbose_json', + 0.5, + 'en' + ) + + expect(result.data.text).to.eql('Hello, friend.') + } - expect(result.data.text).to.eql('Hello, friend.') + await checkTraces expect(externalLoggerStub).to.have.been.calledWith({ status: 'info', - message: 'sampled createTranscription', + message: semver.satisfies(realVersion, '>=4.0.0') + ? 'sampled audio.transcriptions.create' + : 'sampled createTranscription', prompt: 'what does this say', file: 'hello-friend.m4a' }) - - await checkTraces }) }) - describe('createTranslation()', () => { + describe('create translation', () => { let scope before(() => { scope = nock('https://api.openai.com:443') .post('/v1/audio/translations') .reply(200, { - 'task': 'translate', - 'language': 'english', - 'duration': 1.74, - 'segments': [{ - 'id': 0, - 'seek': 0, - 'start': 0, - 'end': 3, - 'text': ' Guten Tag!', - 'tokens': [50364, 42833, 11204, 0, 50514], - 'temperature': 0.5, - 'avg_logprob': -0.5626437266667684, - 'compression_ratio': 0.5555555555555556, - 'no_speech_prob': 0.01843200996518135, - 'transient': false + task: 'translate', + language: 'english', + duration: 1.74, + segments: [{ + id: 0, + seek: 0, + start: 0, + end: 3, + text: ' Guten Tag!', + tokens: [50364, 42833, 11204, 0, 50514], + temperature: 0.5, + avg_logprob: -0.5626437266667684, + compression_ratio: 0.5555555555555556, + no_speech_prob: 0.01843200996518135, + transient: false }], - 'text': 'Guten Tag!' + text: 'Guten Tag!' }, [ 'Date', 'Fri, 19 May 2023 03:41:25 GMT', 'Content-Type', 'application/json', @@ -2203,7 +3176,11 @@ describe('Plugin', () => { .use(traces => { expect(traces[0][0]).to.have.property('name', 'openai.request') expect(traces[0][0]).to.have.property('type', 'openai') - expect(traces[0][0]).to.have.property('resource', 'createTranslation') + if (semver.satisfies(realVersion, '>=4.0.0')) { + expect(traces[0][0]).to.have.property('resource', 'audio.translations.create') + } else { + expect(traces[0][0]).to.have.property('resource', 'createTranslation') + } expect(traces[0][0]).to.have.property('error', 0) expect(traces[0][0].meta).to.have.property('openai.organization.name', 'kill-9') @@ -2221,25 +3198,530 @@ describe('Plugin', () => { }) // TODO: Should test each of 'json, text, srt, verbose_json, vtt' since response formats differ - const result = await openai.createTranslation( - fs.createReadStream(Path.join(__dirname, 'guten-tag.m4a')), - 'whisper-1', - 'greeting', - 'verbose_json', - 0.5 - ) + if (semver.satisfies(realVersion, '>=4.0.0')) { + const result = await openai.audio.translations.create({ + file: fs.createReadStream(Path.join(__dirname, 'guten-tag.m4a')), + model: 'whisper-1', + prompt: 'greeting', + response_format: 'verbose_json', + temperature: 0.5 + }) + + expect(result.text).to.eql('Guten Tag!') + } else { + const result = await openai.createTranslation( + fs.createReadStream(Path.join(__dirname, 'guten-tag.m4a')), + 'whisper-1', + 'greeting', + 'verbose_json', + 0.5 + ) + + expect(result.data.text).to.eql('Guten Tag!') + } - expect(result.data.text).to.eql('Guten Tag!') + await checkTraces expect(externalLoggerStub).to.have.been.calledWith({ status: 'info', - message: 'sampled createTranslation', + message: semver.satisfies(realVersion, '>=4.0.0') + ? 'sampled audio.translations.create' + : 'sampled createTranslation', prompt: 'greeting', file: 'guten-tag.m4a' }) + }) + }) + } + + if (semver.intersects('>4.1.0', version)) { + describe('streamed responses', () => { + afterEach(() => { + nock.cleanAll() + }) + + it('makes a successful chat completion call', async () => { + nock('https://api.openai.com:443') + .post('/v1/chat/completions') + .reply(200, function () { + return fs.createReadStream(Path.join(__dirname, 'streamed-responses/chat.completions.simple.txt')) + }, { + 'Content-Type': 'text/plain', + 'openai-organization': 'kill-9' + }) + + const checkTraces = agent + .use(traces => { + const span = traces[0][0] + expect(span).to.have.property('name', 'openai.request') + expect(span).to.have.property('type', 'openai') + expect(span).to.have.property('error', 0) + expect(span.meta).to.have.property('openai.organization.name', 'kill-9') + expect(span.meta).to.have.property('openai.request.method', 'POST') + expect(span.meta).to.have.property('openai.request.endpoint', '/v1/chat/completions') + expect(span.meta).to.have.property('openai.request.model', 'gpt-4o') + expect(span.meta).to.have.property('openai.request.messages.0.content', + 'Hello, OpenAI!') + expect(span.meta).to.have.property('openai.request.messages.0.role', 'user') + expect(span.meta).to.have.property('openai.request.messages.0.name', 'hunter2') + expect(span.meta).to.have.property('openai.response.choices.0.finish_reason', 'stop') + expect(span.meta).to.have.property('openai.response.choices.0.logprobs', 'returned') + expect(span.meta).to.have.property('openai.response.choices.0.message.role', 'assistant') + expect(span.meta).to.have.property('openai.response.choices.0.message.content', + 'Hello! How can I assist you today?') + + expect(span.metrics).to.have.property('openai.response.usage.prompt_tokens') + expect(span.metrics).to.not.have.property('openai.response.usage.prompt_tokens_estimated') + expect(span.metrics).to.have.property('openai.response.usage.completion_tokens') + expect(span.metrics).to.not.have.property('openai.response.usage.completion_tokens_estimated') + expect(span.metrics).to.have.property('openai.response.usage.total_tokens') + }) + + const stream = await openai.chat.completions.create({ + model: 'gpt-4o', + messages: [{ role: 'user', content: 'Hello, OpenAI!', name: 'hunter2' }], + temperature: 0.5, + stream: true + }) + + for await (const part of stream) { + expect(part).to.have.property('choices') + expect(part.choices[0]).to.have.property('delta') + } + + await checkTraces + + expect(metricStub).to.have.been.calledWith('openai.tokens.prompt') + expect(metricStub).to.have.been.calledWith('openai.tokens.completion') + expect(metricStub).to.have.been.calledWith('openai.tokens.total') + }) + + it('makes a successful chat completion call with empty stream', async () => { + nock('https://api.openai.com:443') + .post('/v1/chat/completions') + .reply(200, function () { + return fs.createReadStream(Path.join(__dirname, 'streamed-responses/chat.completions.empty.txt')) + }, { + 'Content-Type': 'text/plain', + 'openai-organization': 'kill-9' + }) + + const checkTraces = agent + .use(traces => { + const span = traces[0][0] + expect(span).to.have.property('name', 'openai.request') + expect(span).to.have.property('type', 'openai') + expect(span).to.have.property('error', 0) + expect(span.meta).to.have.property('openai.organization.name', 'kill-9') + expect(span.meta).to.have.property('openai.request.method', 'POST') + expect(span.meta).to.have.property('openai.request.endpoint', '/v1/chat/completions') + expect(span.meta).to.have.property('openai.request.model', 'gpt-4o') + expect(span.meta).to.have.property('openai.request.messages.0.content', 'Hello, OpenAI!') + expect(span.meta).to.have.property('openai.request.messages.0.role', 'user') + expect(span.meta).to.have.property('openai.request.messages.0.name', 'hunter2') + }) + + const stream = await openai.chat.completions.create({ + model: 'gpt-4o', + messages: [{ role: 'user', content: 'Hello, OpenAI!', name: 'hunter2' }], + temperature: 0.5, + stream: true + }) + + for await (const part of stream) { + expect(part).to.have.property('choices') + } + + await checkTraces + }) + + it('makes a successful chat completion call with multiple choices', async () => { + nock('https://api.openai.com:443') + .post('/v1/chat/completions') + .reply(200, function () { + return fs.createReadStream(Path.join(__dirname, 'streamed-responses/chat.completions.multiple.txt')) + }, { + 'Content-Type': 'text/plain', + 'openai-organization': 'kill-9' + }) + + const checkTraces = agent + .use(traces => { + const span = traces[0][0] + expect(span).to.have.property('name', 'openai.request') + expect(span).to.have.property('type', 'openai') + expect(span).to.have.property('error', 0) + expect(span.meta).to.have.property('openai.organization.name', 'kill-9') + expect(span.meta).to.have.property('openai.request.method', 'POST') + expect(span.meta).to.have.property('openai.request.endpoint', '/v1/chat/completions') + expect(span.meta).to.have.property('openai.request.model', 'gpt-4') + expect(span.meta).to.have.property('openai.request.messages.0.content', 'How are you?') + expect(span.meta).to.have.property('openai.request.messages.0.role', 'user') + expect(span.meta).to.have.property('openai.request.messages.0.name', 'hunter2') + + // message 0 + expect(span.meta).to.have.property('openai.response.choices.0.finish_reason', 'stop') + expect(span.meta).to.have.property('openai.response.choices.0.logprobs', 'returned') + expect(span.meta).to.have.property('openai.response.choices.0.message.role', 'assistant') + expect(span.meta).to.have.property('openai.response.choices.0.message.content', + 'As an AI, I don\'t have feelings, but I\'m here to assist you. How can I help you today?' + ) + + // message 1 + expect(span.meta).to.have.property('openai.response.choices.1.finish_reason', 'stop') + expect(span.meta).to.have.property('openai.response.choices.1.logprobs', 'returned') + expect(span.meta).to.have.property('openai.response.choices.1.message.role', 'assistant') + expect(span.meta).to.have.property('openai.response.choices.1.message.content', + 'I\'m just a computer program so I don\'t have feelings, ' + + 'but I\'m here and ready to help you with anything you need. How can I assis...' + ) + + // message 2 + expect(span.meta).to.have.property('openai.response.choices.2.finish_reason', 'stop') + expect(span.meta).to.have.property('openai.response.choices.2.logprobs', 'returned') + expect(span.meta).to.have.property('openai.response.choices.2.message.role', 'assistant') + expect(span.meta).to.have.property('openai.response.choices.2.message.content', + 'I\'m just a computer program, so I don\'t have feelings like humans do. ' + + 'I\'m here and ready to assist you with any questions or tas...' + ) + }) + + const stream = await openai.chat.completions.create({ + model: 'gpt-4', + messages: [{ role: 'user', content: 'How are you?', name: 'hunter2' }], + stream: true, + n: 3 + }) + + for await (const part of stream) { + expect(part).to.have.property('choices') + expect(part.choices[0]).to.have.property('delta') + } + + await checkTraces + }) + + it('makes a successful chat completion call with usage included', async () => { + nock('https://api.openai.com:443') + .post('/v1/chat/completions') + .reply(200, function () { + return fs.createReadStream(Path.join(__dirname, 'streamed-responses/chat.completions.simple.usage.txt')) + }, { + 'Content-Type': 'text/plain', + 'openai-organization': 'kill-9' + }) + + const checkTraces = agent + .use(traces => { + const span = traces[0][0] + + expect(span.meta).to.have.property('openai.response.choices.0.message.content', 'I\'m just a computer') + expect(span.metrics).to.have.property('openai.response.usage.prompt_tokens', 11) + expect(span.metrics).to.have.property('openai.response.usage.completion_tokens', 5) + expect(span.metrics).to.have.property('openai.response.usage.total_tokens', 16) + }) + + const stream = await openai.chat.completions.create({ + model: 'gpt-3.5-turbo', + messages: [{ role: 'user', content: 'How are you?', name: 'hunter2' }], + max_tokens: 5, + stream: true, + stream_options: { + include_usage: true + } + }) + + for await (const part of stream) { + expect(part).to.have.property('choices') + } + + await checkTraces + + const expectedTags = [ + 'error:0', + 'org:kill-9', + 'endpoint:/v1/chat/completions', + 'model:gpt-3.5-turbo-0125' + ] + + expect(metricStub).to.have.been.calledWith('openai.tokens.prompt', 11, 'd', expectedTags) + expect(metricStub).to.have.been.calledWith('openai.tokens.completion', 5, 'd', expectedTags) + expect(metricStub).to.have.been.calledWith('openai.tokens.total', 16, 'd', expectedTags) + }) + + it('makes a successful chat completion call without image_url usage computed', async () => { + nock('https://api.openai.com:443') + .post('/v1/chat/completions') + .reply(200, function () { + return fs.createReadStream(Path.join(__dirname, 'streamed-responses/chat.completions.simple.txt')) + }, { + 'Content-Type': 'text/plain', + 'openai-organization': 'kill-9' + }) + + const checkTraces = agent + .use(traces => { + const span = traces[0][0] + + // we shouldn't be trying to capture the image_url tokens + expect(span.metrics).to.have.property('openai.response.usage.prompt_tokens', 1) + }) + + const stream = await openai.chat.completions.create({ + stream: 1, + model: 'gpt-4o', + messages: [ + { + role: 'user', + name: 'hunter2', + content: [ + { + type: 'text', + text: 'One' // one token, for ease of testing + }, + { + type: 'image_url', + image_url: { + url: 'dummy/url/peanut_food.png' + } + } + ] + } + ] + }) + + for await (const part of stream) { + expect(part).to.have.property('choices') + } + + await checkTraces + }) + + it('makes a successful completion call', async () => { + nock('https://api.openai.com:443') + .post('/v1/completions') + .reply(200, function () { + return fs.createReadStream(Path.join(__dirname, 'streamed-responses/completions.simple.txt')) + }, { + 'Content-Type': 'text/plain', + 'openai-organization': 'kill-9' + }) + + const checkTraces = agent + .use(traces => { + const span = traces[0][0] + + expect(span).to.have.property('name', 'openai.request') + expect(span).to.have.property('type', 'openai') + expect(span).to.have.property('error', 0) + expect(span.meta).to.have.property('openai.organization.name', 'kill-9') + expect(span.meta).to.have.property('openai.request.method', 'POST') + expect(span.meta).to.have.property('openai.request.endpoint', '/v1/completions') + expect(span.meta).to.have.property('openai.request.model', 'text-davinci-002') + expect(span.meta).to.have.property('openai.request.prompt', 'Hello, OpenAI!') + expect(span.meta).to.have.property('openai.response.choices.0.finish_reason', 'stop') + expect(span.meta).to.have.property('openai.response.choices.0.logprobs', 'returned') + expect(span.meta).to.have.property('openai.response.choices.0.text', ' this is a test.') + + expect(span.metrics).to.have.property('openai.response.usage.prompt_tokens') + expect(span.metrics).to.not.have.property('openai.response.usage.prompt_tokens_estimated') + expect(span.metrics).to.have.property('openai.response.usage.completion_tokens') + expect(span.metrics).to.not.have.property('openai.response.usage.completion_tokens_estimated') + expect(span.metrics).to.have.property('openai.response.usage.total_tokens') + }) + + const stream = await openai.completions.create({ + model: 'text-davinci-002', + prompt: 'Hello, OpenAI!', + temperature: 0.5, + stream: true + }) + + for await (const part of stream) { + expect(part).to.have.property('choices') + expect(part.choices[0]).to.have.property('text') + } await checkTraces + + expect(metricStub).to.have.been.calledWith('openai.tokens.prompt') + expect(metricStub).to.have.been.calledWith('openai.tokens.completion') + expect(metricStub).to.have.been.calledWith('openai.tokens.total') }) + + it('makes a successful completion call with usage included', async () => { + nock('https://api.openai.com:443') + .post('/v1/completions') + .reply(200, function () { + return fs.createReadStream(Path.join(__dirname, 'streamed-responses/completions.simple.usage.txt')) + }, { + 'Content-Type': 'text/plain', + 'openai-organization': 'kill-9' + }) + + const checkTraces = agent + .use(traces => { + const span = traces[0][0] + + expect(span.meta).to.have.property('openai.response.choices.0.text', '\\n\\nI am an AI') + expect(span.metrics).to.have.property('openai.response.usage.prompt_tokens', 4) + expect(span.metrics).to.have.property('openai.response.usage.completion_tokens', 5) + expect(span.metrics).to.have.property('openai.response.usage.total_tokens', 9) + }) + + const stream = await openai.completions.create({ + model: 'gpt-3.5-turbo-instruct', + prompt: 'How are you?', + stream: true, + max_tokens: 5, + stream_options: { + include_usage: true + } + }) + + for await (const part of stream) { + expect(part).to.have.property('choices') + } + + await checkTraces + + const expectedTags = [ + 'error:0', + 'org:kill-9', + 'endpoint:/v1/completions', + 'model:gpt-3.5-turbo-instruct' + ] + + expect(metricStub).to.have.been.calledWith('openai.tokens.prompt', 4, 'd', expectedTags) + expect(metricStub).to.have.been.calledWith('openai.tokens.completion', 5, 'd', expectedTags) + expect(metricStub).to.have.been.calledWith('openai.tokens.total', 9, 'd', expectedTags) + }) + + if (semver.intersects('>4.16.0', version)) { + it('makes a successful chat completion call with tools', async () => { + nock('https://api.openai.com:443') + .post('/v1/chat/completions') + .reply(200, function () { + return fs.createReadStream(Path.join(__dirname, 'streamed-responses/chat.completions.tools.txt')) + }, { + 'Content-Type': 'text/plain', + 'openai-organization': 'kill-9' + }) + + const checkTraces = agent + .use(traces => { + const span = traces[0][0] + + expect(span).to.have.property('name', 'openai.request') + expect(span).to.have.property('type', 'openai') + expect(span).to.have.property('error', 0) + expect(span.meta).to.have.property('openai.organization.name', 'kill-9') + expect(span.meta).to.have.property('openai.request.method', 'POST') + expect(span.meta).to.have.property('openai.request.endpoint', '/v1/chat/completions') + expect(span.meta).to.have.property('openai.request.model', 'gpt-4') + expect(span.meta).to.have.property('openai.request.messages.0.content', 'Hello, OpenAI!') + expect(span.meta).to.have.property('openai.request.messages.0.role', 'user') + expect(span.meta).to.have.property('openai.request.messages.0.name', 'hunter2') + expect(span.meta).to.have.property('openai.response.choices.0.finish_reason', 'tool_calls') + expect(span.meta).to.have.property('openai.response.choices.0.logprobs', 'returned') + expect(span.meta).to.have.property('openai.response.choices.0.message.role', 'assistant') + expect(span.meta).to.have.property('openai.response.choices.0.message.tool_calls.0.function.name', + 'get_current_weather') + }) + + const tools = [ + { + type: 'function', + function: { + name: 'get_current_weather', + description: 'Get the current weather in a given location', + parameters: { + type: 'object', + properties: { + location: { + type: 'string', + description: 'The city and state, e.g. San Francisco, CA' + }, + unit: { type: 'string', enum: ['celsius', 'fahrenheit'] } + }, + required: ['location'] + } + } + } + ] + + const stream = await openai.chat.completions.create({ + model: 'gpt-4', + messages: [{ role: 'user', content: 'Hello, OpenAI!', name: 'hunter2' }], + temperature: 0.5, + tools, + tool_choice: 'auto', + stream: true + }) + + for await (const part of stream) { + expect(part).to.have.property('choices') + expect(part.choices[0]).to.have.property('delta') + } + + await checkTraces + }) + + it('makes a successful chat completion call with tools and content', async () => { + nock('https://api.openai.com:443') + .post('/v1/chat/completions') + .reply(200, function () { + return fs.createReadStream( + Path.join(__dirname, 'streamed-responses/chat.completions.tool.and.content.txt') + ) + }, { + 'Content-Type': 'text/plain', + 'openai-organization': 'kill-9' + }) + + const checkTraces = agent + .use(traces => { + const span = traces[0][0] + + expect(span).to.have.property('name', 'openai.request') + expect(span).to.have.property('type', 'openai') + expect(span).to.have.property('error', 0) + expect(span.meta).to.have.property('openai.organization.name', 'kill-9') + expect(span.meta).to.have.property('openai.request.method', 'POST') + expect(span.meta).to.have.property('openai.request.endpoint', '/v1/chat/completions') + expect(span.meta).to.have.property('openai.request.model', 'gpt-4') + expect(span.meta).to.have.property('openai.request.messages.0.content', 'Hello, OpenAI!') + expect(span.meta).to.have.property('openai.request.messages.0.role', 'user') + expect(span.meta).to.have.property('openai.request.messages.0.name', 'hunter2') + expect(span.meta).to.have.property('openai.response.choices.0.message.role', 'assistant') + expect(span.meta).to.have.property('openai.response.choices.0.message.content', + 'THOUGHT: Hi') + expect(span.meta).to.have.property('openai.response.choices.0.finish_reason', 'tool_calls') + expect(span.meta).to.have.property('openai.response.choices.0.logprobs', 'returned') + expect(span.meta).to.have.property('openai.response.choices.0.message.tool_calls.0.function.name', + 'finish') + expect(span.meta).to.have.property( + 'openai.response.choices.0.message.tool_calls.0.function.arguments', + '{\n"answer": "5"\n}' + ) + }) + + const stream = await openai.chat.completions.create({ + model: 'gpt-4', + messages: [{ role: 'user', content: 'Hello, OpenAI!', name: 'hunter2' }], + temperature: 0.5, + tools: [], // dummy tools, the response is hardcoded + stream: true + }) + + for await (const part of stream) { + expect(part).to.have.property('choices') + expect(part.choices[0]).to.have.property('delta') + } + + await checkTraces + }) + } }) } }) diff --git a/packages/datadog-plugin-openai/test/integration-test/client.spec.js b/packages/datadog-plugin-openai/test/integration-test/client.spec.js index c8b38afb994..a68613f47fd 100644 --- a/packages/datadog-plugin-openai/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-openai/test/integration-test/client.spec.js @@ -13,11 +13,13 @@ describe('esm', () => { let proc let sandbox - withVersions('openai', 'openai', version => { + // limit v4 tests while the IITM issue is resolved or a workaround is introduced + // issue link: https://github.com/DataDog/import-in-the-middle/issues/60 + withVersions('openai', 'openai', '>=3 <4', version => { before(async function () { this.timeout(20000) sandbox = await createSandbox([`'openai@${version}'`, 'nock'], false, [ - `./packages/datadog-plugin-openai/test/integration-test/*`]) + './packages/datadog-plugin-openai/test/integration-test/*']) }) after(async () => { diff --git a/packages/datadog-plugin-openai/test/streamed-responses/chat.completions.empty.txt b/packages/datadog-plugin-openai/test/streamed-responses/chat.completions.empty.txt new file mode 100644 index 00000000000..e1c6271262e --- /dev/null +++ b/packages/datadog-plugin-openai/test/streamed-responses/chat.completions.empty.txt @@ -0,0 +1,5 @@ +data: {"id":"chatcmpl-9S9XTKSaDNOTtVqvF2hAbdu4UGYQa","object":"chat.completion.chunk","created":1716496879,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9S9XTKSaDNOTtVqvF2hAbdu4UGYQa","object":"chat.completion.chunk","created":1716496879,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"length"}]} + +data: [DONE] \ No newline at end of file diff --git a/packages/datadog-plugin-openai/test/streamed-responses/chat.completions.multiple.txt b/packages/datadog-plugin-openai/test/streamed-responses/chat.completions.multiple.txt new file mode 100644 index 00000000000..273475c39f9 --- /dev/null +++ b/packages/datadog-plugin-openai/test/streamed-responses/chat.completions.multiple.txt @@ -0,0 +1,212 @@ +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"As"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":1,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":1,"delta":{"content":"I"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":"I"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" an"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":1,"delta":{"content":"'m"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":"'m"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" AI"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":1,"delta":{"content":" just"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":" just"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":","},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":1,"delta":{"content":" a"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":" a"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" I"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":1,"delta":{"content":" computer"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":" computer"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" don"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":1,"delta":{"content":" program"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":" program"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"'t"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":1,"delta":{"content":" so"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":","},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" have"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":1,"delta":{"content":" I"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":" so"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" feelings"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":1,"delta":{"content":" don"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":" I"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":","},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":" don"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" but"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":1,"delta":{"content":"'t"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":1,"delta":{"content":" have"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":1,"delta":{"content":" feelings"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":1,"delta":{"content":","},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":1,"delta":{"content":" but"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":"'t"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":" have"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":" feelings"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":" like"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" I"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"'m"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" here"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" to"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":" humans"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":" do"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" assist"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" you"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":"."},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":1,"delta":{"content":" I"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":1,"delta":{"content":"'m"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":1,"delta":{"content":" here"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":1,"delta":{"content":" and"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":" I"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" How"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":"'m"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" can"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":1,"delta":{"content":" ready"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":1,"delta":{"content":" to"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":" here"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" I"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":" and"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" help"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":1,"delta":{"content":" help"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":1,"delta":{"content":" you"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":" ready"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":1,"delta":{"content":" with"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":" to"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":1,"delta":{"content":" anything"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":" assist"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" you"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" today"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"?"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":1,"delta":{"content":" you"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":" you"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":1,"delta":{"content":" need"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":" with"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":1,"delta":{"content":"."},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":" any"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":1,"delta":{"content":" How"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":" questions"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":1,"delta":{"content":" can"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":" or"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":1,"delta":{"content":" I"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":" tasks"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":1,"delta":{"content":" assist"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":" you"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":1,"delta":{"content":" you"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":" have"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":1,"delta":{"content":" today"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":"."},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":1,"delta":{"content":"?"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":" How"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":" can"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":" I"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":" help"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":" you"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":" today"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{"content":"?"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":1,"delta":{},"logprobs":null,"finish_reason":"stop"}]} + +data: {"id":"chatcmpl-9UI1IdBePvsMlmjSRj8LRojnwpm9f","object":"chat.completion.chunk","created":1717006136,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":2,"delta":{},"logprobs":null,"finish_reason":"stop"}]} + +data: [DONE] + diff --git a/packages/datadog-plugin-openai/test/streamed-responses/chat.completions.simple.txt b/packages/datadog-plugin-openai/test/streamed-responses/chat.completions.simple.txt new file mode 100644 index 00000000000..42bd5f10a6c --- /dev/null +++ b/packages/datadog-plugin-openai/test/streamed-responses/chat.completions.simple.txt @@ -0,0 +1,24 @@ +data: {"id":"chatcmpl-9EHaxj6mPeGvYIJ6lSzaTIcjSfDIi","object":"chat.completion.chunk","created":1713191255,"model":"gpt-3.5-turbo-0125","system_fingerprint":"fp_c2295e73ad","choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9EHaxj6mPeGvYIJ6lSzaTIcjSfDIi","object":"chat.completion.chunk","created":1713191255,"model":"gpt-3.5-turbo-0125","system_fingerprint":"fp_c2295e73ad","choices":[{"index":0,"delta":{"content":"Hello"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9EHaxj6mPeGvYIJ6lSzaTIcjSfDIi","object":"chat.completion.chunk","created":1713191255,"model":"gpt-3.5-turbo-0125","system_fingerprint":"fp_c2295e73ad","choices":[{"index":0,"delta":{"content":"!"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9EHaxj6mPeGvYIJ6lSzaTIcjSfDIi","object":"chat.completion.chunk","created":1713191255,"model":"gpt-3.5-turbo-0125","system_fingerprint":"fp_c2295e73ad","choices":[{"index":0,"delta":{"content":" How"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9EHaxj6mPeGvYIJ6lSzaTIcjSfDIi","object":"chat.completion.chunk","created":1713191255,"model":"gpt-3.5-turbo-0125","system_fingerprint":"fp_c2295e73ad","choices":[{"index":0,"delta":{"content":" can"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9EHaxj6mPeGvYIJ6lSzaTIcjSfDIi","object":"chat.completion.chunk","created":1713191255,"model":"gpt-3.5-turbo-0125","system_fingerprint":"fp_c2295e73ad","choices":[{"index":0,"delta":{"content":" I"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9EHaxj6mPeGvYIJ6lSzaTIcjSfDIi","object":"chat.completion.chunk","created":1713191255,"model":"gpt-3.5-turbo-0125","system_fingerprint":"fp_c2295e73ad","choices":[{"index":0,"delta":{"content":" assist"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9EHaxj6mPeGvYIJ6lSzaTIcjSfDIi","object":"chat.completion.chunk","created":1713191255,"model":"gpt-3.5-turbo-0125","system_fingerprint":"fp_c2295e73ad","choices":[{"index":0,"delta":{"content":" you"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9EHaxj6mPeGvYIJ6lSzaTIcjSfDIi","object":"chat.completion.chunk","created":1713191255,"model":"gpt-3.5-turbo-0125","system_fingerprint":"fp_c2295e73ad","choices":[{"index":0,"delta":{"content":" today"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9EHaxj6mPeGvYIJ6lSzaTIcjSfDIi","object":"chat.completion.chunk","created":1713191255,"model":"gpt-3.5-turbo-0125","system_fingerprint":"fp_c2295e73ad","choices":[{"index":0,"delta":{"content":"?"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9EHaxj6mPeGvYIJ6lSzaTIcjSfDIi","object":"chat.completion.chunk","created":1713191255,"model":"gpt-3.5-turbo-0125","system_fingerprint":"fp_c2295e73ad","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}]} + +data: [DONE] + diff --git a/packages/datadog-plugin-openai/test/streamed-responses/chat.completions.simple.usage.txt b/packages/datadog-plugin-openai/test/streamed-responses/chat.completions.simple.usage.txt new file mode 100644 index 00000000000..83af1ac9a65 --- /dev/null +++ b/packages/datadog-plugin-openai/test/streamed-responses/chat.completions.simple.usage.txt @@ -0,0 +1,17 @@ +data: {"id":"chatcmpl-9V17JDcZNGGQucdBOBKsQ7LQ5jyLk","object":"chat.completion.chunk","created":1717179489,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}],"usage":null} + +data: {"id":"chatcmpl-9V17JDcZNGGQucdBOBKsQ7LQ5jyLk","object":"chat.completion.chunk","created":1717179489,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"I"},"logprobs":null,"finish_reason":null}],"usage":null} + +data: {"id":"chatcmpl-9V17JDcZNGGQucdBOBKsQ7LQ5jyLk","object":"chat.completion.chunk","created":1717179489,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"'m"},"logprobs":null,"finish_reason":null}],"usage":null} + +data: {"id":"chatcmpl-9V17JDcZNGGQucdBOBKsQ7LQ5jyLk","object":"chat.completion.chunk","created":1717179489,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" just"},"logprobs":null,"finish_reason":null}],"usage":null} + +data: {"id":"chatcmpl-9V17JDcZNGGQucdBOBKsQ7LQ5jyLk","object":"chat.completion.chunk","created":1717179489,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" a"},"logprobs":null,"finish_reason":null}],"usage":null} + +data: {"id":"chatcmpl-9V17JDcZNGGQucdBOBKsQ7LQ5jyLk","object":"chat.completion.chunk","created":1717179489,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" computer"},"logprobs":null,"finish_reason":null}],"usage":null} + +data: {"id":"chatcmpl-9V17JDcZNGGQucdBOBKsQ7LQ5jyLk","object":"chat.completion.chunk","created":1717179489,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"length"}],"usage":null} + +data: {"id":"chatcmpl-9V17JDcZNGGQucdBOBKsQ7LQ5jyLk","object":"chat.completion.chunk","created":1717179489,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[],"usage":{"prompt_tokens":11,"completion_tokens":5,"total_tokens":16}} + +data: [DONE] \ No newline at end of file diff --git a/packages/datadog-plugin-openai/test/streamed-responses/chat.completions.tool.and.content.txt b/packages/datadog-plugin-openai/test/streamed-responses/chat.completions.tool.and.content.txt new file mode 100644 index 00000000000..3947339157d --- /dev/null +++ b/packages/datadog-plugin-openai/test/streamed-responses/chat.completions.tool.and.content.txt @@ -0,0 +1,33 @@ +data: {"id":"chatcmpl-A3juVlDlz6tV3bfCY2WZfYxRlKiAH","object":"chat.completion.chunk","created":1725454827,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-A3juVlDlz6tV3bfCY2WZfYxRlKiAH","object":"chat.completion.chunk","created":1725454827,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"TH"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-A3juVlDlz6tV3bfCY2WZfYxRlKiAH","object":"chat.completion.chunk","created":1725454827,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"O"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-A3juVlDlz6tV3bfCY2WZfYxRlKiAH","object":"chat.completion.chunk","created":1725454827,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"UGHT"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-A3juVlDlz6tV3bfCY2WZfYxRlKiAH","object":"chat.completion.chunk","created":1725454827,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":":"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-A3juVlDlz6tV3bfCY2WZfYxRlKiAH","object":"chat.completion.chunk","created":1725454827,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" Hi"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-A3juVlDlz6tV3bfCY2WZfYxRlKiAH","object":"chat.completion.chunk","created":1725454827,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_Tg0o5wgoNSKF2iggAPmfWwem","type":"function","function":{"name":"finish","arguments":""}}]},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-A3juVlDlz6tV3bfCY2WZfYxRlKiAH","object":"chat.completion.chunk","created":1725454827,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\n"}}]},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-A3juVlDlz6tV3bfCY2WZfYxRlKiAH","object":"chat.completion.chunk","created":1725454827,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\""}}]},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-A3juVlDlz6tV3bfCY2WZfYxRlKiAH","object":"chat.completion.chunk","created":1725454827,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"answer"}}]},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-A3juVlDlz6tV3bfCY2WZfYxRlKiAH","object":"chat.completion.chunk","created":1725454827,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\":"}}]},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-A3juVlDlz6tV3bfCY2WZfYxRlKiAH","object":"chat.completion.chunk","created":1725454827,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":" \""}}]},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-A3juVlDlz6tV3bfCY2WZfYxRlKiAH","object":"chat.completion.chunk","created":1725454827,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"5"}}]},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-A3juVlDlz6tV3bfCY2WZfYxRlKiAH","object":"chat.completion.chunk","created":1725454827,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"\n"}}]},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-A3juVlDlz6tV3bfCY2WZfYxRlKiAH","object":"chat.completion.chunk","created":1725454827,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"}"}}]},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-A3juVlDlz6tV3bfCY2WZfYxRlKiAH","object":"chat.completion.chunk","created":1725454827,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"tool_calls"}]} + +data: [DONE] \ No newline at end of file diff --git a/packages/datadog-plugin-openai/test/streamed-responses/chat.completions.tools.txt b/packages/datadog-plugin-openai/test/streamed-responses/chat.completions.tools.txt new file mode 100644 index 00000000000..f9b7a3a102b --- /dev/null +++ b/packages/datadog-plugin-openai/test/streamed-responses/chat.completions.tools.txt @@ -0,0 +1,29 @@ +data: {"id":"chatcmpl-9S8QJE6AyQxIFu9lf1U3VGOWWEkmA","object":"chat.completion.chunk","created":1716492591,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"role":"assistant","content":null,"tool_calls":[{"index":0,"id":"call_2K7z4ywTEOTuDpFgpVTs3vCF","type":"function","function":{"name":"get_current_weather","arguments":""}}]},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9S8QJE6AyQxIFu9lf1U3VGOWWEkmA","object":"chat.completion.chunk","created":1716492591,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\n"}}]},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9S8QJE6AyQxIFu9lf1U3VGOWWEkmA","object":"chat.completion.chunk","created":1716492591,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":" "}}]},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9S8QJE6AyQxIFu9lf1U3VGOWWEkmA","object":"chat.completion.chunk","created":1716492591,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":" \""}}]},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9S8QJE6AyQxIFu9lf1U3VGOWWEkmA","object":"chat.completion.chunk","created":1716492591,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"location"}}]},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9S8QJE6AyQxIFu9lf1U3VGOWWEkmA","object":"chat.completion.chunk","created":1716492591,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\":"}}]},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9S8QJE6AyQxIFu9lf1U3VGOWWEkmA","object":"chat.completion.chunk","created":1716492591,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":" \""}}]},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9S8QJE6AyQxIFu9lf1U3VGOWWEkmA","object":"chat.completion.chunk","created":1716492591,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"San"}}]},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9S8QJE6AyQxIFu9lf1U3VGOWWEkmA","object":"chat.completion.chunk","created":1716492591,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":" Francisco"}}]},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9S8QJE6AyQxIFu9lf1U3VGOWWEkmA","object":"chat.completion.chunk","created":1716492591,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":","}}]},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9S8QJE6AyQxIFu9lf1U3VGOWWEkmA","object":"chat.completion.chunk","created":1716492591,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":" CA"}}]},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9S8QJE6AyQxIFu9lf1U3VGOWWEkmA","object":"chat.completion.chunk","created":1716492591,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"\n"}}]},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9S8QJE6AyQxIFu9lf1U3VGOWWEkmA","object":"chat.completion.chunk","created":1716492591,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"}"}}]},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-9S8QJE6AyQxIFu9lf1U3VGOWWEkmA","object":"chat.completion.chunk","created":1716492591,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"tool_calls"}]} + +data: [DONE] \ No newline at end of file diff --git a/packages/datadog-plugin-openai/test/streamed-responses/completions.simple.txt b/packages/datadog-plugin-openai/test/streamed-responses/completions.simple.txt new file mode 100644 index 00000000000..b0f9045ae9e --- /dev/null +++ b/packages/datadog-plugin-openai/test/streamed-responses/completions.simple.txt @@ -0,0 +1,15 @@ +data: {"id":"cmpl-9SBFwkdjMAXO6n7Z0cz7ScN1SKJSr","object":"text_completion","created":1716503480,"choices":[{"text":" ","index":0,"logprobs":null,"finish_reason":null}],"model":"text-davinci-002"} + +data: {"id":"cmpl-9SBFwkdjMAXO6n7Z0cz7ScN1SKJSr","object":"text_completion","created":1716503480,"choices":[{"text":"this","index":0,"logprobs":null,"finish_reason":null}],"model":"text-davinci-002"} + +data: {"id":"cmpl-9SBFwkdjMAXO6n7Z0cz7ScN1SKJSr","object":"text_completion","created":1716503480,"choices":[{"text":" is","index":0,"logprobs":null,"finish_reason":null}],"model":"text-davinci-002"} + +data: {"id":"cmpl-9SBFwkdjMAXO6n7Z0cz7ScN1SKJSr","object":"text_completion","created":1716503480,"choices":[{"text":" a","index":0,"logprobs":null,"finish_reason":null}],"model":"text-davinci-002"} + +data: {"id":"cmpl-9SBFwkdjMAXO6n7Z0cz7ScN1SKJSr","object":"text_completion","created":1716503480,"choices":[{"text":" test","index":0,"logprobs":null,"finish_reason":null}],"model":"text-davinci-002"} + +data: {"id":"cmpl-9SBFwkdjMAXO6n7Z0cz7ScN1SKJSr","object":"text_completion","created":1716503480,"choices":[{"text":".","index":0,"logprobs":null,"finish_reason":null}],"model":"text-davinci-002"} + +data: {"id":"cmpl-9SBFwkdjMAXO6n7Z0cz7ScN1SKJSr","object":"text_completion","created":1716503480,"choices":[{"text":"","index":0,"logprobs":null,"finish_reason":"stop"}],"model":"text-davinci-002"} + +data: [DONE] \ No newline at end of file diff --git a/packages/datadog-plugin-openai/test/streamed-responses/completions.simple.usage.txt b/packages/datadog-plugin-openai/test/streamed-responses/completions.simple.usage.txt new file mode 100644 index 00000000000..4b2014266aa --- /dev/null +++ b/packages/datadog-plugin-openai/test/streamed-responses/completions.simple.usage.txt @@ -0,0 +1,15 @@ +data: {"id":"cmpl-9V1c0TcDX8x5tpPWvYvWtW4PxHGmB","object":"text_completion","created":1717181392,"choices":[{"text":"\n\n","index":0,"logprobs":null,"finish_reason":null}],"model":"gpt-3.5-turbo-instruct","usage":null} + +data: {"id":"cmpl-9V1c0TcDX8x5tpPWvYvWtW4PxHGmB","object":"text_completion","created":1717181392,"choices":[{"text":"I","index":0,"logprobs":null,"finish_reason":null}],"model":"gpt-3.5-turbo-instruct","usage":null} + +data: {"id":"cmpl-9V1c0TcDX8x5tpPWvYvWtW4PxHGmB","object":"text_completion","created":1717181392,"choices":[{"text":" am","index":0,"logprobs":null,"finish_reason":null}],"model":"gpt-3.5-turbo-instruct","usage":null} + +data: {"id":"cmpl-9V1c0TcDX8x5tpPWvYvWtW4PxHGmB","object":"text_completion","created":1717181392,"choices":[{"text":" an","index":0,"logprobs":null,"finish_reason":null}],"model":"gpt-3.5-turbo-instruct","usage":null} + +data: {"id":"cmpl-9V1c0TcDX8x5tpPWvYvWtW4PxHGmB","object":"text_completion","created":1717181392,"choices":[{"text":" AI","index":0,"logprobs":null,"finish_reason":"length"}],"model":"gpt-3.5-turbo-instruct","usage":null} + +data: {"id":"cmpl-9V1c0TcDX8x5tpPWvYvWtW4PxHGmB","object":"text_completion","created":1717181392,"choices":[{"text":"","index":0,"logprobs":null,"finish_reason":"length"}],"model":"gpt-3.5-turbo-instruct","usage":null} + +data: {"id":"cmpl-9V1c0TcDX8x5tpPWvYvWtW4PxHGmB","object":"text_completion","created":1717181392,"model":"gpt-3.5-turbo-instruct","usage":{"prompt_tokens":4,"completion_tokens":5,"total_tokens":9},"choices":[]} + +data: [DONE] \ No newline at end of file diff --git a/packages/datadog-plugin-openai/test/token-estimator.spec.js b/packages/datadog-plugin-openai/test/token-estimator.spec.js new file mode 100644 index 00000000000..375c655738a --- /dev/null +++ b/packages/datadog-plugin-openai/test/token-estimator.spec.js @@ -0,0 +1,28 @@ +'use strict' + +const { estimateTokens } = require('../src/token-estimator') + +describe('Plugin', () => { + describe('openai token estimation', () => { + function testEstimation (input, expected) { + const tokens = estimateTokens(input) + expect(tokens).to.equal(expected) + } + + it('should compute the number of tokens in a string', () => { + testEstimation('hello world', 2) + }) + + it('should not throw for an empty string', () => { + testEstimation('', 0) + }) + + it('should compute the number of tokens in an array of integer inputs', () => { + testEstimation([1, 2, 3], 3) + }) + + it('should compute no tokens for invalid content', () => { + testEstimation({}, 0) + }) + }) +}) diff --git a/packages/datadog-plugin-opensearch/test/integration-test/client.spec.js b/packages/datadog-plugin-opensearch/test/integration-test/client.spec.js index 626ed563fea..7121c2acf72 100644 --- a/packages/datadog-plugin-opensearch/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-opensearch/test/integration-test/client.spec.js @@ -17,7 +17,7 @@ describe('esm', () => { before(async function () { this.timeout(20000) sandbox = await createSandbox([`'@opensearch-project/opensearch@${version}'`], false, [ - `./packages/datadog-plugin-opensearch/test/integration-test/*`]) + './packages/datadog-plugin-opensearch/test/integration-test/*']) }) after(async () => { diff --git a/packages/datadog-plugin-opensearch/test/naming.js b/packages/datadog-plugin-opensearch/test/naming.js index bd594c91383..72b158196a0 100644 --- a/packages/datadog-plugin-opensearch/test/naming.js +++ b/packages/datadog-plugin-opensearch/test/naming.js @@ -14,6 +14,6 @@ const rawExpectedSchema = { } module.exports = { - rawExpectedSchema: rawExpectedSchema, + rawExpectedSchema, expectedSchema: resolveNaming(rawExpectedSchema) } diff --git a/packages/datadog-plugin-oracledb/test/index.spec.js b/packages/datadog-plugin-oracledb/test/index.spec.js index 0aa1d086973..e6e6f051d87 100644 --- a/packages/datadog-plugin-oracledb/test/index.spec.js +++ b/packages/datadog-plugin-oracledb/test/index.spec.js @@ -28,6 +28,7 @@ describe('Plugin', () => { oracledb = require(`../../../versions/oracledb@${version}`).get() tracer = require('../../dd-trace') }) + after(async () => { await agent.close({ ritmReset: false }) }) @@ -36,6 +37,7 @@ describe('Plugin', () => { before(async () => { connection = await oracledb.getConnection(config) }) + after(async () => { await connection.close() }) @@ -68,6 +70,7 @@ describe('Plugin', () => { ` }) }) + after(async () => { await connection.close() }) @@ -165,6 +168,7 @@ describe('Plugin', () => { pool = await oracledb.createPool(config) connection = await pool.getConnection() }) + after(async () => { await connection.close() await pool.close() @@ -194,6 +198,7 @@ describe('Plugin', () => { }) connection = await pool.getConnection() }) + after(async () => { await connection.close() await pool.close() @@ -262,12 +267,15 @@ describe('Plugin', () => { oracledb = require(`../../../versions/oracledb@${version}`).get() tracer = require('../../dd-trace') }) + before(async () => { connection = await oracledb.getConnection(config) }) + after(async () => { await connection.close() }) + after(async () => { await agent.close({ ritmReset: false }) }) @@ -284,6 +292,7 @@ describe('Plugin', () => { } } ) + it('should set the service name', done => { agent.use(traces => { expect(traces[0][0]).to.have.property('name', expectedSchema.outbound.opName) @@ -292,18 +301,22 @@ describe('Plugin', () => { connection.execute(dbQuery) }) }) + describe('with service function', () => { before(async () => { await agent.load('oracledb', { service: connAttrs => `${connAttrs.connectString}` }) oracledb = require(`../../../versions/oracledb@${version}`).get() tracer = require('../../dd-trace') }) + before(async () => { connection = await oracledb.getConnection(config) }) + after(async () => { await connection.close() }) + after(async () => { await agent.close({ ritmReset: false }) }) @@ -320,6 +333,7 @@ describe('Plugin', () => { } } ) + it('should set the service name', done => { agent.use(traces => { expect(traces[0][0]).to.have.property('name', expectedSchema.outbound.opName) diff --git a/packages/datadog-plugin-oracledb/test/integration-test/client.spec.js b/packages/datadog-plugin-oracledb/test/integration-test/client.spec.js index cd34ed135b7..f486e69e1cd 100644 --- a/packages/datadog-plugin-oracledb/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-oracledb/test/integration-test/client.spec.js @@ -17,7 +17,7 @@ describe('esm', () => { before(async function () { this.timeout(20000) sandbox = await createSandbox([`'oracledb@${version}'`], false, [ - `./packages/datadog-plugin-oracledb/test/integration-test/*`]) + './packages/datadog-plugin-oracledb/test/integration-test/*']) }) after(async () => { diff --git a/packages/datadog-plugin-oracledb/test/naming.js b/packages/datadog-plugin-oracledb/test/naming.js index 5bd78e51d4d..eaed0fa7516 100644 --- a/packages/datadog-plugin-oracledb/test/naming.js +++ b/packages/datadog-plugin-oracledb/test/naming.js @@ -14,6 +14,6 @@ const rawExpectedSchema = { } module.exports = { - rawExpectedSchema: rawExpectedSchema, + rawExpectedSchema, expectedSchema: resolveNaming(rawExpectedSchema) } diff --git a/packages/datadog-plugin-paperplane/test/index.spec.js b/packages/datadog-plugin-paperplane/test/index.spec.js index 167fb3841a4..5499e9cc98b 100644 --- a/packages/datadog-plugin-paperplane/test/index.spec.js +++ b/packages/datadog-plugin-paperplane/test/index.spec.js @@ -1,7 +1,6 @@ 'use strict' const axios = require('axios') -const getPort = require('get-port') const semver = require('semver') const agent = require('../../dd-trace/test/plugins/agent') @@ -76,7 +75,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -94,11 +95,9 @@ describe('Plugin', () => { .then(done) .catch(done) - server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -113,7 +112,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -123,11 +124,9 @@ describe('Plugin', () => { .then(done) .catch(done) - server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user/1`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user/1`) + .catch(done) }) }) @@ -155,7 +154,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -165,11 +166,9 @@ describe('Plugin', () => { .then(done) .catch(done) - server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user/123`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user/123`) + .catch(done) }) }) @@ -191,7 +190,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -201,11 +202,9 @@ describe('Plugin', () => { .then(done) .catch(done) - server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user/123`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user/123`) + .catch(done) }) }) @@ -220,7 +219,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -230,13 +231,11 @@ describe('Plugin', () => { .then(done) .catch(done) - server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) @@ -247,7 +246,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -257,11 +258,9 @@ describe('Plugin', () => { .then(done) .catch(done) - server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app`) + .catch(done) }) }) @@ -280,7 +279,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -290,10 +291,8 @@ describe('Plugin', () => { .then(done) .catch(done) - server.listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/app/user/123`) - .catch(done) - }) + axios.get(`http://localhost:${port}/app/user/123`) + .catch(done) }) }) @@ -308,7 +307,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent.use(traces => { const spans = sort(traces[0]) @@ -318,17 +319,15 @@ describe('Plugin', () => { .then(done) .catch(done) - server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - headers: { - 'x-datadog-trace-id': '1234', - 'x-datadog-parent-id': '5678', - 'ot-baggage-foo': 'bar' - } - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + headers: { + 'x-datadog-trace-id': '1234', + 'x-datadog-parent-id': '5678', + 'ot-baggage-foo': 'bar' + } + }) + .catch(done) }) }) @@ -343,7 +342,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent.use(traces => { const spans = sort(traces[0]) @@ -355,13 +356,11 @@ describe('Plugin', () => { done() }) - server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) @@ -376,7 +375,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent.use(traces => { const spans = sort(traces[0]) @@ -388,13 +389,11 @@ describe('Plugin', () => { done() }) - server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 400 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 400 + }) + .catch(done) }) }) @@ -405,7 +404,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -417,13 +418,11 @@ describe('Plugin', () => { .then(done) .catch(done) - server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) @@ -477,7 +476,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -487,11 +488,9 @@ describe('Plugin', () => { .then(done) .catch(done) - server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -506,7 +505,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -516,13 +517,11 @@ describe('Plugin', () => { .then(done) .catch(done) - server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 400 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 400 + }) + .catch(done) }) }) @@ -537,7 +536,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -547,13 +548,11 @@ describe('Plugin', () => { .then(done) .catch(done) - server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - headers: { 'User-Agent': 'test' } - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + headers: { 'User-Agent': 'test' } + }) + .catch(done) }) }) diff --git a/packages/datadog-plugin-pg/test/index.spec.js b/packages/datadog-plugin-pg/test/index.spec.js index ae3f0dcdcc8..72dab8b3ece 100644 --- a/packages/datadog-plugin-pg/test/index.spec.js +++ b/packages/datadog-plugin-pg/test/index.spec.js @@ -8,6 +8,8 @@ const net = require('net') const { expectedSchema, rawExpectedSchema } = require('./naming') const EventEmitter = require('events') +const ddpv = require('mocha/package.json').version + const clients = { pg: pg => pg.Client } @@ -371,7 +373,8 @@ describe('Plugin', () => { if (client.queryQueue[0] !== undefined) { try { expect(client.queryQueue[0].text).to.equal( - `/*dddbs='serviced',dde='tester',ddps='test',ddpv='8.4.0'*/ SELECT $1::text as message`) + '/*dddb=\'postgres\',dddbs=\'serviced\',dde=\'tester\',ddh=\'127.0.0.1\',ddps=\'test\',' + + `ddpv='${ddpv}'*/ SELECT $1::text as message`) } catch (e) { done(e) } @@ -428,8 +431,8 @@ describe('Plugin', () => { if (clientDBM.queryQueue[0] !== undefined) { try { expect(clientDBM.queryQueue[0].text).to.equal( - `/*dddbs='~!%40%23%24%25%5E%26*()_%2B%7C%3F%3F%2F%3C%3E',dde='tester',` + - `ddps='test',ddpv='8.4.0'*/ SELECT $1::text as message`) + '/*dddb=\'postgres\',dddbs=\'~!%40%23%24%25%5E%26*()_%2B%7C%3F%3F%2F%3C%3E\',dde=\'tester\',' + + `ddh='127.0.0.1',ddps='test',ddpv='${ddpv}'*/ SELECT $1::text as message`) done() } catch (e) { done(e) @@ -570,7 +573,7 @@ describe('Plugin', () => { const spanId = traces[0][0].span_id.toString(16).padStart(16, '0') expect(queryText).to.equal( - `/*dddbs='post',dde='tester',ddps='test',ddpv='8.4.0',` + + `/*dddb='postgres',dddbs='post',dde='tester',ddh='127.0.0.1',ddps='test',ddpv='${ddpv}',` + `traceparent='00-${traceId}-${spanId}-00'*/ SELECT $1::text as message`) }).then(done, done) @@ -616,8 +619,8 @@ describe('Plugin', () => { agent.use(traces => { expect(queryText).to.equal( - `/*dddbs='post',dde='tester',ddps='test',ddpv='8.4.0'` + - `*/ SELECT $1::text as message`) + `/*dddb='postgres',dddbs='post',dde='tester',ddh='127.0.0.1',ddps='test',ddpv='${ddpv}'` + + '*/ SELECT $1::text as message') }).then(done, done) client.query(query, ['Hello world!'], (err) => { @@ -640,8 +643,8 @@ describe('Plugin', () => { agent.use(traces => { expect(queryText).to.equal( - `/*dddbs='post',dde='tester',ddps='test',ddpv='8.4.0'` + - `*/ SELECT $1::text as message`) + `/*dddb='postgres',dddbs='post',dde='tester',ddh='127.0.0.1',ddps='test',ddpv='${ddpv}'` + + '*/ SELECT $1::text as message') }).then(done, done) client.query(query, ['Hello world!'], (err) => { @@ -674,8 +677,8 @@ describe('Plugin', () => { agent.use(traces => { expect(queryText).to.equal( - `/*dddbs='post',dde='tester',ddps='test',ddpv='8.4.0'` + - `*/ SELECT $1::text as greeting`) + `/*dddb='postgres',dddbs='post',dde='tester',ddh='127.0.0.1',ddps='test',ddpv='${ddpv}'` + + '*/ SELECT $1::text as greeting') }).then(done, done) client.query(query, ['Goodbye'], (err) => { diff --git a/packages/datadog-plugin-pg/test/integration-test/client.spec.js b/packages/datadog-plugin-pg/test/integration-test/client.spec.js index 83542c39758..18e97dac50e 100644 --- a/packages/datadog-plugin-pg/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-pg/test/integration-test/client.spec.js @@ -17,7 +17,7 @@ describe('esm', () => { before(async function () { this.timeout(20000) sandbox = await createSandbox([`'pg@${version}'`], false, [ - `./packages/datadog-plugin-pg/test/integration-test/*`]) + './packages/datadog-plugin-pg/test/integration-test/*']) }) after(async () => { diff --git a/packages/datadog-plugin-pg/test/leak.js b/packages/datadog-plugin-pg/test/leak.js deleted file mode 100644 index d5230394227..00000000000 --- a/packages/datadog-plugin-pg/test/leak.js +++ /dev/null @@ -1,28 +0,0 @@ -'use strict' - -require('../../dd-trace') - .init({ plugins: false, sampleRate: 0 }) - .use('pg') - -const test = require('tape') -const pg = require('../../../versions/pg').get() -const profile = require('../../dd-trace/test/profile') - -test('pg plugin should not leak', t => { - const client = new pg.Client({ - user: 'postgres', - password: 'postgres', - database: 'postgres', - application_name: 'test' - }) - - client.connect(err => { - if (err) return t.fail(err) - - profile(t, operation).then(() => client.end()) - }) - - function operation (done) { - client.query('SELECT 1 + 1 AS solution', done) - } -}) diff --git a/packages/datadog-plugin-pg/test/naming.js b/packages/datadog-plugin-pg/test/naming.js index 5d7efe729eb..a961906a417 100644 --- a/packages/datadog-plugin-pg/test/naming.js +++ b/packages/datadog-plugin-pg/test/naming.js @@ -14,6 +14,6 @@ const rawExpectedSchema = { } module.exports = { - rawExpectedSchema: rawExpectedSchema, + rawExpectedSchema, expectedSchema: resolveNaming(rawExpectedSchema) } diff --git a/packages/datadog-plugin-pino/test/index.spec.js b/packages/datadog-plugin-pino/test/index.spec.js index 77048a6d6ac..913885c8fca 100644 --- a/packages/datadog-plugin-pino/test/index.spec.js +++ b/packages/datadog-plugin-pino/test/index.spec.js @@ -3,6 +3,7 @@ const Writable = require('stream').Writable const agent = require('../../dd-trace/test/plugins/agent') const semver = require('semver') +const { NODE_MAJOR } = require('../../../version') describe('Plugin', () => { let logger @@ -34,7 +35,7 @@ describe('Plugin', () => { if (semver.intersects(version, '>=8') && options.prettyPrint) { delete options.prettyPrint // deprecated - const pretty = require(`../../../versions/pino-pretty@8.0.0`).get() + const pretty = require('../../../versions/pino-pretty@8.0.0').get() stream = pretty().pipe(stream) } @@ -131,8 +132,11 @@ describe('Plugin', () => { expect(record.err).to.have.property('stack', error.stack) } else { // pino <7 expect(record).to.have.property('msg', error.message) - expect(record).to.have.property('type', 'Error') - expect(record).to.have.property('stack', error.stack) + // ** TODO ** add this back once we fix it + if (NODE_MAJOR < 21) { + expect(record).to.have.property('type', 'Error') + expect(record).to.have.property('stack', error.stack) + } } }) }) diff --git a/packages/datadog-plugin-pino/test/integration-test/client.spec.js b/packages/datadog-plugin-pino/test/integration-test/client.spec.js index 33cee141165..389b765eaa9 100644 --- a/packages/datadog-plugin-pino/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-pino/test/integration-test/client.spec.js @@ -16,7 +16,7 @@ describe('esm', () => { before(async function () { this.timeout(20000) sandbox = await createSandbox([`'pino@${version}'`], - false, [`./packages/datadog-plugin-pino/test/integration-test/*`]) + false, ['./packages/datadog-plugin-pino/test/integration-test/*']) }) after(async () => { diff --git a/packages/datadog-plugin-playwright/src/index.js b/packages/datadog-plugin-playwright/src/index.js index 928477ffc3b..941f779ff54 100644 --- a/packages/datadog-plugin-playwright/src/index.js +++ b/packages/datadog-plugin-playwright/src/index.js @@ -8,10 +8,22 @@ const { finishAllTraceSpans, getTestSuitePath, getTestSuiteCommonTags, - TEST_SOURCE_START + TEST_SOURCE_START, + TEST_CODE_OWNERS, + TEST_SOURCE_FILE, + TEST_CONFIGURATION_BROWSER_NAME, + TEST_IS_NEW, + TEST_IS_RETRY, + TEST_EARLY_FLAKE_ENABLED, + TELEMETRY_TEST_SESSION } = require('../../dd-trace/src/plugins/util/test') const { RESOURCE_NAME } = require('../../../ext/tags') const { COMPONENT } = require('../../dd-trace/src/constants') +const { + TELEMETRY_EVENT_CREATED, + TELEMETRY_EVENT_FINISHED +} = require('../../dd-trace/src/ci-visibility/telemetry') +const { appClosing: appClosingTelemetry } = require('../../dd-trace/src/telemetry') class PlaywrightPlugin extends CiPlugin { static get id () { @@ -22,20 +34,42 @@ class PlaywrightPlugin extends CiPlugin { super(...args) this._testSuites = new Map() + this.numFailedTests = 0 + this.numFailedSuites = 0 - this.addSub('ci:playwright:session:finish', ({ status, onDone }) => { + this.addSub('ci:playwright:session:finish', ({ status, isEarlyFlakeDetectionEnabled, onDone }) => { this.testModuleSpan.setTag(TEST_STATUS, status) this.testSessionSpan.setTag(TEST_STATUS, status) + if (isEarlyFlakeDetectionEnabled) { + this.testSessionSpan.setTag(TEST_EARLY_FLAKE_ENABLED, 'true') + } + + if (this.numFailedSuites > 0) { + let errorMessage = `Test suites failed: ${this.numFailedSuites}.` + if (this.numFailedTests > 0) { + errorMessage += ` Tests failed: ${this.numFailedTests}` + } + const error = new Error(errorMessage) + this.testModuleSpan.setTag('error', error) + this.testSessionSpan.setTag('error', error) + } + this.testModuleSpan.finish() + this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'module') this.testSessionSpan.finish() + this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'session') finishAllTraceSpans(this.testSessionSpan) + this.telemetry.count(TELEMETRY_TEST_SESSION, { provider: this.ciProviderName }) + appClosingTelemetry() this.tracer._exporter.flush(onDone) + this.numFailedTests = 0 }) this.addSub('ci:playwright:test-suite:start', (testSuiteAbsolutePath) => { const store = storage.getStore() const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.rootDir) + const testSourceFile = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) const testSuiteMetadata = getTestSuiteCommonTags( this.command, @@ -43,6 +77,14 @@ class PlaywrightPlugin extends CiPlugin { testSuite, 'playwright' ) + if (testSourceFile) { + testSuiteMetadata[TEST_SOURCE_FILE] = testSourceFile + testSuiteMetadata[TEST_SOURCE_START] = 1 + } + const codeOwners = this.getCodeOwners(testSuiteMetadata) + if (codeOwners) { + testSuiteMetadata[TEST_CODE_OWNERS] = codeOwners + } const testSuiteSpan = this.tracer.startSpan('playwright.test_suite', { childOf: this.testModuleSpan, @@ -52,27 +94,40 @@ class PlaywrightPlugin extends CiPlugin { ...testSuiteMetadata } }) + this.telemetry.ciVisEvent(TELEMETRY_EVENT_CREATED, 'suite') this.enter(testSuiteSpan, store) this._testSuites.set(testSuite, testSuiteSpan) }) - this.addSub('ci:playwright:test-suite:finish', (status) => { + this.addSub('ci:playwright:test-suite:finish', ({ status, error }) => { const store = storage.getStore() const span = store && store.span if (!span) return - span.setTag(TEST_STATUS, status) + if (error) { + span.setTag('error', error) + span.setTag(TEST_STATUS, 'fail') + } else { + span.setTag(TEST_STATUS, status) + } + + if (status === 'fail' || error) { + this.numFailedSuites++ + } + span.finish() + this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'suite') }) - this.addSub('ci:playwright:test:start', ({ testName, testSuiteAbsolutePath, testSourceLine }) => { + this.addSub('ci:playwright:test:start', ({ testName, testSuiteAbsolutePath, testSourceLine, browserName }) => { const store = storage.getStore() const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.rootDir) - const span = this.startTestSpan(testName, testSuite, testSourceLine) + const testSourceFile = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) + const span = this.startTestSpan(testName, testSuite, testSourceFile, testSourceLine, browserName) this.enter(span, store) }) - this.addSub('ci:playwright:test:finish', ({ testStatus, steps, error, extraTags }) => { + this.addSub('ci:playwright:test:finish', ({ testStatus, steps, error, extraTags, isNew, isEfdRetry, isRetry }) => { const store = storage.getStore() const span = store && store.span if (!span) return @@ -85,6 +140,15 @@ class PlaywrightPlugin extends CiPlugin { if (extraTags) { span.addTags(extraTags) } + if (isNew) { + span.setTag(TEST_IS_NEW, 'true') + if (isEfdRetry) { + span.setTag(TEST_IS_RETRY, 'true') + } + } + if (isRetry) { + span.setTag(TEST_IS_RETRY, 'true') + } steps.forEach(step => { const stepStartTime = step.startTime.getTime() @@ -100,17 +164,46 @@ class PlaywrightPlugin extends CiPlugin { if (step.error) { stepSpan.setTag('error', step.error) } - stepSpan.finish(stepStartTime + step.duration) + let stepDuration = step.duration + if (stepDuration <= 0 || isNaN(stepDuration)) { + stepDuration = 0 + } + stepSpan.finish(stepStartTime + stepDuration) }) + if (testStatus === 'fail') { + this.numFailedTests++ + } + + this.telemetry.ciVisEvent( + TELEMETRY_EVENT_FINISHED, + 'test', + { + hasCodeOwners: !!span.context()._tags[TEST_CODE_OWNERS], + isNew, + browserDriver: 'playwright' + } + ) span.finish() + finishAllTraceSpans(span) }) } - startTestSpan (testName, testSuite, testSourceLine) { + startTestSpan (testName, testSuite, testSourceFile, testSourceLine, browserName) { const testSuiteSpan = this._testSuites.get(testSuite) - return super.startTestSpan(testName, testSuite, testSuiteSpan, { [TEST_SOURCE_START]: testSourceLine }) + + const extraTags = { + [TEST_SOURCE_START]: testSourceLine + } + if (testSourceFile) { + extraTags[TEST_SOURCE_FILE] = testSourceFile || testSuite + } + if (browserName) { + extraTags[TEST_CONFIGURATION_BROWSER_NAME] = browserName + } + + return super.startTestSpan(testName, testSuite, testSuiteSpan, extraTags) } } diff --git a/packages/datadog-plugin-protobufjs/src/index.js b/packages/datadog-plugin-protobufjs/src/index.js new file mode 100644 index 00000000000..800c3d9e3cb --- /dev/null +++ b/packages/datadog-plugin-protobufjs/src/index.js @@ -0,0 +1,14 @@ +const SchemaPlugin = require('../../dd-trace/src/plugins/schema') +const SchemaExtractor = require('./schema_iterator') + +class ProtobufjsPlugin extends SchemaPlugin { + static get id () { + return 'protobufjs' + } + + static get schemaExtractor () { + return SchemaExtractor + } +} + +module.exports = ProtobufjsPlugin diff --git a/packages/datadog-plugin-protobufjs/src/schema_iterator.js b/packages/datadog-plugin-protobufjs/src/schema_iterator.js new file mode 100644 index 00000000000..ea3c8ba2bf0 --- /dev/null +++ b/packages/datadog-plugin-protobufjs/src/schema_iterator.js @@ -0,0 +1,180 @@ +const PROTOBUF = 'protobuf' +const { + SCHEMA_DEFINITION, + SCHEMA_ID, + SCHEMA_NAME, + SCHEMA_OPERATION, + SCHEMA_WEIGHT, + SCHEMA_TYPE +} = require('../../dd-trace/src/constants') +const log = require('../../dd-trace/src/log') +const { + SchemaBuilder +} = require('../../dd-trace/src/datastreams/schemas/schema_builder') + +class SchemaExtractor { + constructor (schema) { + this.schema = schema + } + + static getTypeAndFormat (type) { + const typeFormatMapping = { + int32: ['integer', 'int32'], + int64: ['integer', 'int64'], + uint32: ['integer', 'uint32'], + uint64: ['integer', 'uint64'], + sint32: ['integer', 'sint32'], + sint64: ['integer', 'sint64'], + fixed32: ['integer', 'fixed32'], + fixed64: ['integer', 'fixed64'], + sfixed32: ['integer', 'sfixed32'], + sfixed64: ['integer', 'sfixed64'], + float: ['number', 'float'], + double: ['number', 'double'], + bool: ['boolean', null], + string: ['string', null], + bytes: ['string', 'byte'], + Enum: ['enum', null], + Type: ['type', null], + map: ['map', null], + repeated: ['array', null] + } + + return typeFormatMapping[type] || ['string', null] + } + + static extractProperty (field, schemaName, fieldName, builder, depth) { + let array = false + let description + let ref + let enumValues + + const resolvedType = field.resolvedType ? field.resolvedType.constructor.name : field.type + + const isRepeatedField = field.rule === 'repeated' + + let typeFormat = this.getTypeAndFormat(isRepeatedField ? 'repeated' : resolvedType) + let type = typeFormat[0] + let format = typeFormat[1] + + if (type === 'array') { + array = true + typeFormat = this.getTypeAndFormat(resolvedType) + type = typeFormat[0] + format = typeFormat[1] + } + + if (type === 'type') { + format = null + ref = `#/components/schemas/${removeLeadingPeriod(field.resolvedType.fullName)}` + // keep a reference to the original builder iterator since when we recurse this reference will get reset to + // deeper schemas + const originalSchemaExtractor = builder.iterator + if (!this.extractSchema(field.resolvedType, builder, depth, this)) { + return false + } + type = 'object' + builder.iterator = originalSchemaExtractor + } else if (type === 'enum') { + enumValues = [] + let i = 0 + while (field.resolvedType.valuesById[i]) { + enumValues.push(field.resolvedType.valuesById[i]) + i += 1 + } + } + return builder.addProperty(schemaName, fieldName, array, type, description, ref, format, enumValues) + } + + static extractSchema (schema, builder, depth, extractor) { + depth += 1 + const schemaName = removeLeadingPeriod(schema.resolvedType ? schema.resolvedType.fullName : schema.fullName) + if (extractor) { + // if we already have a defined extractor, this is a nested schema. create a new extractor for the nested + // schema, ensure it is added to our schema builder's cache, and replace the builders iterator with our + // nested schema iterator / extractor. Once complete, add the new schema to our builder's schemas. + const nestedSchemaExtractor = new SchemaExtractor(schema) + builder.iterator = nestedSchemaExtractor + const nestedSchema = SchemaBuilder.getSchema(schemaName, nestedSchemaExtractor, builder) + for (const nestedSubSchemaName in nestedSchema.components.schemas) { + if (nestedSchema.components.schemas.hasOwnProperty(nestedSubSchemaName)) { + builder.schema.components.schemas[nestedSubSchemaName] = nestedSchema.components.schemas[nestedSubSchemaName] + } + } + return true + } else { + if (!builder.shouldExtractSchema(schemaName, depth)) { + return false + } + for (const field of schema.fieldsArray) { + if (!this.extractProperty(field, schemaName, field.name, builder, depth)) { + log.warn(`DSM: Unable to extract field with name: ${field.name} from Avro schema with name: ${schemaName}`) + } + } + return true + } + } + + static extractSchemas (descriptor, dataStreamsProcessor) { + const schemaName = removeLeadingPeriod( + descriptor.resolvedType ? descriptor.resolvedType.fullName : descriptor.fullName + ) + return dataStreamsProcessor.getSchema(schemaName, new SchemaExtractor(descriptor)) + } + + iterateOverSchema (builder) { + this.constructor.extractSchema(this.schema, builder, 0) + } + + static attachSchemaOnSpan (args, span, operation, tracer) { + const { messageClass } = args + const descriptor = messageClass.$type ?? messageClass + + if (!descriptor || !span) { + return + } + + if (span.context()._tags[SCHEMA_TYPE] && operation === 'serialization') { + // we have already added a schema to this span, this call is an encode of nested schema types + return + } + + span.setTag(SCHEMA_TYPE, PROTOBUF) + span.setTag(SCHEMA_NAME, removeLeadingPeriod(descriptor.fullName)) + span.setTag(SCHEMA_OPERATION, operation) + + if (!tracer._dataStreamsProcessor.canSampleSchema(operation)) { + return + } + + // if the span is unsampled, do not sample the schema + if (!tracer._prioritySampler.isSampled(span)) { + return + } + + const weight = tracer._dataStreamsProcessor.trySampleSchema(operation) + if (weight === 0) { + return + } + + const schemaData = SchemaBuilder.getSchemaDefinition( + this.extractSchemas(descriptor, tracer._dataStreamsProcessor) + ) + + span.setTag(SCHEMA_DEFINITION, schemaData.definition) + span.setTag(SCHEMA_WEIGHT, weight) + span.setTag(SCHEMA_ID, schemaData.id) + } +} + +function removeLeadingPeriod (str) { + // Check if the first character is a period + if (str.charAt(0) === '.') { + // Remove the first character + return str.slice(1) + } + // Return the original string if the first character is not a period + return str +} + +module.exports = SchemaExtractor diff --git a/packages/datadog-plugin-protobufjs/test/helpers.js b/packages/datadog-plugin-protobufjs/test/helpers.js new file mode 100644 index 00000000000..d91be2e496b --- /dev/null +++ b/packages/datadog-plugin-protobufjs/test/helpers.js @@ -0,0 +1,104 @@ +async function loadMessage (protobuf, messageTypeName) { + if (messageTypeName === 'OtherMessage') { + const root = await protobuf.load('packages/datadog-plugin-protobufjs/test/schemas/other_message.proto') + const OtherMessage = root.lookupType('OtherMessage') + const message = OtherMessage.create({ + name: ['Alice'], + age: 30 + }) + return { + OtherMessage: { + type: OtherMessage, + instance: message + } + } + } else if (messageTypeName === 'MyMessage') { + const messageProto = await protobuf.load('packages/datadog-plugin-protobufjs/test/schemas/message.proto') + const otherMessageProto = await protobuf.load( + 'packages/datadog-plugin-protobufjs/test/schemas/other_message.proto' + ) + const Status = messageProto.lookupEnum('Status') + const MyMessage = messageProto.lookupType('MyMessage') + const OtherMessage = otherMessageProto.lookupType('OtherMessage') + const message = MyMessage.create({ + id: '123', + value: 'example_value', + status: Status.values.ACTIVE, + otherMessage: [ + OtherMessage.create({ name: ['Alice'], age: 30 }), + OtherMessage.create({ name: ['Bob'], age: 25 }) + ] + }) + return { + OtherMessage: { + type: OtherMessage, + instance: null + }, + MyMessage: { + type: MyMessage, + instance: message + } + } + } else if (messageTypeName === 'MainMessage') { + const root = await protobuf.load('packages/datadog-plugin-protobufjs/test/schemas/all_types.proto') + + const Status = root.lookupEnum('example.Status') + const Scalars = root.lookupType('example.Scalars') + const NestedMessage = root.lookupType('example.NestedMessage') + const ComplexMessage = root.lookupType('example.ComplexMessage') + const MainMessage = root.lookupType('example.MainMessage') + + // Create instances of the messages + const scalarsInstance = Scalars.create({ + int32Field: 42, + int64Field: 123456789012345, + uint32Field: 123, + uint64Field: 123456789012345, + sint32Field: -42, + sint64Field: -123456789012345, + fixed32Field: 42, + fixed64Field: 123456789012345, + sfixed32Field: -42, + sfixed64Field: -123456789012345, + floatField: 3.14, + doubleField: 2.718281828459, + boolField: true, + stringField: 'Hello, world!', + bytesField: Buffer.from('bytes data') + }) + + const nestedMessageInstance = NestedMessage.create({ + id: 'nested_id_123', + scalars: scalarsInstance + }) + + const complexMessageInstance = ComplexMessage.create({ + repeatedField: ['item1', 'item2', 'item3'], + mapField: { + key1: scalarsInstance, + key2: Scalars.create({ + int32Field: 24, + stringField: 'Another string' + }) + } + }) + + const mainMessageInstance = MainMessage.create({ + status: Status.values.ACTIVE, + scalars: scalarsInstance, + nested: nestedMessageInstance, + complex: complexMessageInstance + }) + + return { + MainMessage: { + type: MainMessage, + instance: mainMessageInstance + } + } + } +} + +module.exports = { + loadMessage +} diff --git a/packages/datadog-plugin-protobufjs/test/index.spec.js b/packages/datadog-plugin-protobufjs/test/index.spec.js new file mode 100644 index 00000000000..30e95687bac --- /dev/null +++ b/packages/datadog-plugin-protobufjs/test/index.spec.js @@ -0,0 +1,352 @@ +'use strict' + +const fs = require('fs') +const { expect } = require('chai') +const agent = require('../../dd-trace/test/plugins/agent') +const path = require('path') +const { + SCHEMA_DEFINITION, + SCHEMA_ID, + SCHEMA_NAME, + SCHEMA_OPERATION, + SCHEMA_WEIGHT, + SCHEMA_TYPE +} = require('../../dd-trace/src/constants') +const sinon = require('sinon') +const { loadMessage } = require('./helpers') +const { SchemaBuilder } = require('../../dd-trace/src/datastreams/schemas/schema_builder') + +const schemas = JSON.parse(fs.readFileSync(path.join(__dirname, 'schemas/expected_schemas.json'), 'utf8')) +const MESSAGE_SCHEMA_DEF = schemas.MESSAGE_SCHEMA_DEF +const OTHER_MESSAGE_SCHEMA_DEF = schemas.OTHER_MESSAGE_SCHEMA_DEF +const ALL_TYPES_MESSAGE_SCHEMA_DEF = schemas.ALL_TYPES_MESSAGE_SCHEMA_DEF + +const MESSAGE_SCHEMA_ID = '666607144722735562' +const OTHER_MESSAGE_SCHEMA_ID = '2691489402935632768' +const ALL_TYPES_MESSAGE_SCHEMA_ID = '15890948796193489151' + +function compareJson (expected, span) { + const actual = JSON.parse(span.context()._tags[SCHEMA_DEFINITION]) + return JSON.stringify(actual) === JSON.stringify(expected) +} + +describe('Plugin', () => { + describe('protobufjs', function () { + let tracer + let protobuf + let dateNowStub + let mockTime = 0 + + withVersions('protobufjs', ['protobufjs'], (version) => { + before(() => { + tracer = require('../../dd-trace').init() + // reset sampled schemas + if (tracer._dataStreamsProcessor?._schemaSamplers) { + tracer._dataStreamsProcessor._schemaSamplers = [] + } + }) + + describe('without configuration', () => { + before(() => { + dateNowStub = sinon.stub(Date, 'now').callsFake(() => { + const returnValue = mockTime + mockTime += 50000 // Increment by 50000 ms to ensure each DSM schema is sampled + return returnValue + }) + const cache = SchemaBuilder.getCache() + cache.clear() + return agent.load('protobufjs').then(() => { + protobuf = require(`../../../versions/protobufjs@${version}`).get() + }) + }) + + after(() => { + dateNowStub.restore() + return agent.close({ ritmReset: false }) + }) + + it('should serialize basic schema correctly', async () => { + const loadedMessages = await loadMessage(protobuf, 'OtherMessage') + + tracer.trace('other_message.serialize', span => { + loadedMessages.OtherMessage.type.encode(loadedMessages.OtherMessage.instance).finish() + + expect(span._name).to.equal('other_message.serialize') + + expect(compareJson(OTHER_MESSAGE_SCHEMA_DEF, span)).to.equal(true) + expect(span.context()._tags).to.have.property(SCHEMA_TYPE, 'protobuf') + expect(span.context()._tags).to.have.property(SCHEMA_NAME, 'OtherMessage') + expect(span.context()._tags).to.have.property(SCHEMA_OPERATION, 'serialization') + expect(span.context()._tags).to.have.property(SCHEMA_ID, OTHER_MESSAGE_SCHEMA_ID) + expect(span.context()._tags).to.have.property(SCHEMA_WEIGHT, 1) + }) + }) + + it('should load using a callback instead of promise', async () => { + const loadedMessages = loadMessage(protobuf, 'OtherMessage', () => { + tracer.trace('other_message.serialize', span => { + loadedMessages.OtherMessage.type.encode(loadedMessages.OtherMessage.instance).finish() + + expect(span._name).to.equal('other_message.serialize') + + expect(compareJson(OTHER_MESSAGE_SCHEMA_DEF, span)).to.equal(true) + expect(span.context()._tags).to.have.property(SCHEMA_TYPE, 'protobuf') + expect(span.context()._tags).to.have.property(SCHEMA_NAME, 'OtherMessage') + expect(span.context()._tags).to.have.property(SCHEMA_OPERATION, 'serialization') + expect(span.context()._tags).to.have.property(SCHEMA_ID, OTHER_MESSAGE_SCHEMA_ID) + expect(span.context()._tags).to.have.property(SCHEMA_WEIGHT, 1) + }) + }) + }) + + it('should serialize complex schema correctly', async () => { + const loadedMessages = await loadMessage(protobuf, 'MyMessage') + + tracer.trace('message_pb2.serialize', span => { + loadedMessages.MyMessage.type.encode(loadedMessages.MyMessage.instance).finish() + + expect(span._name).to.equal('message_pb2.serialize') + + expect(compareJson(MESSAGE_SCHEMA_DEF, span)).to.equal(true) + expect(span.context()._tags).to.have.property(SCHEMA_TYPE, 'protobuf') + expect(span.context()._tags).to.have.property(SCHEMA_NAME, 'MyMessage') + expect(span.context()._tags).to.have.property(SCHEMA_OPERATION, 'serialization') + expect(span.context()._tags).to.have.property(SCHEMA_ID, MESSAGE_SCHEMA_ID) + expect(span.context()._tags).to.have.property(SCHEMA_WEIGHT, 1) + }) + }) + + it('should serialize schema with all types correctly', async () => { + const loadedMessages = await loadMessage(protobuf, 'MainMessage') + + tracer.trace('all_types.serialize', span => { + loadedMessages.MainMessage.type.encode(loadedMessages.MainMessage.instance).finish() + + expect(span._name).to.equal('all_types.serialize') + + expect(compareJson(ALL_TYPES_MESSAGE_SCHEMA_DEF, span)).to.equal(true) + expect(span.context()._tags).to.have.property(SCHEMA_TYPE, 'protobuf') + expect(span.context()._tags).to.have.property(SCHEMA_NAME, 'example.MainMessage') + expect(span.context()._tags).to.have.property(SCHEMA_OPERATION, 'serialization') + expect(span.context()._tags).to.have.property(SCHEMA_ID, ALL_TYPES_MESSAGE_SCHEMA_ID) + expect(span.context()._tags).to.have.property(SCHEMA_WEIGHT, 1) + }) + }) + + it('should deserialize basic schema correctly', async () => { + const loadedMessages = await loadMessage(protobuf, 'OtherMessage') + + const bytes = loadedMessages.OtherMessage.type.encode(loadedMessages.OtherMessage.instance).finish() + + tracer.trace('other_message.deserialize', span => { + loadedMessages.OtherMessage.type.decode(bytes) + + expect(span._name).to.equal('other_message.deserialize') + + expect(compareJson(OTHER_MESSAGE_SCHEMA_DEF, span)).to.equal(true) + expect(span.context()._tags).to.have.property(SCHEMA_TYPE, 'protobuf') + expect(span.context()._tags).to.have.property(SCHEMA_NAME, 'OtherMessage') + expect(span.context()._tags).to.have.property(SCHEMA_OPERATION, 'deserialization') + expect(span.context()._tags).to.have.property(SCHEMA_ID, OTHER_MESSAGE_SCHEMA_ID) + expect(span.context()._tags).to.have.property(SCHEMA_WEIGHT, 1) + }) + }) + + it('should deserialize complex schema correctly', async () => { + const loadedMessages = await loadMessage(protobuf, 'MyMessage') + + const bytes = loadedMessages.MyMessage.type.encode(loadedMessages.MyMessage.instance).finish() + + tracer.trace('my_message.deserialize', span => { + loadedMessages.MyMessage.type.decode(bytes) + + expect(span._name).to.equal('my_message.deserialize') + + expect(compareJson(MESSAGE_SCHEMA_DEF, span)).to.equal(true) + expect(span.context()._tags).to.have.property(SCHEMA_TYPE, 'protobuf') + expect(span.context()._tags).to.have.property(SCHEMA_NAME, 'MyMessage') + expect(span.context()._tags).to.have.property(SCHEMA_OPERATION, 'deserialization') + expect(span.context()._tags).to.have.property(SCHEMA_ID, MESSAGE_SCHEMA_ID) + expect(span.context()._tags).to.have.property(SCHEMA_WEIGHT, 1) + }) + }) + + it('should deserialize all types schema correctly', async () => { + const loadedMessages = await loadMessage(protobuf, 'MainMessage') + + const bytes = loadedMessages.MainMessage.type.encode(loadedMessages.MainMessage.instance).finish() + + tracer.trace('all_types.deserialize', span => { + loadedMessages.MainMessage.type.decode(bytes) + + expect(span._name).to.equal('all_types.deserialize') + + expect(compareJson(ALL_TYPES_MESSAGE_SCHEMA_DEF, span)).to.equal(true) + expect(span.context()._tags).to.have.property(SCHEMA_TYPE, 'protobuf') + expect(span.context()._tags).to.have.property(SCHEMA_NAME, 'example.MainMessage') + expect(span.context()._tags).to.have.property(SCHEMA_OPERATION, 'deserialization') + expect(span.context()._tags).to.have.property(SCHEMA_ID, ALL_TYPES_MESSAGE_SCHEMA_ID) + expect(span.context()._tags).to.have.property(SCHEMA_WEIGHT, 1) + }) + }) + + it('should wrap encode and decode for fromObject', async () => { + const root = await protobuf.load('packages/datadog-plugin-protobufjs/test/schemas/other_message.proto') + const OtherMessage = root.lookupType('OtherMessage') + const messageObject = { + name: ['Alice'], + age: 30 + } + const message = OtherMessage.fromObject(messageObject) + + const bytes = OtherMessage.encode(message).finish() + + tracer.trace('other_message.deserialize', span => { + OtherMessage.decode(bytes) + + expect(span._name).to.equal('other_message.deserialize') + + expect(compareJson(OTHER_MESSAGE_SCHEMA_DEF, span)).to.equal(true) + expect(span.context()._tags).to.have.property(SCHEMA_TYPE, 'protobuf') + expect(span.context()._tags).to.have.property(SCHEMA_NAME, 'OtherMessage') + expect(span.context()._tags).to.have.property(SCHEMA_OPERATION, 'deserialization') + expect(span.context()._tags).to.have.property(SCHEMA_ID, OTHER_MESSAGE_SCHEMA_ID) + expect(span.context()._tags).to.have.property(SCHEMA_WEIGHT, 1) + }) + }) + + it('should wrap decodeDelimited', async () => { + const root = await protobuf.load('packages/datadog-plugin-protobufjs/test/schemas/other_message.proto') + const OtherMessage = root.lookupType('OtherMessage') + const message = OtherMessage.create({ + name: ['Alice'], + age: 30 + }) + + const bytes = OtherMessage.encodeDelimited(message).finish() + + tracer.trace('other_message.deserialize', span => { + OtherMessage.decodeDelimited(bytes) + + expect(span._name).to.equal('other_message.deserialize') + + expect(compareJson(OTHER_MESSAGE_SCHEMA_DEF, span)).to.equal(true) + expect(span.context()._tags).to.have.property(SCHEMA_TYPE, 'protobuf') + expect(span.context()._tags).to.have.property(SCHEMA_NAME, 'OtherMessage') + expect(span.context()._tags).to.have.property(SCHEMA_OPERATION, 'deserialization') + expect(span.context()._tags).to.have.property(SCHEMA_ID, OTHER_MESSAGE_SCHEMA_ID) + expect(span.context()._tags).to.have.property(SCHEMA_WEIGHT, 1) + }) + }) + + it('should load using direct type creation', () => { + const OtherMessage = new protobuf.Type('OtherMessage') + .add(new protobuf.Field('name', 1, 'string', 'repeated')) + .add(new protobuf.Field('age', 2, 'int32')) + + const message = OtherMessage.create({ + name: ['Alice'], + age: 30 + }) + + const bytes = OtherMessage.encodeDelimited(message).finish() + + tracer.trace('other_message.deserialize', span => { + OtherMessage.decodeDelimited(bytes) + + expect(span._name).to.equal('other_message.deserialize') + + expect(compareJson(OTHER_MESSAGE_SCHEMA_DEF, span)).to.equal(true) + expect(span.context()._tags).to.have.property(SCHEMA_TYPE, 'protobuf') + expect(span.context()._tags).to.have.property(SCHEMA_NAME, 'OtherMessage') + expect(span.context()._tags).to.have.property(SCHEMA_OPERATION, 'deserialization') + expect(span.context()._tags).to.have.property(SCHEMA_ID, OTHER_MESSAGE_SCHEMA_ID) + expect(span.context()._tags).to.have.property(SCHEMA_WEIGHT, 1) + }) + }) + + it('should load using JSON descriptors', () => { + const jsonDescriptor = require('./schemas/other_message_proto.json') + const root = protobuf.Root.fromJSON(jsonDescriptor) + const OtherMessage = root.lookupType('OtherMessage') + + const message = OtherMessage.create({ + name: ['Alice'], + age: 30 + }) + + const bytes = OtherMessage.encodeDelimited(message).finish() + + tracer.trace('other_message.deserialize', span => { + OtherMessage.decodeDelimited(bytes) + + expect(span._name).to.equal('other_message.deserialize') + + expect(compareJson(OTHER_MESSAGE_SCHEMA_DEF, span)).to.equal(true) + expect(span.context()._tags).to.have.property(SCHEMA_TYPE, 'protobuf') + expect(span.context()._tags).to.have.property(SCHEMA_NAME, 'OtherMessage') + expect(span.context()._tags).to.have.property(SCHEMA_OPERATION, 'deserialization') + expect(span.context()._tags).to.have.property(SCHEMA_ID, OTHER_MESSAGE_SCHEMA_ID) + expect(span.context()._tags).to.have.property(SCHEMA_WEIGHT, 1) + }) + }) + + describe('during schema sampling', function () { + let cacheSetSpy + let cacheGetSpy + + beforeEach(() => { + const cache = SchemaBuilder.getCache() + cache.clear() + cacheSetSpy = sinon.spy(cache, 'set') + cacheGetSpy = sinon.spy(cache, 'get') + }) + + afterEach(() => { + cacheSetSpy.restore() + cacheGetSpy.restore() + }) + + it('should use the schema cache and not re-extract an already sampled schema', async () => { + const loadedMessages = await loadMessage(protobuf, 'MyMessage') + + tracer.trace('message_pb2.serialize', span => { + loadedMessages.MyMessage.type.encode(loadedMessages.MyMessage.instance).finish() + + expect(span._name).to.equal('message_pb2.serialize') + + expect(compareJson(MESSAGE_SCHEMA_DEF, span)).to.equal(true) + expect(span.context()._tags).to.have.property(SCHEMA_TYPE, 'protobuf') + expect(span.context()._tags).to.have.property(SCHEMA_NAME, 'MyMessage') + expect(span.context()._tags).to.have.property(SCHEMA_OPERATION, 'serialization') + expect(span.context()._tags).to.have.property(SCHEMA_ID, MESSAGE_SCHEMA_ID) + expect(span.context()._tags).to.have.property(SCHEMA_WEIGHT, 1) + + // we sampled 1 schema with 1 subschema, so the constructor should've only been called twice + expect(cacheSetSpy.callCount).to.equal(2) + expect(cacheGetSpy.callCount).to.equal(2) + }) + + tracer.trace('message_pb2.serialize', span => { + loadedMessages.MyMessage.type.encode(loadedMessages.MyMessage.instance).finish() + + expect(span._name).to.equal('message_pb2.serialize') + + expect(compareJson(MESSAGE_SCHEMA_DEF, span)).to.equal(true) + expect(span.context()._tags).to.have.property(SCHEMA_TYPE, 'protobuf') + expect(span.context()._tags).to.have.property(SCHEMA_NAME, 'MyMessage') + expect(span.context()._tags).to.have.property(SCHEMA_OPERATION, 'serialization') + expect(span.context()._tags).to.have.property(SCHEMA_ID, MESSAGE_SCHEMA_ID) + expect(span.context()._tags).to.have.property(SCHEMA_WEIGHT, 1) + + // ensure schema was sampled and returned via the cache, so no extra cache set + // calls were needed, only gets + expect(cacheSetSpy.callCount).to.equal(2) + expect(cacheGetSpy.callCount).to.equal(3) + }) + }) + }) + }) + }) + }) +}) diff --git a/packages/datadog-plugin-protobufjs/test/schemas/all_types.proto b/packages/datadog-plugin-protobufjs/test/schemas/all_types.proto new file mode 100644 index 00000000000..6cfc3b3ee3d --- /dev/null +++ b/packages/datadog-plugin-protobufjs/test/schemas/all_types.proto @@ -0,0 +1,49 @@ +syntax = "proto3"; + +package example; + +// Enum definition +enum Status { + UNKNOWN = 0; + ACTIVE = 1; + INACTIVE = 2; +} + +// Message with various number types and other scalar types +message Scalars { + int32 int32Field = 1; + int64 int64Field = 2; + uint32 uint32Field = 3; + uint64 uint64Field = 4; + sint32 sint32Field = 5; + sint64 sint64Field = 6; + fixed32 fixed32Field = 7; + fixed64 fixed64Field = 8; + sfixed32 sfixed32Field = 9; + sfixed64 sfixed64Field = 10; + float floatField = 11; + double doubleField = 12; + bool boolField = 13; + string stringField = 14; + bytes bytesField = 15; +} + +// Nested message definition +message NestedMessage { + string id = 1; + Scalars scalars = 2; +} + +// Message demonstrating the use of repeated fields and maps +message ComplexMessage { + repeated string repeatedField = 1; + map mapField = 2; +} + +// Main message that uses all the above elements +message MainMessage { + Status status = 1; + Scalars scalars = 2; + NestedMessage nested = 3; + ComplexMessage complex = 4; +} \ No newline at end of file diff --git a/packages/datadog-plugin-protobufjs/test/schemas/expected_schemas.json b/packages/datadog-plugin-protobufjs/test/schemas/expected_schemas.json new file mode 100644 index 00000000000..1825013519d --- /dev/null +++ b/packages/datadog-plugin-protobufjs/test/schemas/expected_schemas.json @@ -0,0 +1,195 @@ +{ + "MESSAGE_SCHEMA_DEF":{ + "openapi":"3.0.0", + "components":{ + "schemas":{ + "MyMessage":{ + "type":"object", + "properties":{ + "id":{ + "type":"string" + }, + "value":{ + "type":"string" + }, + "otherMessage":{ + "type":"array", + "items":{ + "type":"object", + "$ref":"#/components/schemas/OtherMessage" + } + }, + "status":{ + "type":"enum", + "enum":[ + "UNKNOWN", + "ACTIVE", + "INACTIVE", + "DELETED" + ] + } + } + }, + "OtherMessage":{ + "type":"object", + "properties":{ + "name":{ + "type":"array", + "items":{ + "type":"string" + } + }, + "age":{ + "type":"integer", + "format":"int32" + } + } + } + } + } + }, + "OTHER_MESSAGE_SCHEMA_DEF":{ + "openapi":"3.0.0", + "components":{ + "schemas":{ + "OtherMessage":{ + "type":"object", + "properties":{ + "name":{ + "type":"array", + "items":{ + "type":"string" + } + }, + "age":{ + "type":"integer", + "format":"int32" + } + } + } + } + } + }, + "ALL_TYPES_MESSAGE_SCHEMA_DEF":{ + "openapi":"3.0.0", + "components":{ + "schemas":{ + "example.MainMessage":{ + "type":"object", + "properties":{ + "status":{ + "type":"enum", + "enum":[ + "UNKNOWN", + "ACTIVE", + "INACTIVE" + ] + }, + "scalars":{ + "type":"object", + "$ref":"#/components/schemas/example.Scalars" + }, + "nested":{ + "type":"object", + "$ref":"#/components/schemas/example.NestedMessage" + }, + "complex":{ + "type":"object", + "$ref":"#/components/schemas/example.ComplexMessage" + } + } + }, + "example.Scalars":{ + "type":"object", + "properties":{ + "int32Field":{ + "type":"integer", + "format":"int32" + }, + "int64Field":{ + "type":"integer", + "format":"int64" + }, + "uint32Field":{ + "type":"integer", + "format":"uint32" + }, + "uint64Field":{ + "type":"integer", + "format":"uint64" + }, + "sint32Field":{ + "type":"integer", + "format":"sint32" + }, + "sint64Field":{ + "type":"integer", + "format":"sint64" + }, + "fixed32Field":{ + "type":"integer", + "format":"fixed32" + }, + "fixed64Field":{ + "type":"integer", + "format":"fixed64" + }, + "sfixed32Field":{ + "type":"integer", + "format":"sfixed32" + }, + "sfixed64Field":{ + "type":"integer", + "format":"sfixed64" + }, + "floatField":{ + "type":"number", + "format":"float" + }, + "doubleField":{ + "type":"number", + "format":"double" + }, + "boolField":{ + "type":"boolean" + }, + "stringField":{ + "type":"string" + }, + "bytesField":{ + "type":"string", + "format":"byte" + } + } + }, + "example.NestedMessage":{ + "type":"object", + "properties":{ + "id":{ + "type":"string" + }, + "scalars":{ + "type":"object", + "$ref":"#/components/schemas/example.Scalars" + } + } + }, + "example.ComplexMessage":{ + "type":"object", + "properties":{ + "repeatedField":{ + "type":"array", + "items":{ + "type":"string" + } + }, + "mapField":{ + "type":"object", + "$ref":"#/components/schemas/example.Scalars" + } + } + } + } + } + } +} \ No newline at end of file diff --git a/packages/datadog-plugin-protobufjs/test/schemas/message.proto b/packages/datadog-plugin-protobufjs/test/schemas/message.proto new file mode 100644 index 00000000000..6fd1c65fe06 --- /dev/null +++ b/packages/datadog-plugin-protobufjs/test/schemas/message.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; + +import "other_message.proto"; + +enum Status { + UNKNOWN = 0; + ACTIVE = 1; + INACTIVE = 2; + DELETED = 3; +} + +message MyMessage { + string id = 1; + string value = 2; + repeated OtherMessage otherMessage = 3; + Status status = 4; +} \ No newline at end of file diff --git a/packages/datadog-plugin-protobufjs/test/schemas/other_message.proto b/packages/datadog-plugin-protobufjs/test/schemas/other_message.proto new file mode 100644 index 00000000000..dbd6f368d7d --- /dev/null +++ b/packages/datadog-plugin-protobufjs/test/schemas/other_message.proto @@ -0,0 +1,6 @@ +syntax = "proto3"; + +message OtherMessage { + repeated string name = 1; + int32 age = 2; +} \ No newline at end of file diff --git a/packages/datadog-plugin-protobufjs/test/schemas/other_message_proto.json b/packages/datadog-plugin-protobufjs/test/schemas/other_message_proto.json new file mode 100644 index 00000000000..5a682ec89ca --- /dev/null +++ b/packages/datadog-plugin-protobufjs/test/schemas/other_message_proto.json @@ -0,0 +1,17 @@ +{ + "nested": { + "OtherMessage": { + "fields": { + "name": { + "rule": "repeated", + "type": "string", + "id": 1 + }, + "age": { + "type": "int32", + "id": 2 + } + } + } + } + } \ No newline at end of file diff --git a/packages/datadog-plugin-redis/test/integration-test/client.spec.js b/packages/datadog-plugin-redis/test/integration-test/client.spec.js index 9c65daa604e..89836ba66d8 100644 --- a/packages/datadog-plugin-redis/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-redis/test/integration-test/client.spec.js @@ -17,7 +17,7 @@ describe('esm', () => { before(async function () { this.timeout(20000) sandbox = await createSandbox([`'redis@${version}'`], false, [ - `./packages/datadog-plugin-redis/test/integration-test/*`]) + './packages/datadog-plugin-redis/test/integration-test/*']) }) after(async () => { diff --git a/packages/datadog-plugin-redis/test/leak.js b/packages/datadog-plugin-redis/test/leak.js deleted file mode 100644 index aaa0c96b40d..00000000000 --- a/packages/datadog-plugin-redis/test/leak.js +++ /dev/null @@ -1,19 +0,0 @@ -'use strict' - -require('../../dd-trace') - .init({ plugins: false, sampleRate: 0 }) - .use('redis') - -const test = require('tape') -const redis = require('../../../versions/redis').get() -const profile = require('../../dd-trace/test/profile') - -test('redis plugin should not leak', t => { - const client = redis.createClient() - - profile(t, operation).then(() => client.quit()) - - function operation (done) { - client.get('foo', done) - } -}) diff --git a/packages/datadog-plugin-redis/test/naming.js b/packages/datadog-plugin-redis/test/naming.js index 2df9f5056d8..1ed3f17e428 100644 --- a/packages/datadog-plugin-redis/test/naming.js +++ b/packages/datadog-plugin-redis/test/naming.js @@ -14,6 +14,6 @@ const rawExpectedSchema = { } module.exports = { - rawExpectedSchema: rawExpectedSchema, + rawExpectedSchema, expectedSchema: resolveNaming(rawExpectedSchema) } diff --git a/packages/datadog-plugin-restify/test/index.spec.js b/packages/datadog-plugin-restify/test/index.spec.js index ea7a5f17aa7..64284626c7c 100644 --- a/packages/datadog-plugin-restify/test/index.spec.js +++ b/packages/datadog-plugin-restify/test/index.spec.js @@ -2,7 +2,6 @@ const { AsyncLocalStorage } = require('async_hooks') const axios = require('axios') -const getPort = require('get-port') const semver = require('semver') const agent = require('../../dd-trace/test/plugins/agent') const { ERROR_MESSAGE } = require('../../dd-trace/src/constants') @@ -30,12 +29,15 @@ describe('Plugin', () => { describe('without configuration', () => { before(() => agent.load(['restify', 'find-my-way', 'http'], [{}, {}, { client: false }])) + after(() => agent.close({ ritmReset: false })) it('should do automatic instrumentation', done => { const server = restify.createServer() - getPort().then(port => { + appListener = server.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { expect(traces[0][0]).to.have.property('name', 'restify.request') @@ -51,11 +53,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(() => {}) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(() => {}) }) }) @@ -67,7 +67,9 @@ describe('Plugin', () => { return next() }) - getPort().then(port => { + appListener = server.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { expect(traces[0][0]).to.have.property('resource', 'GET /user/:id') @@ -77,11 +79,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user/123`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user/123`) + .catch(done) }) }) @@ -96,7 +96,9 @@ describe('Plugin', () => { } ) - getPort().then(port => { + appListener = server.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { expect(traces[0][0]).to.have.property('resource', 'GET /user/:id') @@ -106,11 +108,46 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user/123`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user/123`) + .catch(done) + }) + }) + + it('should route without producing any warnings', done => { + const warningSpy = sinon.spy((_, msg) => { + // eslint-disable-next-line no-console + console.error(`route called with warning: ${msg}`) + }) + + const server = restify.createServer({ + log: { + trace: () => {}, + warn: warningSpy + } + }) + + server.get( + '/user/:id', + async function middleware () {}, + async function handler (req, res) { + res.send('hello, ' + req.params.id) + } + ) + + appListener = server.listen(0, 'localhost', () => { + const port = appListener.address().port + + agent + .use(traces => { + expect(warningSpy).to.not.have.been.called + }) + .then(done) + .catch(done) + + axios + .get(`http://localhost:${port}/user/123`) + .catch(done) }) }) @@ -147,12 +184,12 @@ describe('Plugin', () => { next() }) - getPort().then(port => { - appListener = server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + appListener = server.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -164,7 +201,9 @@ describe('Plugin', () => { return next() }]) - getPort().then(port => { + appListener = server.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { expect(traces[0][0]).to.have.property('resource', 'GET /user/:id') @@ -174,11 +213,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user/123`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user/123`) + .catch(done) }) }) @@ -202,12 +239,12 @@ describe('Plugin', () => { res.end() }) - getPort().then(port => { - appListener = server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + appListener = server.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -227,7 +264,9 @@ describe('Plugin', () => { throw new Error('uncaught') }]) - getPort().then(port => { + appListener = server.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { expect(traces[0][0]).to.have.property('resource', 'GET /error') @@ -239,13 +278,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/error`, { - validateStatus: status => status === 599 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/error`, { + validateStatus: status => status === 599 + }) + .catch(done) }) }) }) diff --git a/packages/datadog-plugin-restify/test/integration-test/client.spec.js b/packages/datadog-plugin-restify/test/integration-test/client.spec.js index c7d09d33a68..674dbc9bc08 100644 --- a/packages/datadog-plugin-restify/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-restify/test/integration-test/client.spec.js @@ -19,7 +19,7 @@ describe('esm', () => { before(async function () { this.timeout(20000) sandbox = await createSandbox([`'restify@${version}'`], - false, [`./packages/datadog-plugin-restify/test/integration-test/*`]) + false, ['./packages/datadog-plugin-restify/test/integration-test/*']) }) after(async () => { diff --git a/packages/datadog-plugin-rhea/src/consumer.js b/packages/datadog-plugin-rhea/src/consumer.js index 1adece16fbd..56aad8f7b9d 100644 --- a/packages/datadog-plugin-rhea/src/consumer.js +++ b/packages/datadog-plugin-rhea/src/consumer.js @@ -2,6 +2,7 @@ const ConsumerPlugin = require('../../dd-trace/src/plugins/consumer') const { storage } = require('../../datadog-core') +const { getAmqpMessageSize } = require('../../dd-trace/src/datastreams/processor') class RheaConsumerPlugin extends ConsumerPlugin { static get id () { return 'rhea' } @@ -19,16 +20,28 @@ class RheaConsumerPlugin extends ConsumerPlugin { const name = getResourceNameFromMessage(msgObj) const childOf = extractTextMap(msgObj, this.tracer) - this.startSpan({ + const span = this.startSpan({ childOf, resource: name, type: 'worker', meta: { - 'component': 'rhea', + component: 'rhea', 'amqp.link.source.address': name, 'amqp.link.role': 'receiver' } }) + + if ( + this.config.dsmEnabled && + msgObj?.message?.delivery_annotations + ) { + const payloadSize = getAmqpMessageSize( + { headers: msgObj.message.delivery_annotations, content: msgObj.message.body } + ) + this.tracer.decodeDataStreamsContext(msgObj.message.delivery_annotations) + this.tracer + .setCheckpoint(['direction:in', `topic:${name}`, 'type:rabbitmq'], span, payloadSize) + } } } diff --git a/packages/datadog-plugin-rhea/src/producer.js b/packages/datadog-plugin-rhea/src/producer.js index 332aff1276d..8f2116d3d75 100644 --- a/packages/datadog-plugin-rhea/src/producer.js +++ b/packages/datadog-plugin-rhea/src/producer.js @@ -2,6 +2,8 @@ const { CLIENT_PORT_KEY } = require('../../dd-trace/src/constants') const ProducerPlugin = require('../../dd-trace/src/plugins/producer') +const { DsmPathwayCodec } = require('../../dd-trace/src/datastreams/pathway') +const { getAmqpMessageSize } = require('../../dd-trace/src/datastreams/processor') class RheaProducerPlugin extends ProducerPlugin { static get id () { return 'rhea' } @@ -17,7 +19,7 @@ class RheaProducerPlugin extends ProducerPlugin { this.startSpan({ resource: name, meta: { - 'component': 'rhea', + component: 'rhea', 'amqp.link.target.address': name, 'amqp.link.role': 'sender', 'out.host': host, @@ -36,6 +38,14 @@ function addDeliveryAnnotations (msg, tracer, span) { msg.delivery_annotations = msg.delivery_annotations || {} tracer.inject(span, 'text_map', msg.delivery_annotations) + + if (tracer._config.dsmEnabled) { + const targetName = span.context()._tags['amqp.link.target.address'] + const payloadSize = getAmqpMessageSize({ content: msg.body, headers: msg.delivery_annotations }) + const dataStreamsContext = tracer + .setCheckpoint(['direction:out', `exchange:${targetName}`, 'type:rabbitmq'], span, payloadSize) + DsmPathwayCodec.encode(dataStreamsContext, msg.delivery_annotations) + } } } diff --git a/packages/datadog-plugin-rhea/test/index.spec.js b/packages/datadog-plugin-rhea/test/index.spec.js index f55ead76fb8..8503046f0dd 100644 --- a/packages/datadog-plugin-rhea/test/index.spec.js +++ b/packages/datadog-plugin-rhea/test/index.spec.js @@ -8,8 +8,11 @@ const { expectedSchema, rawExpectedSchema } = require('./naming') describe('Plugin', () => { let tracer - describe('rhea', () => { - before(() => agent.load('rhea')) + describe('rhea', function () { + before(() => { + agent.load('rhea') + }) + after(() => agent.close({ ritmReset: false })) withVersions('rhea', 'rhea', version => { @@ -46,6 +49,84 @@ describe('Plugin', () => { connection.open_receiver('amq.topic') }) + const expectedProducerHash = '15837999642856815456' + const expectedConsumerHash = '18403970455318595370' + + it('Should set pathway hash tag on a span when producing', (done) => { + let produceSpanMeta = {} + agent.use(traces => { + const span = traces[0][0] + + if (span.meta['span.kind'] === 'producer') { + produceSpanMeta = span.meta + } + + expect(produceSpanMeta).to.include({ + 'pathway.hash': expectedProducerHash + }) + }, { timeoutMs: 2000 }).then(done, done) + + context.sender.send({ body: 'hello from DSM' }) + }) + + it('Should set pathway hash tag on a span when consuming', (done) => { + context.sender.send({ body: 'hello from DSM' }) + + container.once('message', msg => { + let consumeSpanMeta = {} + agent.use(traces => { + const span = traces[0][0] + + if (span.meta['span.kind'] === 'consumer') { + consumeSpanMeta = span.meta + } + + expect(consumeSpanMeta).to.include({ + 'pathway.hash': expectedConsumerHash + }) + }, { timeoutMs: 2000 }).then(done, done) + }) + }) + + it('Should emit DSM stats to the agent when sending a message', done => { + agent.expectPipelineStats(dsmStats => { + let statsPointsReceived = 0 + // we should have 1 dsm stats points + dsmStats.forEach((timeStatsBucket) => { + if (timeStatsBucket && timeStatsBucket.Stats) { + timeStatsBucket.Stats.forEach((statsBuckets) => { + statsPointsReceived += statsBuckets.Stats.length + }) + } + }, { timeoutMs: 2000 }) + expect(statsPointsReceived).to.be.at.least(1) + expect(agent.dsmStatsExist(agent, expectedProducerHash)).to.equal(true) + }).then(done, done) + + context.sender.send({ body: 'hello from DSM' }) + }) + + it('Should emit DSM stats to the agent when receiving a message', done => { + agent.expectPipelineStats(dsmStats => { + let statsPointsReceived = 0 + // we should have 2 dsm stats points + dsmStats.forEach((timeStatsBucket) => { + if (timeStatsBucket && timeStatsBucket.Stats) { + timeStatsBucket.Stats.forEach((statsBuckets) => { + statsPointsReceived += statsBuckets.Stats.length + }) + } + }) + expect(statsPointsReceived).to.be.at.least(2) + expect(agent.dsmStatsExist(agent, expectedConsumerHash)).to.equal(true) + }, { timeoutMs: 2000 }).then(done, done) + + context.sender.send({ body: 'hello from DSM' }) + container.once('message', msg => { + msg.delivery.accept() + }) + }) + describe('sending a message', () => { withPeerService( () => tracer, @@ -71,7 +152,7 @@ describe('Plugin', () => { 'amqp.link.role': 'sender', 'amqp.delivery.state': 'accepted', 'out.host': 'localhost', - 'component': 'rhea' + component: 'rhea' }) expect(span.metrics).to.include({ 'network.destination.port': 5673 @@ -125,7 +206,7 @@ describe('Plugin', () => { 'span.kind': 'consumer', 'amqp.link.source.address': 'amq.topic', 'amqp.link.role': 'receiver', - 'component': 'rhea' + component: 'rhea' }) }) .then(done, done) @@ -206,6 +287,51 @@ describe('Plugin', () => { }) }) + describe('connection cleanup', () => { + let container + let context + let spy + let rheaInstumentation + + beforeEach(() => agent.reload('rhea')) + + beforeEach(done => { + rheaInstumentation = require('../../datadog-instrumentations/src/rhea') + spy = sinon.spy(rheaInstumentation, 'beforeFinish') + container = require(`../../../versions/rhea@${version}`).get() + + container.once('sendable', _context => { + context = _context + done() + }) + const connection = container.connect({ + username: 'admin', + password: 'admin', + host: 'localhost', + port: 5673 + }) + connection.open_sender('amq.topic') + connection.open_receiver('amq.topic') + }) + + it('should automatically instrument', (done) => { + agent.use(traces => { + const beforeFinishContext = rheaInstumentation.contexts.get(spy.firstCall.firstArg) + expect(spy).to.have.been.called + expect(beforeFinishContext).to.have.property('connection') + expect(beforeFinishContext.connection).to.have.property(rheaInstumentation.inFlightDeliveries) + expect(beforeFinishContext.connection[rheaInstumentation.inFlightDeliveries]).to.be.instanceof(Set) + expect(beforeFinishContext.connection[rheaInstumentation.inFlightDeliveries].size).to.equal(0) + }) + .then(done, done) + context.sender.send({ body: 'Hello World!' }) + }) + + afterEach(() => { + spy.restore() + }) + }) + describe('without broker', () => { let server let serverContext @@ -317,7 +443,7 @@ describe('Plugin', () => { [ERROR_MESSAGE]: 'this is an error', [ERROR_TYPE]: 'Error', [ERROR_STACK]: error.stack, - 'component': 'rhea' + component: 'rhea' }) Session.prototype.on_transfer = onTransfer }).then(done, done) @@ -525,7 +651,7 @@ describe('Plugin', () => { [ERROR_TYPE]: 'Error', [ERROR_MESSAGE]: 'fake protocol error', [ERROR_STACK]: err.stack, - 'component': 'rhea' + component: 'rhea' }) expect(span.metrics).to.include({ 'network.destination.port': expectedServerPort @@ -557,7 +683,7 @@ describe('Plugin', () => { [ERROR_TYPE]: 'Error', [ERROR_MESSAGE]: 'fake protocol error', [ERROR_STACK]: err.stack, - 'component': 'rhea' + component: 'rhea' }) }).then(done, done) client.on('message', msg => { @@ -590,7 +716,7 @@ function expectReceiving (agent, expectedSchema, deliveryState, topic) { 'span.kind': 'consumer', 'amqp.link.source.address': topic, 'amqp.link.role': 'receiver', - 'component': 'rhea' + component: 'rhea' } if (deliveryState) { expectedMeta['amqp.delivery.state'] = deliveryState @@ -615,7 +741,7 @@ function expectSending (agent, expectedSchema, deliveryState, topic) { 'span.kind': 'producer', 'amqp.link.target.address': topic, 'amqp.link.role': 'sender', - 'component': 'rhea' + component: 'rhea' } if (deliveryState) { expectedMeta['amqp.delivery.state'] = deliveryState diff --git a/packages/datadog-plugin-rhea/test/integration-test/client.spec.js b/packages/datadog-plugin-rhea/test/integration-test/client.spec.js index bf3485b45f8..cbcae01ecba 100644 --- a/packages/datadog-plugin-rhea/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-rhea/test/integration-test/client.spec.js @@ -17,7 +17,7 @@ describe('esm', () => { before(async function () { this.timeout(20000) sandbox = await createSandbox([`'rhea@${version}'`], false, [ - `./packages/datadog-plugin-rhea/test/integration-test/*`]) + './packages/datadog-plugin-rhea/test/integration-test/*']) }) after(async () => { diff --git a/packages/datadog-plugin-rhea/test/naming.js b/packages/datadog-plugin-rhea/test/naming.js index ea505cec556..f4e4508e8a7 100644 --- a/packages/datadog-plugin-rhea/test/naming.js +++ b/packages/datadog-plugin-rhea/test/naming.js @@ -24,6 +24,6 @@ const rawExpectedSchema = { } module.exports = { - rawExpectedSchema: rawExpectedSchema, + rawExpectedSchema, expectedSchema: resolveNaming(rawExpectedSchema) } diff --git a/packages/datadog-plugin-router/src/index.js b/packages/datadog-plugin-router/src/index.js index 682a4301580..439f2d08332 100644 --- a/packages/datadog-plugin-router/src/index.js +++ b/packages/datadog-plugin-router/src/index.js @@ -71,7 +71,7 @@ class RouterPlugin extends WebPlugin { span.setTag('error', error) }) - this.addSub(`apm:http:server:request:finish`, ({ req }) => { + this.addSub('apm:http:server:request:finish', ({ req }) => { const context = this._contexts.get(req) if (!context) return diff --git a/packages/datadog-plugin-router/test/index.spec.js b/packages/datadog-plugin-router/test/index.spec.js index 894dacc3883..ac208f0e2a1 100644 --- a/packages/datadog-plugin-router/test/index.spec.js +++ b/packages/datadog-plugin-router/test/index.spec.js @@ -5,7 +5,6 @@ const axios = require('axios') const http = require('http') const { once } = require('events') -const getPort = require('get-port') const agent = require('../../dd-trace/test/plugins/agent') const web = require('../../dd-trace/src/plugins/util/web') @@ -87,7 +86,9 @@ describe('Plugin', () => { router.use('/parent', childRouter) - getPort().then(port => { + appListener = server(router).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -97,11 +98,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(router).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/parent/child/123`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/parent/child/123`) + .catch(done) }) }) @@ -115,15 +114,15 @@ describe('Plugin', () => { res.end() }) - const port = await getPort() const agentPromise = agent.use(traces => { for (const span of traces[0]) { expect(span.error).to.equal(0) } }, { rejectFirst: true }) - const httpd = server(router).listen(port, 'localhost') + const httpd = server(router).listen(0, 'localhost') await once(httpd, 'listening') + const port = httpd.address().port const reqPromise = axios.get(`http://localhost:${port}/foo`) return Promise.all([agentPromise, reqPromise]) @@ -139,15 +138,16 @@ describe('Plugin', () => { res.end() }) - const port = await getPort() const agentPromise = agent.use(traces => { for (const span of traces[0]) { expect(span.error).to.equal(0) } }, { rejectFirst: true }) - const httpd = server(router, (req, res) => err => res.end()).listen(port, 'localhost') + // eslint-disable-next-line n/handle-callback-err + const httpd = server(router, (req, res) => err => res.end()).listen(0, 'localhost') await once(httpd, 'listening') + const port = httpd.address().port const reqPromise = axios.get(`http://localhost:${port}/foo`) return Promise.all([agentPromise, reqPromise]) diff --git a/packages/datadog-plugin-router/test/integration-test/client.spec.js b/packages/datadog-plugin-router/test/integration-test/client.spec.js index 7982db2f7c1..3b32836e64c 100644 --- a/packages/datadog-plugin-router/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-router/test/integration-test/client.spec.js @@ -18,7 +18,7 @@ describe('esm', () => { before(async function () { this.timeout(20000) sandbox = await createSandbox([`'router@${version}'`] - , false, [`./packages/datadog-plugin-router/test/integration-test/*`]) + , false, ['./packages/datadog-plugin-router/test/integration-test/*']) }) after(async () => { diff --git a/packages/datadog-plugin-selenium/src/index.js b/packages/datadog-plugin-selenium/src/index.js new file mode 100644 index 00000000000..2ff542e9e73 --- /dev/null +++ b/packages/datadog-plugin-selenium/src/index.js @@ -0,0 +1,71 @@ +const CiPlugin = require('../../dd-trace/src/plugins/ci_plugin') +const { storage } = require('../../datadog-core') + +const { + TEST_IS_RUM_ACTIVE, + TEST_BROWSER_DRIVER, + TEST_BROWSER_DRIVER_VERSION, + TEST_BROWSER_NAME, + TEST_BROWSER_VERSION, + TEST_TYPE +} = require('../../dd-trace/src/plugins/util/test') +const { SPAN_TYPE } = require('../../../ext/tags') + +function isTestSpan (span) { + return span.context()._tags[SPAN_TYPE] === 'test' +} + +function getTestSpanFromTrace (trace) { + for (const span of trace.started) { + if (isTestSpan(span)) { + return span + } + } + return null +} + +class SeleniumPlugin extends CiPlugin { + static get id () { + return 'selenium' + } + + constructor (...args) { + super(...args) + + this.addSub('ci:selenium:driver:get', ({ + setTraceId, + seleniumVersion, + browserName, + browserVersion, + isRumActive + }) => { + const store = storage.getStore() + const span = store?.span + if (!span) { + return + } + let testSpan + if (isTestSpan(span)) { + testSpan = span + } else { + testSpan = getTestSpanFromTrace(span.context()._trace) + } + if (!testSpan) { + return + } + if (setTraceId) { + setTraceId(testSpan.context().toTraceId()) + } + if (isRumActive) { + testSpan.setTag(TEST_IS_RUM_ACTIVE, 'true') + } + testSpan.setTag(TEST_BROWSER_DRIVER, 'selenium') + testSpan.setTag(TEST_BROWSER_DRIVER_VERSION, seleniumVersion) + testSpan.setTag(TEST_BROWSER_NAME, browserName) + testSpan.setTag(TEST_BROWSER_VERSION, browserVersion) + testSpan.setTag(TEST_TYPE, 'browser') + }) + } +} + +module.exports = SeleniumPlugin diff --git a/packages/datadog-plugin-sharedb/src/index.js b/packages/datadog-plugin-sharedb/src/index.js index 1257d608d56..d0c2c1bf819 100644 --- a/packages/datadog-plugin-sharedb/src/index.js +++ b/packages/datadog-plugin-sharedb/src/index.js @@ -54,7 +54,7 @@ function sanitize (input) { } function isObject (val) { - return typeof val === 'object' && val !== null && !(val instanceof Array) + return val !== null && typeof val === 'object' && !Array.isArray(val) } module.exports = SharedbPlugin diff --git a/packages/datadog-plugin-sharedb/test/integration-test/client.spec.js b/packages/datadog-plugin-sharedb/test/integration-test/client.spec.js index 04cfd2b3f10..9d2aa161ca8 100644 --- a/packages/datadog-plugin-sharedb/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-sharedb/test/integration-test/client.spec.js @@ -18,7 +18,7 @@ describe('esm', () => { before(async function () { this.timeout(20000) sandbox = await createSandbox([`'sharedb@${version}'`], false, [ - `./packages/datadog-plugin-sharedb/test/integration-test/*`]) + './packages/datadog-plugin-sharedb/test/integration-test/*']) }) after(async () => { diff --git a/packages/datadog-plugin-tedious/src/index.js b/packages/datadog-plugin-tedious/src/index.js index 56df5e453ba..97eab3617dc 100644 --- a/packages/datadog-plugin-tedious/src/index.js +++ b/packages/datadog-plugin-tedious/src/index.js @@ -16,7 +16,7 @@ class TediousPlugin extends DatabasePlugin { kind: 'client', meta: { 'db.type': 'mssql', - 'component': 'tedious', + component: 'tedious', 'out.host': connectionConfig.server, [CLIENT_PORT_KEY]: connectionConfig.options.port, 'db.user': connectionConfig.userName || connectionConfig.authentication.options.userName, diff --git a/packages/datadog-plugin-tedious/test/integration-test/client.spec.js b/packages/datadog-plugin-tedious/test/integration-test/client.spec.js index 271135800c3..aa32944f541 100644 --- a/packages/datadog-plugin-tedious/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-tedious/test/integration-test/client.spec.js @@ -24,7 +24,7 @@ describe('esm', () => { before(async function () { this.timeout(20000) sandbox = await createSandbox([`'tedious@${version}'`], false, [ - `./packages/datadog-plugin-tedious/test/integration-test/*`]) + './packages/datadog-plugin-tedious/test/integration-test/*']) }) after(async () => { diff --git a/packages/datadog-plugin-tedious/test/naming.js b/packages/datadog-plugin-tedious/test/naming.js index 3473f6e86c9..da01aad5a8f 100644 --- a/packages/datadog-plugin-tedious/test/naming.js +++ b/packages/datadog-plugin-tedious/test/naming.js @@ -14,6 +14,6 @@ const rawExpectedSchema = { } module.exports = { - rawExpectedSchema: rawExpectedSchema, + rawExpectedSchema, expectedSchema: resolveNaming(rawExpectedSchema) } diff --git a/packages/datadog-plugin-undici/src/index.js b/packages/datadog-plugin-undici/src/index.js new file mode 100644 index 00000000000..c436aceb882 --- /dev/null +++ b/packages/datadog-plugin-undici/src/index.js @@ -0,0 +1,12 @@ +'use strict' + +const FetchPlugin = require('../../datadog-plugin-fetch/src/index.js') + +class UndiciPlugin extends FetchPlugin { + static get id () { return 'undici' } + static get prefix () { + return 'tracing:apm:undici:fetch' + } +} + +module.exports = UndiciPlugin diff --git a/packages/datadog-plugin-undici/test/index.spec.js b/packages/datadog-plugin-undici/test/index.spec.js new file mode 100644 index 00000000000..03541e0eb7c --- /dev/null +++ b/packages/datadog-plugin-undici/test/index.spec.js @@ -0,0 +1,498 @@ +'use strict' + +const semver = require('semver') + +const agent = require('../../dd-trace/test/plugins/agent') +const tags = require('../../../ext/tags') +const { expect } = require('chai') +const { rawExpectedSchema } = require('./naming') +const { DD_MAJOR } = require('../../../version') +const { ERROR_MESSAGE, ERROR_TYPE, ERROR_STACK } = require('../../dd-trace/src/constants') +const { NODE_MAJOR } = require('../../../version') + +const HTTP_REQUEST_HEADERS = tags.HTTP_REQUEST_HEADERS +const HTTP_RESPONSE_HEADERS = tags.HTTP_RESPONSE_HEADERS + +const SERVICE_NAME = DD_MAJOR < 3 ? 'test-http-client' : 'test' + +describe('Plugin', () => { + let express + let fetch + let appListener + + describe('undici-fetch', () => { + withVersions('undici', 'undici', version => { + const specificVersion = require(`../../../versions/undici@${version}`).version() + if (NODE_MAJOR <= 16 && semver.satisfies(specificVersion, '>=6')) return + + function server (app, listener) { + const server = require('http').createServer(app) + server.listen(0, 'localhost', () => listener(server.address().port)) + return server + } + + beforeEach(() => { + appListener = null + }) + + afterEach(() => { + if (appListener) { + appListener.close() + } + return agent.close({ ritmReset: false }) + }) + + describe('without configuration', () => { + beforeEach(() => { + return agent.load('undici', { + service: 'test' + }) + .then(() => { + express = require('express') + fetch = require(`../../../versions/undici@${version}`, {}).get() + }) + }) + + afterEach(() => { + express = null + }) + + withNamingSchema( + () => { + const app = express() + app.get('/user', (req, res) => { + res.status(200).send() + }) + + appListener = server(app, port => { + fetch.fetch(`http://localhost:${port}/user`, { method: 'GET' }) + }) + }, + rawExpectedSchema.client + ) + + it('should do automatic instrumentation', function (done) { + const app = express() + app.get('/user', (req, res) => { + res.status(200).send() + }) + appListener = server(app, port => { + agent + .use(traces => { + expect(traces[0][0]).to.have.property('service', 'test') + expect(traces[0][0]).to.have.property('type', 'http') + expect(traces[0][0]).to.have.property('resource', 'GET') + expect(traces[0][0].meta).to.have.property('span.kind', 'client') + expect(traces[0][0].meta).to.have.property('http.url', `http://localhost:${port}/user`) + expect(traces[0][0].meta).to.have.property('http.method', 'GET') + expect(traces[0][0].meta).to.have.property('http.status_code', '200') + expect(traces[0][0].meta).to.have.property('component', 'undici') + expect(traces[0][0].meta).to.have.property('out.host', 'localhost') + }) + .then(done) + .catch(done) + + fetch.fetch(`http://localhost:${port}/user`, { method: 'GET' }) + }) + }) + + it('should support URL input', done => { + const app = express() + app.post('/user', (req, res) => { + res.status(200).send() + }) + appListener = server(app, port => { + agent + .use(traces => { + expect(traces[0][0]).to.have.property('service', SERVICE_NAME) + expect(traces[0][0]).to.have.property('type', 'http') + expect(traces[0][0]).to.have.property('resource', 'POST') + expect(traces[0][0].meta).to.have.property('span.kind', 'client') + expect(traces[0][0].meta).to.have.property('http.url', `http://localhost:${port}/user`) + expect(traces[0][0].meta).to.have.property('http.method', 'POST') + expect(traces[0][0].meta).to.have.property('http.status_code', '200') + expect(traces[0][0].meta).to.have.property('component', 'undici') + expect(traces[0][0].meta).to.have.property('out.host', 'localhost') + }) + .then(done) + .catch(done) + + fetch.fetch(new URL(`http://localhost:${port}/user`), { method: 'POST' }) + }) + }) + + it('should return the response', done => { + const app = express() + app.get('/user', (req, res) => { + res.status(200).send() + }) + appListener = server(app, port => { + fetch.fetch((`http://localhost:${port}/user`)) + .then(res => { + expect(res).to.have.property('status', 200) + done() + }) + .catch(done) + }) + }) + + it('should remove the query string from the URL', done => { + const app = express() + + app.get('/user', (req, res) => { + res.status(200).send() + }) + + appListener = server(app, port => { + agent + .use(traces => { + expect(traces[0][0].meta).to.have.property('http.status_code', '200') + expect(traces[0][0].meta).to.have.property('http.url', `http://localhost:${port}/user`) + }) + .then(done) + .catch(done) + + fetch.fetch(`http://localhost:${port}/user?foo=bar`) + }) + }) + + it('should inject its parent span in the headers', done => { + const app = express() + + app.get('/user', (req, res) => { + expect(req.get('x-datadog-trace-id')).to.be.a('string') + expect(req.get('x-datadog-parent-id')).to.be.a('string') + + res.status(200).send() + }) + + appListener = server(app, port => { + agent + .use(traces => { + expect(traces[0][0].meta).to.have.property('http.status_code', '200') + }) + .then(done) + .catch(done) + + fetch.fetch(`http://localhost:${port}/user?foo=bar`) + }) + }) + + it('should inject its parent span in the existing headers', done => { + const app = express() + + app.get('/user', (req, res) => { + expect(req.get('foo')).to.be.a('string') + expect(req.get('x-datadog-trace-id')).to.be.a('string') + expect(req.get('x-datadog-parent-id')).to.be.a('string') + + res.status(200).send() + }) + + appListener = server(app, port => { + agent + .use(traces => { + expect(traces[0][0].meta).to.have.property('http.status_code', '200') + }) + .then(done) + .catch(done) + + fetch.fetch(`http://localhost:${port}/user?foo=bar`, { headers: { foo: 'bar' } }) + }) + }) + + it('should handle connection errors', done => { + let error + + agent + .use(traces => { + expect(traces[0][0].meta).to.have.property(ERROR_TYPE, error.name) + expect(traces[0][0].meta).to.have.property(ERROR_MESSAGE, error.message || error.code) + expect(traces[0][0].meta).to.have.property(ERROR_STACK, error.stack) + expect(traces[0][0].meta).to.have.property('component', 'undici') + }) + .then(done) + .catch(done) + + fetch.fetch('http://localhost:7357/user').catch(err => { + error = err + }) + }) + + it('should not record HTTP 5XX responses as errors by default', done => { + const app = express() + + app.get('/user', (req, res) => { + res.status(500).send() + }) + + appListener = server(app, port => { + agent + .use(traces => { + expect(traces[0][0]).to.have.property('error', 0) + }) + .then(done) + .catch(done) + + fetch.fetch(`http://localhost:${port}/user`) + }) + }) + + it('should record HTTP 4XX responses as errors by default', done => { + const app = express() + + app.get('/user', (req, res) => { + res.status(400).send() + }) + + appListener = server(app, port => { + agent + .use(traces => { + expect(traces[0][0]).to.have.property('error', 1) + }) + .then(done) + .catch(done) + + fetch.fetch(`http://localhost:${port}/user`) + }) + }) + + it('should not record aborted requests as errors', done => { + const app = express() + + app.get('/user', (req, res) => {}) + + appListener = server(app, port => { + agent + .use(traces => { + expect(traces[0][0]).to.have.property('error', 0) + expect(traces[0][0].meta).to.not.have.property('http.status_code') + }) + .then(done) + .catch(done) + + const controller = new AbortController() + + fetch.fetch(`http://localhost:${port}/user`, { + signal: controller.signal + }).catch(() => {}) + + controller.abort() + }) + }) + + it('should record when the request was aborted', done => { + const app = express() + + app.get('/abort', (req, res) => { + res.status(200).send() + }) + + appListener = server(app, port => { + agent + .use(traces => { + expect(traces[0][0]).to.have.property('service', SERVICE_NAME) + }) + .then(done) + .catch(done) + + const controller = new AbortController() + + fetch.fetch(`http://localhost:${port}/user`, { + signal: controller.signal + }).catch(() => {}) + + controller.abort() + }) + }) + }) + describe('with service configuration', () => { + let config + + beforeEach(() => { + config = { + service: 'custom' + } + + return agent.load('undici', config) + .then(() => { + express = require('express') + fetch = require(`../../../versions/undici@${version}`, {}).get() + }) + }) + + it('should be configured with the correct values', done => { + const app = express() + + app.get('/user', (req, res) => { + res.status(200).send() + }) + + appListener = server(app, port => { + agent + .use(traces => { + expect(traces[0][0]).to.have.property('service', 'custom') + }) + .then(done) + .catch(done) + + fetch.fetch(`http://localhost:${port}/user`).catch(() => {}) + }) + }) + }) + describe('with headers configuration', () => { + let config + + beforeEach(() => { + config = { + headers: ['x-baz', 'x-foo'] + } + + return agent.load('undici', config) + .then(() => { + express = require('express') + fetch = require(`../../../versions/undici@${version}`, {}).get() + }) + }) + + it('should add tags for the configured headers', done => { + const app = express() + + app.get('/user', (req, res) => { + res.setHeader('x-foo', 'bar') + res.status(200).send() + }) + + appListener = server(app, port => { + agent + .use(traces => { + const meta = traces[0][0].meta + expect(meta).to.have.property(`${HTTP_REQUEST_HEADERS}.x-baz`, 'qux') + expect(meta).to.have.property(`${HTTP_RESPONSE_HEADERS}.x-foo`, 'bar') + }) + .then(done) + .catch(done) + + fetch.fetch(`http://localhost:${port}/user`, { + headers: { + 'x-baz': 'qux' + } + }).catch(() => {}) + }) + }) + }) + describe('with hooks configuration', () => { + let config + + beforeEach(() => { + config = { + hooks: { + request: (span, req, res) => { + span.setTag('foo', '/foo') + } + } + } + + return agent.load('undici', config) + .then(() => { + express = require('express') + fetch = require(`../../../versions/undici@${version}`, {}).get() + }) + }) + + it('should run the request hook before the span is finished', done => { + const app = express() + + app.get('/user', (req, res) => { + res.status(200).send() + }) + + appListener = server(app, port => { + agent + .use(traces => { + expect(traces[0][0].meta).to.have.property('foo', '/foo') + }) + .then(done) + .catch(done) + + fetch.fetch(`http://localhost:${port}/user`).catch(() => {}) + }) + }) + }) + + describe('with propagationBlocklist configuration', () => { + let config + + beforeEach(() => { + config = { + propagationBlocklist: [/\/users/] + } + + return agent.load('undici', config) + .then(() => { + express = require('express') + fetch = require(`../../../versions/undici@${version}`, {}).get() + }) + }) + + it('should skip injecting if the url matches an item in the propagationBlacklist', done => { + const app = express() + + app.get('/users', (req, res) => { + try { + expect(req.get('x-datadog-trace-id')).to.be.undefined + expect(req.get('x-datadog-parent-id')).to.be.undefined + + res.status(200).send() + + done() + } catch (e) { + done(e) + } + }) + + appListener = server(app, port => { + fetch.fetch(`http://localhost:${port}/users`).catch(() => {}) + }) + }) + }) + + describe('with blocklist configuration', () => { + let config + + beforeEach(() => { + config = { + blocklist: [/\/user/] + } + + return agent.load('undici', config) + .then(() => { + express = require('express') + fetch = require(`../../../versions/undici@${version}`, {}).get() + }) + }) + + it('should skip recording if the url matches an item in the blocklist', done => { + const app = express() + + app.get('/user', (req, res) => { + res.status(200).send() + }) + + appListener = server(app, port => { + const timer = setTimeout(done, 100) + + agent + .use(() => { + clearTimeout(timer) + done(new Error('Blocklisted requests should not be recorded.')) + }) + .catch(done) + + fetch.fetch(`http://localhost:${port}/users`).catch(() => {}) + }) + }) + }) + }) + }) +}) diff --git a/packages/datadog-plugin-undici/test/naming.js b/packages/datadog-plugin-undici/test/naming.js new file mode 100644 index 00000000000..5bf2be387c3 --- /dev/null +++ b/packages/datadog-plugin-undici/test/naming.js @@ -0,0 +1,19 @@ +const { resolveNaming } = require('../../dd-trace/test/plugins/helpers') + +const rawExpectedSchema = { + client: { + v0: { + serviceName: 'test', + opName: 'undici.request' + }, + v1: { + serviceName: 'test', + opName: 'undici.request' + } + } +} + +module.exports = { + rawExpectedSchema, + expectedSchema: resolveNaming(rawExpectedSchema) +} diff --git a/packages/datadog-plugin-vitest/src/index.js b/packages/datadog-plugin-vitest/src/index.js new file mode 100644 index 00000000000..34617bdb1ac --- /dev/null +++ b/packages/datadog-plugin-vitest/src/index.js @@ -0,0 +1,259 @@ +const CiPlugin = require('../../dd-trace/src/plugins/ci_plugin') +const { storage } = require('../../datadog-core') + +const { + TEST_STATUS, + finishAllTraceSpans, + getTestSuitePath, + getTestSuiteCommonTags, + getTestSessionName, + getIsFaultyEarlyFlakeDetection, + TEST_SOURCE_FILE, + TEST_IS_RETRY, + TEST_CODE_COVERAGE_LINES_PCT, + TEST_CODE_OWNERS, + TEST_LEVEL_EVENT_TYPES, + TEST_SESSION_NAME, + TEST_SOURCE_START, + TEST_IS_NEW, + TEST_EARLY_FLAKE_ENABLED, + TEST_EARLY_FLAKE_ABORT_REASON +} = require('../../dd-trace/src/plugins/util/test') +const { COMPONENT } = require('../../dd-trace/src/constants') +const { + TELEMETRY_EVENT_CREATED, + TELEMETRY_EVENT_FINISHED, + TELEMETRY_TEST_SESSION +} = require('../../dd-trace/src/ci-visibility/telemetry') + +// Milliseconds that we subtract from the error test duration +// so that they do not overlap with the following test +// This is because there's some loss of resolution. +const MILLISECONDS_TO_SUBTRACT_FROM_FAILED_TEST_DURATION = 5 + +class VitestPlugin extends CiPlugin { + static get id () { + return 'vitest' + } + + constructor (...args) { + super(...args) + + this.taskToFinishTime = new WeakMap() + + this.addSub('ci:vitest:test:is-new', ({ knownTests, testSuiteAbsolutePath, testName, onDone }) => { + const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) + const testsForThisTestSuite = knownTests[testSuite] || [] + onDone(!testsForThisTestSuite.includes(testName)) + }) + + this.addSub('ci:vitest:is-early-flake-detection-faulty', ({ + knownTests, + testFilepaths, + onDone + }) => { + const isFaulty = getIsFaultyEarlyFlakeDetection( + testFilepaths.map(testFilepath => getTestSuitePath(testFilepath, this.repositoryRoot)), + knownTests, + this.libraryConfig.earlyFlakeDetectionFaultyThreshold + ) + onDone(isFaulty) + }) + + this.addSub('ci:vitest:test:start', ({ testName, testSuiteAbsolutePath, isRetry, isNew }) => { + const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) + const store = storage.getStore() + + const extraTags = { + [TEST_SOURCE_FILE]: testSuite + } + if (isRetry) { + extraTags[TEST_IS_RETRY] = 'true' + } + if (isNew) { + extraTags[TEST_IS_NEW] = 'true' + } + + const span = this.startTestSpan( + testName, + testSuite, + this.testSuiteSpan, + extraTags + ) + + this.enter(span, store) + }) + + this.addSub('ci:vitest:test:finish-time', ({ status, task }) => { + const store = storage.getStore() + const span = store?.span + + // we store the finish time to finish at a later hook + // this is because the test might fail at a `afterEach` hook + if (span) { + span.setTag(TEST_STATUS, status) + this.taskToFinishTime.set(task, span._getTime()) + } + }) + + this.addSub('ci:vitest:test:pass', ({ task }) => { + const store = storage.getStore() + const span = store?.span + + if (span) { + this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'test', { + hasCodeowners: !!span.context()._tags[TEST_CODE_OWNERS] + }) + span.setTag(TEST_STATUS, 'pass') + span.finish(this.taskToFinishTime.get(task)) + finishAllTraceSpans(span) + } + }) + + this.addSub('ci:vitest:test:error', ({ duration, error }) => { + const store = storage.getStore() + const span = store?.span + + if (span) { + this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'test', { + hasCodeowners: !!span.context()._tags[TEST_CODE_OWNERS] + }) + span.setTag(TEST_STATUS, 'fail') + + if (error) { + span.setTag('error', error) + } + if (duration) { + span.finish(span._startTime + duration - MILLISECONDS_TO_SUBTRACT_FROM_FAILED_TEST_DURATION) // milliseconds + } else { + span.finish() // retries will not have a duration + } + finishAllTraceSpans(span) + } + }) + + this.addSub('ci:vitest:test:skip', ({ testName, testSuiteAbsolutePath }) => { + const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) + const testSpan = this.startTestSpan( + testName, + testSuite, + this.testSuiteSpan, + { + [TEST_SOURCE_FILE]: testSuite, + [TEST_SOURCE_START]: 1, // we can't get the proper start line in vitest + [TEST_STATUS]: 'skip' + } + ) + this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'test', { + hasCodeowners: !!testSpan.context()._tags[TEST_CODE_OWNERS] + }) + testSpan.finish() + }) + + this.addSub('ci:vitest:test-suite:start', ({ testSuiteAbsolutePath, frameworkVersion }) => { + this.command = process.env.DD_CIVISIBILITY_TEST_COMMAND + this.frameworkVersion = frameworkVersion + const testSessionSpanContext = this.tracer.extract('text_map', { + 'x-datadog-trace-id': process.env.DD_CIVISIBILITY_TEST_SESSION_ID, + 'x-datadog-parent-id': process.env.DD_CIVISIBILITY_TEST_MODULE_ID + }) + + // test suites run in a different process, so they also need to init the metadata dictionary + const testSessionName = getTestSessionName(this.config, this.command, this.testEnvironmentMetadata) + const metadataTags = {} + for (const testLevel of TEST_LEVEL_EVENT_TYPES) { + metadataTags[testLevel] = { + [TEST_SESSION_NAME]: testSessionName + } + } + if (this.tracer._exporter.setMetadataTags) { + this.tracer._exporter.setMetadataTags(metadataTags) + } + + const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) + const testSuiteMetadata = getTestSuiteCommonTags( + this.command, + this.frameworkVersion, + testSuite, + 'vitest' + ) + testSuiteMetadata[TEST_SOURCE_FILE] = testSuite + testSuiteMetadata[TEST_SOURCE_START] = 1 + + const codeOwners = this.getCodeOwners(testSuiteMetadata) + if (codeOwners) { + testSuiteMetadata[TEST_CODE_OWNERS] = codeOwners + } + + const testSuiteSpan = this.tracer.startSpan('vitest.test_suite', { + childOf: testSessionSpanContext, + tags: { + [COMPONENT]: this.constructor.id, + ...this.testEnvironmentMetadata, + ...testSuiteMetadata + } + }) + this.telemetry.ciVisEvent(TELEMETRY_EVENT_CREATED, 'suite') + const store = storage.getStore() + this.enter(testSuiteSpan, store) + this.testSuiteSpan = testSuiteSpan + }) + + this.addSub('ci:vitest:test-suite:finish', ({ status, onFinish }) => { + const store = storage.getStore() + const span = store?.span + if (span) { + span.setTag(TEST_STATUS, status) + span.finish() + finishAllTraceSpans(span) + } + this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'suite') + // TODO: too frequent flush - find for method in worker to decrease frequency + this.tracer._exporter.flush(onFinish) + }) + + this.addSub('ci:vitest:test-suite:error', ({ error }) => { + const store = storage.getStore() + const span = store?.span + if (span && error) { + span.setTag('error', error) + span.setTag(TEST_STATUS, 'fail') + } + }) + + this.addSub('ci:vitest:session:finish', ({ + status, + error, + testCodeCoverageLinesTotal, + isEarlyFlakeDetectionEnabled, + isEarlyFlakeDetectionFaulty, + onFinish + }) => { + this.testSessionSpan.setTag(TEST_STATUS, status) + this.testModuleSpan.setTag(TEST_STATUS, status) + if (error) { + this.testModuleSpan.setTag('error', error) + this.testSessionSpan.setTag('error', error) + } + if (testCodeCoverageLinesTotal) { + this.testModuleSpan.setTag(TEST_CODE_COVERAGE_LINES_PCT, testCodeCoverageLinesTotal) + this.testSessionSpan.setTag(TEST_CODE_COVERAGE_LINES_PCT, testCodeCoverageLinesTotal) + } + if (isEarlyFlakeDetectionEnabled) { + this.testSessionSpan.setTag(TEST_EARLY_FLAKE_ENABLED, 'true') + } + if (isEarlyFlakeDetectionFaulty) { + this.testSessionSpan.setTag(TEST_EARLY_FLAKE_ABORT_REASON, 'faulty') + } + this.testModuleSpan.finish() + this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'module') + this.testSessionSpan.finish() + this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'session') + finishAllTraceSpans(this.testSessionSpan) + this.telemetry.count(TELEMETRY_TEST_SESSION, { provider: this.ciProviderName }) + this.tracer._exporter.flush(onFinish) + }) + } +} + +module.exports = VitestPlugin diff --git a/packages/datadog-plugin-winston/test/integration-test/client.spec.js b/packages/datadog-plugin-winston/test/integration-test/client.spec.js index 505280d836d..cb94108145a 100644 --- a/packages/datadog-plugin-winston/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-winston/test/integration-test/client.spec.js @@ -17,7 +17,7 @@ describe('esm', () => { before(async function () { this.timeout(50000) sandbox = await createSandbox([`'winston@${version}'`] - , false, [`./packages/datadog-plugin-winston/test/integration-test/*`]) + , false, ['./packages/datadog-plugin-winston/test/integration-test/*']) }) after(async function () { diff --git a/packages/datadog-shimmer/src/shimmer.js b/packages/datadog-shimmer/src/shimmer.js index c876d018782..d12c4c130ef 100644 --- a/packages/datadog-shimmer/src/shimmer.js +++ b/packages/datadog-shimmer/src/shimmer.js @@ -1,5 +1,7 @@ 'use strict' +const log = require('../../dd-trace/src/log') + // Use a weak map to avoid polluting the wrapped function/method. const unwrappers = new WeakMap() @@ -18,9 +20,12 @@ function copyProperties (original, wrapped) { } } -function wrapFn (original, delegate) { - assertFunction(delegate) - assertNotClass(original) // TODO: support constructors of native classes +function wrapFunction (original, wrapper) { + if (typeof original === 'function') assertNotClass(original) + // TODO This needs to be re-done so that this and wrapMethod are distinct. + const target = { func: original } + wrapMethod(target, 'func', wrapper, typeof original !== 'function') + let delegate = target.func const shim = function shim () { return delegate.apply(this, arguments) @@ -30,17 +35,144 @@ function wrapFn (original, delegate) { delegate = original }) - copyProperties(original, shim) + if (typeof original === 'function') copyProperties(original, shim) return shim } -function wrapMethod (target, name, wrapper) { - assertMethod(target, name) - assertFunction(wrapper) +const wrapFn = function (original, delegate) { + throw new Error('calling `wrap()` with 2 args is deprecated. Use wrapFunction instead.') +} + +// This is only used in safe mode. It's a simple state machine to track if the +// original method was called and if it returned. We need this to determine if +// an error was thrown by the original method, or by us. We'll use one of these +// per call to a wrapped method. +class CallState { + constructor () { + this.called = false + this.completed = false + this.retVal = undefined + } + + startCall () { + this.called = true + } + + endCall (retVal) { + this.completed = true + this.retVal = retVal + } +} + +function isPromise (obj) { + return obj && typeof obj === 'object' && typeof obj.then === 'function' +} + +let safeMode = !!process.env.DD_INEJCTION_ENABLED +function setSafe (value) { + safeMode = value +} + +function wrapMethod (target, name, wrapper, noAssert) { + if (!noAssert) { + assertMethod(target, name) + assertFunction(wrapper) + } const original = target[name] - const wrapped = wrapper(original) + let wrapped + + if (safeMode && original) { + // In this mode, we make a best-effort attempt to handle errors that are thrown + // by us, rather than wrapped code. With such errors, we log them, and then attempt + // to return the result as if no wrapping was done at all. + // + // Caveats: + // * If the original function is called in a later iteration of the event loop, + // and we throw _then_, then it won't be caught by this. In practice, we always call + // the original function synchronously, so this is not a problem. + // * While async errors are dealt with here, errors in callbacks are not. This + // is because we don't necessarily know _for sure_ that any function arguments + // are wrapped by us. We could wrap them all anyway and just make that assumption, + // or just assume that the last argument is always a callback set by us if it's a + // function, but those don't seem like things we can rely on. We could add a + // `shimmer.markCallbackAsWrapped()` function that's a no-op outside safe-mode, + // but that means modifying every instrumentation. Even then, the complexity of + // this code increases because then we'd need to effectively do the reverse of + // what we're doing for synchronous functions. This is a TODO. + + // We're going to hold on to current callState in this variable in this scope, + // which is fine because any time we reference it, we're referencing it synchronously. + // We'll use it in the our wrapper (which, again, is called syncrhonously), and in the + // errorHandler, which will already have been bound to this callState. + let currentCallState + + // Rather than calling the original function directly from the shim wrapper, we wrap + // it again so that we can track if it was called and if it returned. This is because + // we need to know if an error was thrown by the original function, or by us. + // We could do this inside the `wrapper` function defined below, which would simplify + // managing the callState, but then we'd be calling `wrapper` on each invocation, so + // instead we do it here, once. + const innerWrapped = wrapper(function (...args) { + // We need to stash the callState here because of recursion. + const callState = currentCallState + callState.startCall() + const retVal = original.apply(this, args) + if (isPromise(retVal)) { + retVal.then(callState.endCall.bind(callState)) + } else { + callState.endCall(retVal) + } + return retVal + }) + + // This is the crux of what we're doing in safe mode. It handles errors + // that _we_ cause, by logging them, and transparently providing results + // as if no wrapping was done at all. That means detecting (via callState) + // whether the function has already run or not, and if it has, returning + // the result, and otherwise calling the original function unwrapped. + const handleError = function (args, callState, e) { + if (callState.completed) { + // error was thrown after original function returned/resolved, so + // it was us. log it. + log.error(e) + // original ran and returned something. return it. + return callState.retVal + } + + if (!callState.called) { + // error was thrown before original function was called, so + // it was us. log it. + log.error(e) + // original never ran. call it unwrapped. + return original.apply(this, args) + } + + // error was thrown during original function execution, so + // it was them. throw. + throw e + } + + // The wrapped function is the one that will be called by the user. + // It calls our version of the original function, which manages the + // callState. That way when we use the errorHandler, it can tell where + // the error originated. + wrapped = function (...args) { + currentCallState = new CallState() + const errorHandler = handleError.bind(this, args, currentCallState) + + try { + const retVal = innerWrapped.apply(this, args) + return isPromise(retVal) ? retVal.catch(errorHandler) : retVal + } catch (e) { + return errorHandler(e) + } + } + } else { + // In non-safe mode, we just wrap the original function directly. + wrapped = wrapper(original) + } const descriptor = Object.getOwnPropertyDescriptor(target, name) const attributes = { @@ -48,7 +180,7 @@ function wrapMethod (target, name, wrapper) { ...descriptor } - copyProperties(original, wrapped) + if (typeof original === 'function') copyProperties(original, wrapped) if (descriptor) { unwrappers.set(wrapped, () => Object.defineProperty(target, name, descriptor)) @@ -156,7 +288,9 @@ function assertNotClass (target) { module.exports = { wrap, + wrapFunction, massWrap, unwrap, - massUnwrap + massUnwrap, + setSafe } diff --git a/packages/datadog-shimmer/test/shimmer.spec.js b/packages/datadog-shimmer/test/shimmer.spec.js index afe475762c1..40485ba214b 100644 --- a/packages/datadog-shimmer/test/shimmer.spec.js +++ b/packages/datadog-shimmer/test/shimmer.spec.js @@ -227,13 +227,224 @@ describe('shimmer', () => { it('should not throw when unwrapping a method that was not wrapped', () => { expect(() => shimmer.unwrap({ a: () => {} }, 'a')).to.not.throw() }) + + describe('safe mode', () => { + let obj + + before(() => { + shimmer.setSafe(true) + }) + + after(() => { + shimmer.setSafe(false) + }) + + describe('sync', () => { + beforeEach(() => { + obj = { count: () => 3 } + }) + + it('should not throw when wrapper code is throwing', () => { + shimmer.wrap(obj, 'count', () => { + return () => { + throw new Error('wrapper error') + } + }) + + expect(obj.count()).to.equal(3) + }) + + it('should not throw when wrapper code is throwing after return', () => { + shimmer.wrap(obj, 'count', (count) => { + return () => { + count() + throw new Error('wrapper error') + } + }) + + expect(obj.count()).to.equal(3) + }) + }) + + describe('sync recursive', () => { + beforeEach(() => { + obj = { count: (x = 1) => x === 3 ? 3 : obj.count(x + 1) } + }) + + it('should not throw when wrapper code is throwing', () => { + shimmer.wrap(obj, 'count', (count) => { + return function (x) { + if (x === 2) { + throw new Error('wrapper error') + } + return count.apply(this, arguments) + } + }) + + expect(obj.count()).to.equal(3) + }) + + it('should not throw when wrapper code is throwing mid-recursion', () => { + shimmer.wrap(obj, 'count', (count) => { + return function (x) { + const returnValue = count.apply(this, arguments) + if (x === 2) { + throw new Error('wrapper error') + } + return returnValue + } + }) + + expect(obj.count()).to.equal(3) + }) + + it('should not throw when wrapper code is throwing after return', () => { + shimmer.wrap(obj, 'count', (count) => { + return function (x) { + const returnValue = count.apply(this, arguments) + if (x === 3) { + throw new Error('wrapper error') + } + return returnValue + } + }) + + expect(obj.count()).to.equal(3) + }) + }) + + describe('async', () => { + beforeEach(() => { + obj = { count: async () => await Promise.resolve(3) } + }) + + it('should not throw when wrapper code is throwing', async () => { + shimmer.wrap(obj, 'count', (count) => { + return async function (x) { + if (x === 2) { + throw new Error('wrapper error') + } + return await count.apply(this, arguments) + } + }) + + expect(await obj.count()).to.equal(3) + }) + + it('should not throw when wrapper code is throwing after return', async () => { + shimmer.wrap(obj, 'count', (count) => { + return async () => { + await count() + throw new Error('wrapper error') + } + }) + + expect(await obj.count()).to.equal(3) + }) + }) + + describe('async recursion', () => { + beforeEach(() => { + obj = { + async count (x = 1) { + if (x === 3) return await Promise.resolve(3) + else return await obj.count(x + 1) + } + } + }) + + it('should not throw when wrapper code is throwing', async () => { + shimmer.wrap(obj, 'count', (count) => { + return async function (x) { + if (x === 2) { + throw new Error('wrapper error') + } + return await count.apply(this, arguments) + } + }) + + expect(await obj.count()).to.equal(3) + }) + + it('should not throw when wrapper code is throwing mid-recursion', async () => { + shimmer.wrap(obj, 'count', (count) => { + return async function (x) { + const returnValue = await count.apply(this, arguments) + if (x === 2) { + throw new Error('wrapper error') + } + return returnValue + } + }) + + expect(await obj.count()).to.equal(3) + }) + + it('should not throw when wrapper code is throwing after return', async () => { + shimmer.wrap(obj, 'count', (count) => { + return async function (x) { + const returnValue = await count.apply(this, arguments) + if (x === 3) { + throw new Error('wrapper error') + } + return returnValue + } + }) + + expect(await obj.count()).to.equal(3) + }) + }) + // describe('callback', () => { + // it('should not throw when wrapper code is throwing', (done) => { + // const obj = { count: cb => setImmediate(() => cb(null, 3)) } + + // shimmer.wrap(obj, 'count', () => { + // return () => { + // throw new Error('wrapper error') + // } + // }) + + // obj.count((err, res) => { + // expect(res).to.equal(3) + // done() + // }) + // }) + // it('should not throw when wrapper code calls cb with error', async () => { + // const obj = { count: cb => setImmediate(() => cb(null, 3)) } + + // shimmer.wrap(obj, 'count', (count) => { + // return (cb) => { + // count((err, val) => { + // cb(new Error('wrapper error')) + // }) + // } + // }) + + // obj.count((err, res) => { + // expect(err).to.be.undefined + // expect(res).to.equal(3) + // done() + // }) + // }) + // }) + }) }) describe('with a function', () => { + it('should not work with a wrap()', () => { + expect(() => shimmer.wrap(() => {}, () => {})).to.throw() + }) + + it('should work without a function', () => { + const a = { b: 1 } + const wrapped = shimmer.wrapFunction(a, x => () => x) + expect(wrapped()).to.equal(a) + }) + it('should wrap the function', () => { const count = inc => inc - const wrapped = shimmer.wrap(count, inc => count(inc) + 1) + const wrapped = shimmer.wrapFunction(count, count => inc => count(inc) + 1) expect(wrapped).to.not.equal(count) expect(wrapped(1)).to.equal(2) @@ -244,7 +455,7 @@ describe('shimmer', () => { this.value = start } - const WrappedCounter = shimmer.wrap(Counter, function (...args) { + const WrappedCounter = shimmer.wrapFunction(Counter, Counter => function (...args) { Counter.apply(this, arguments) this.value++ }) @@ -262,7 +473,7 @@ describe('shimmer', () => { } } - expect(() => shimmer.wrap(Counter, function () {})).to.throw( + expect(() => shimmer.wrapFunction(Counter, Counter => function () {})).to.throw( 'Target is a native class constructor and cannot be wrapped.' ) }) @@ -276,7 +487,7 @@ describe('shimmer', () => { Counter.toString = 'invalid' - expect(() => shimmer.wrap(Counter, function () {})).to.throw( + expect(() => shimmer.wrapFunction(Counter, Counter => function () {})).to.throw( 'Target is a native class constructor and cannot be wrapped.' ) }) @@ -291,7 +502,7 @@ describe('shimmer', () => { count.foo = 'foo' count[sym] = 'sym' - const wrapped = shimmer.wrap(count, () => {}) + const wrapped = shimmer.wrapFunction(count, count => () => {}) const bar = Object.getOwnPropertyDescriptor(wrapped, 'bar') expect(wrapped).to.have.property('foo', 'foo') @@ -304,7 +515,7 @@ describe('shimmer', () => { it('should preserve the original function length', () => { const count = (a, b, c) => {} - const wrapped = shimmer.wrap(count, () => {}) + const wrapped = shimmer.wrapFunction(count, count => () => {}) expect(wrapped).to.have.length(3) }) @@ -312,7 +523,7 @@ describe('shimmer', () => { it('should preserve the original function name', () => { const count = function count (a, b, c) {} - const wrapped = shimmer.wrap(count, () => {}) + const wrapped = shimmer.wrapFunction(count, count => () => {}) expect(wrapped).to.have.property('name', 'count') }) @@ -322,7 +533,7 @@ describe('shimmer', () => { Object.getPrototypeOf(count).test = 'test' - const wrapped = shimmer.wrap(count, () => {}) + const wrapped = shimmer.wrapFunction(count, count => () => {}) expect(wrapped).to.have.property('test', 'test') expect(Object.getOwnPropertyNames(wrapped)).to.not.include('test') @@ -331,7 +542,7 @@ describe('shimmer', () => { it('should unwrap a function', () => { const count = inc => inc - const wrapped = shimmer.wrap(count, inc => count(inc) + 1) + const wrapped = shimmer.wrapFunction(count, count => inc => count(inc) + 1) shimmer.unwrap(wrapped) @@ -343,7 +554,7 @@ describe('shimmer', () => { this.value = start } - const WrappedCounter = shimmer.wrap(Counter, function (...args) { + const WrappedCounter = shimmer.wrapFunction(Counter, Counter => function (...args) { Counter.apply(this, arguments) this.value++ }) diff --git a/packages/dd-trace/src/analytics_sampler.js b/packages/dd-trace/src/analytics_sampler.js index e6261700555..8c0ff1a6dcf 100644 --- a/packages/dd-trace/src/analytics_sampler.js +++ b/packages/dd-trace/src/analytics_sampler.js @@ -4,7 +4,7 @@ const { MEASURED } = require('../../../ext/tags') module.exports = { sample (span, measured, measuredByDefault) { - if (typeof measured === 'object') { + if (measured !== null && typeof measured === 'object') { this.sample(span, measured[span.context()._name], measuredByDefault) } else if (measured !== undefined) { span.setTag(MEASURED, !!measured) diff --git a/packages/dd-trace/src/appsec/activation.js b/packages/dd-trace/src/appsec/activation.js new file mode 100644 index 00000000000..8ed6a26fa54 --- /dev/null +++ b/packages/dd-trace/src/appsec/activation.js @@ -0,0 +1,29 @@ +'use strict' + +const Activation = { + ONECLICK: 'OneClick', + ENABLED: 'Enabled', + DISABLED: 'Disabled', + + fromConfig (config) { + switch (config.appsec.enabled) { + // ASM is activated by an env var DD_APPSEC_ENABLED=true + case true: + return Activation.ENABLED + + // ASM is disabled by an env var DD_APPSEC_ENABLED=false + case false: + return Activation.DISABLED + + // ASM is activated by one click remote config + case undefined: + return Activation.ONECLICK + + // Any other value should never occur + default: + return Activation.DISABLED + } + } +} + +module.exports = Activation diff --git a/packages/dd-trace/src/appsec/addresses.js b/packages/dd-trace/src/appsec/addresses.js index a4d47243a67..40c643012ef 100644 --- a/packages/dd-trace/src/appsec/addresses.js +++ b/packages/dd-trace/src/appsec/addresses.js @@ -13,9 +13,21 @@ module.exports = { HTTP_INCOMING_RESPONSE_HEADERS: 'server.response.headers.no_cookies', // TODO: 'server.response.trailers', HTTP_INCOMING_GRAPHQL_RESOLVERS: 'graphql.server.all_resolvers', + HTTP_INCOMING_GRAPHQL_RESOLVER: 'graphql.server.resolver', + + HTTP_INCOMING_RESPONSE_BODY: 'server.response.body', HTTP_CLIENT_IP: 'http.client_ip', USER_ID: 'usr.id', - WAF_CONTEXT_PROCESSOR: 'waf.context.processor' + WAF_CONTEXT_PROCESSOR: 'waf.context.processor', + + HTTP_OUTGOING_URL: 'server.io.net.url', + FS_OPERATION_PATH: 'server.io.fs.file', + + DB_STATEMENT: 'server.db.statement', + DB_SYSTEM: 'server.db.system', + + LOGIN_SUCCESS: 'server.business_logic.users.login.success', + LOGIN_FAILURE: 'server.business_logic.users.login.failure' } diff --git a/packages/dd-trace/src/appsec/api_security_sampler.js b/packages/dd-trace/src/appsec/api_security_sampler.js new file mode 100644 index 00000000000..68bd896af7e --- /dev/null +++ b/packages/dd-trace/src/appsec/api_security_sampler.js @@ -0,0 +1,61 @@ +'use strict' + +const log = require('../log') + +let enabled +let requestSampling + +const sampledRequests = new WeakSet() + +function configure ({ apiSecurity }) { + enabled = apiSecurity.enabled + setRequestSampling(apiSecurity.requestSampling) +} + +function disable () { + enabled = false +} + +function setRequestSampling (sampling) { + requestSampling = parseRequestSampling(sampling) +} + +function parseRequestSampling (requestSampling) { + let parsed = parseFloat(requestSampling) + + if (isNaN(parsed)) { + log.warn(`Incorrect API Security request sampling value: ${requestSampling}`) + + parsed = 0 + } else { + parsed = Math.min(1, Math.max(0, parsed)) + } + + return parsed +} + +function sampleRequest (req) { + if (!enabled || !requestSampling) { + return false + } + + const shouldSample = Math.random() <= requestSampling + + if (shouldSample) { + sampledRequests.add(req) + } + + return shouldSample +} + +function isSampled (req) { + return sampledRequests.has(req) +} + +module.exports = { + configure, + disable, + setRequestSampling, + sampleRequest, + isSampled +} diff --git a/packages/dd-trace/src/appsec/blocked_templates.js b/packages/dd-trace/src/appsec/blocked_templates.js index 7dcd1ffe519..1eb62e22df0 100644 --- a/packages/dd-trace/src/appsec/blocked_templates.js +++ b/packages/dd-trace/src/appsec/blocked_templates.js @@ -5,7 +5,10 @@ const html = `= 400) { statusCode = 303 } + const headers = { + Location: actionParameters.location + } - res.writeHead(statusCode, { - 'Location': blockingConfiguration.parameters.location - }).end() + return { headers, statusCode } +} - if (abortController) { - abortController.abort() +function getSpecificBlockingData (type) { + switch (type) { + case specificBlockingTypes.GRAPHQL: + return { + type: 'application/json', + body: templateGraphqlJson + } } } -function blockWithContent (req, res, rootSpan, abortController) { +function getBlockWithContentData (req, specificType, actionParameters) { let type let body - // parse the Accept header, ex: Accept: text/html, application/xhtml+xml, application/xml;q=0.9, */*;q=0.8 - const accept = req.headers.accept && req.headers.accept.split(',').map((str) => str.split(';', 1)[0].trim()) + const specificBlockingType = specificType || detectedSpecificEndpoints[getSpecificKey(req.method, req.url)] + if (specificBlockingType) { + const specificBlockingContent = getSpecificBlockingData(specificBlockingType) + type = specificBlockingContent?.type + body = specificBlockingContent?.body + } - if (!blockingConfiguration || blockingConfiguration.parameters.type === 'auto') { - if (accept && accept.includes('text/html') && !accept.includes('application/json')) { - type = 'text/html; charset=utf-8' - body = templateHtml - } else { - type = 'application/json' - body = templateJson - } - } else { - if (blockingConfiguration.parameters.type === 'html') { - type = 'text/html; charset=utf-8' - body = templateHtml + if (!type) { + // parse the Accept header, ex: Accept: text/html, application/xhtml+xml, application/xml;q=0.9, */*;q=0.8 + const accept = req.headers.accept?.split(',').map((str) => str.split(';', 1)[0].trim()) + + if (!actionParameters || actionParameters.type === 'auto') { + if (accept?.includes('text/html') && !accept.includes('application/json')) { + type = 'text/html; charset=utf-8' + body = templateHtml + } else { + type = 'application/json' + body = templateJson + } } else { - type = 'application/json' - body = templateJson + if (actionParameters.type === 'html') { + type = 'text/html; charset=utf-8' + body = templateHtml + } else { + type = 'application/json' + body = templateJson + } } } - rootSpan.addTags({ - 'appsec.blocked': 'true' - }) + const statusCode = actionParameters?.status_code || 403 - if (blockingConfiguration && blockingConfiguration.type === 'block_request' && - blockingConfiguration.parameters.status_code) { - res.statusCode = blockingConfiguration.parameters.status_code - } else { - res.statusCode = 403 + const headers = { + 'Content-Type': type, + 'Content-Length': Buffer.byteLength(body) } - res.setHeader('Content-Type', type) - res.setHeader('Content-Length', Buffer.byteLength(body)) - res.end(body) - if (abortController) { - abortController.abort() + return { body, statusCode, headers } +} + +function getBlockingData (req, specificType, actionParameters) { + if (actionParameters?.location) { + return getBlockWithRedirectData(actionParameters) + } else { + return getBlockWithContentData(req, specificType, actionParameters) } } -function block (req, res, rootSpan, abortController) { +function block (req, res, rootSpan, abortController, actionParameters = defaultBlockingActionParameters) { if (res.headersSent) { log.warn('Cannot send blocking response when headers have already been sent') return } - if (blockingConfiguration && blockingConfiguration.type === 'redirect_request' && - blockingConfiguration.parameters.location) { - blockWithRedirect(res, rootSpan, abortController) - } else { - blockWithContent(req, res, rootSpan, abortController) + const { body, headers, statusCode } = getBlockingData(req, null, actionParameters) + + rootSpan.addTags({ + 'appsec.blocked': 'true' + }) + + for (const headerName of res.getHeaderNames()) { + res.removeHeader(headerName) } + + res.writeHead(statusCode, headers).end(body) + + responseBlockedSet.add(res) + + abortController?.abort() +} + +function getBlockingAction (actions) { + // waf only returns one action, but it prioritizes redirect over block + return actions?.redirect_request || actions?.block_request } function setTemplates (config) { if (config.appsec.blockedTemplateHtml) { templateHtml = config.appsec.blockedTemplateHtml + } else { + templateHtml = blockedTemplates.html } + if (config.appsec.blockedTemplateJson) { templateJson = config.appsec.blockedTemplateJson + } else { + templateJson = blockedTemplates.json + } + + if (config.appsec.blockedTemplateGraphql) { + templateGraphqlJson = config.appsec.blockedTemplateGraphql + } else { + templateGraphqlJson = blockedTemplates.graphqlJson } } -function updateBlockingConfiguration (newBlockingConfiguration) { - blockingConfiguration = newBlockingConfiguration +function isBlocked (res) { + return responseBlockedSet.has(res) +} + +function setDefaultBlockingActionParameters (actions) { + const blockAction = actions?.find(action => action.id === 'block') + + defaultBlockingActionParameters = blockAction?.parameters } module.exports = { + addSpecificEndpoint, block, + specificBlockingTypes, + getBlockingData, + getBlockingAction, setTemplates, - updateBlockingConfiguration + isBlocked, + setDefaultBlockingActionParameters } diff --git a/packages/dd-trace/src/appsec/channels.js b/packages/dd-trace/src/appsec/channels.js index cf31b12d233..3081ed9974a 100644 --- a/packages/dd-trace/src/appsec/channels.js +++ b/packages/dd-trace/src/appsec/channels.js @@ -6,12 +6,27 @@ const dc = require('dc-polyfill') module.exports = { bodyParser: dc.channel('datadog:body-parser:read:finish'), cookieParser: dc.channel('datadog:cookie-parser:read:finish'), - graphqlFinishExecute: dc.channel('apm:graphql:execute:finish'), + startGraphqlResolve: dc.channel('datadog:graphql:resolver:start'), + graphqlMiddlewareChannel: dc.tracingChannel('datadog:apollo:middleware'), + apolloChannel: dc.tracingChannel('datadog:apollo:request'), + apolloServerCoreChannel: dc.tracingChannel('datadog:apollo-server-core:request'), incomingHttpRequestStart: dc.channel('dd-trace:incomingHttpRequestStart'), incomingHttpRequestEnd: dc.channel('dd-trace:incomingHttpRequestEnd'), passportVerify: dc.channel('datadog:passport:verify:finish'), queryParser: dc.channel('datadog:query:read:finish'), setCookieChannel: dc.channel('datadog:iast:set-cookie'), nextBodyParsed: dc.channel('apm:next:body-parsed'), - nextQueryParsed: dc.channel('apm:next:query-parsed') + nextQueryParsed: dc.channel('apm:next:query-parsed'), + expressProcessParams: dc.channel('datadog:express:process_params:start'), + responseBody: dc.channel('datadog:express:response:json:start'), + responseWriteHead: dc.channel('apm:http:server:response:writeHead:start'), + httpClientRequestStart: dc.channel('apm:http:client:request:start'), + responseSetHeader: dc.channel('datadog:http:server:response:set-header:start'), + setUncaughtExceptionCaptureCallbackStart: dc.channel('datadog:process:setUncaughtExceptionCaptureCallback:start'), + pgQueryStart: dc.channel('apm:pg:query:start'), + pgPoolQueryStart: dc.channel('datadog:pg:pool:query:start'), + mysql2OuterQueryStart: dc.channel('datadog:mysql2:outerquery:start'), + wafRunFinished: dc.channel('datadog:waf:run:finish'), + fsOperationStart: dc.channel('apm:fs:operation:start'), + expressMiddlewareError: dc.channel('apm:express:middleware:error') } diff --git a/packages/dd-trace/src/appsec/graphql.js b/packages/dd-trace/src/appsec/graphql.js new file mode 100644 index 00000000000..2f715717d27 --- /dev/null +++ b/packages/dd-trace/src/appsec/graphql.js @@ -0,0 +1,155 @@ +'use strict' + +const { storage } = require('../../../datadog-core') +const { + addSpecificEndpoint, + specificBlockingTypes, + getBlockingData, + getBlockingAction +} = require('./blocking') +const waf = require('./waf') +const addresses = require('./addresses') +const web = require('../plugins/util/web') +const { + startGraphqlResolve, + graphqlMiddlewareChannel, + apolloChannel, + apolloServerCoreChannel +} = require('./channels') + +const graphqlRequestData = new WeakMap() + +function enable () { + enableApollo() + enableGraphql() +} + +function disable () { + disableApollo() + disableGraphql() +} + +function onGraphqlStartResolve ({ context, resolverInfo }) { + const req = storage.getStore()?.req + + if (!req) return + + if (!resolverInfo || typeof resolverInfo !== 'object') return + + const actions = waf.run({ ephemeral: { [addresses.HTTP_INCOMING_GRAPHQL_RESOLVER]: resolverInfo } }, req) + const blockingAction = getBlockingAction(actions) + if (blockingAction) { + const requestData = graphqlRequestData.get(req) + if (requestData?.isInGraphqlRequest) { + requestData.blocked = true + requestData.wafAction = blockingAction + context?.abortController?.abort() + } + } +} + +function enterInApolloMiddleware (data) { + const req = data?.req || storage.getStore()?.req + if (!req) return + + graphqlRequestData.set(req, { + inApolloMiddleware: true, + blocked: false + }) +} + +function enterInApolloServerCoreRequest () { + const req = storage.getStore()?.req + if (!req) return + + graphqlRequestData.set(req, { + isInGraphqlRequest: true, + blocked: false + }) +} + +function exitFromApolloMiddleware (data) { + const req = data?.req || storage.getStore()?.req + const requestData = graphqlRequestData.get(req) + if (requestData) requestData.inApolloMiddleware = false +} + +function enterInApolloRequest () { + const req = storage.getStore()?.req + + const requestData = graphqlRequestData.get(req) + if (requestData?.inApolloMiddleware) { + requestData.isInGraphqlRequest = true + addSpecificEndpoint(req.method, req.originalUrl || req.url, specificBlockingTypes.GRAPHQL) + } +} + +function beforeWriteApolloGraphqlResponse ({ abortController, abortData }) { + const req = storage.getStore()?.req + if (!req) return + + const requestData = graphqlRequestData.get(req) + + if (requestData?.blocked) { + const rootSpan = web.root(req) + if (!rootSpan) return + + const blockingData = getBlockingData(req, specificBlockingTypes.GRAPHQL, requestData.wafAction) + abortData.statusCode = blockingData.statusCode + abortData.headers = blockingData.headers + abortData.message = blockingData.body + + rootSpan.setTag('appsec.blocked', 'true') + + abortController?.abort() + } + + graphqlRequestData.delete(req) +} + +function enableApollo () { + graphqlMiddlewareChannel.subscribe({ + start: enterInApolloMiddleware, + end: exitFromApolloMiddleware + }) + + apolloServerCoreChannel.subscribe({ + start: enterInApolloServerCoreRequest, + asyncEnd: beforeWriteApolloGraphqlResponse + }) + + apolloChannel.subscribe({ + start: enterInApolloRequest, + asyncEnd: beforeWriteApolloGraphqlResponse + }) +} + +function disableApollo () { + graphqlMiddlewareChannel.unsubscribe({ + start: enterInApolloMiddleware, + end: exitFromApolloMiddleware + }) + + apolloServerCoreChannel.unsubscribe({ + start: enterInApolloServerCoreRequest, + asyncEnd: beforeWriteApolloGraphqlResponse + }) + + apolloChannel.unsubscribe({ + start: enterInApolloRequest, + asyncEnd: beforeWriteApolloGraphqlResponse + }) +} + +function enableGraphql () { + startGraphqlResolve.subscribe(onGraphqlStartResolve) +} + +function disableGraphql () { + if (startGraphqlResolve.hasSubscribers) startGraphqlResolve.unsubscribe(onGraphqlStartResolve) +} + +module.exports = { + enable, + disable +} diff --git a/packages/dd-trace/src/appsec/iast/analyzers/analyzers.js b/packages/dd-trace/src/appsec/iast/analyzers/analyzers.js index 7152d07458f..36f6036cf54 100644 --- a/packages/dd-trace/src/appsec/iast/analyzers/analyzers.js +++ b/packages/dd-trace/src/appsec/iast/analyzers/analyzers.js @@ -1,20 +1,23 @@ 'use strict' module.exports = { - 'COMMAND_INJECTION_ANALYZER': require('./command-injection-analyzer'), - 'HARCODED_SECRET_ANALYZER': require('./hardcoded-secret-analyzer'), - 'HEADER_INJECTION_ANALYZER': require('./header-injection-analyzer'), - 'HSTS_HEADER_MISSING_ANALYZER': require('./hsts-header-missing-analyzer'), - 'INSECURE_COOKIE_ANALYZER': require('./insecure-cookie-analyzer'), - 'LDAP_ANALYZER': require('./ldap-injection-analyzer'), - 'NO_HTTPONLY_COOKIE_ANALYZER': require('./no-httponly-cookie-analyzer'), - 'NO_SAMESITE_COOKIE_ANALYZER': require('./no-samesite-cookie-analyzer'), - 'NOSQL_MONGODB_INJECTION': require('./nosql-injection-mongodb-analyzer'), - 'PATH_TRAVERSAL_ANALYZER': require('./path-traversal-analyzer'), - 'SQL_INJECTION_ANALYZER': require('./sql-injection-analyzer'), - 'SSRF': require('./ssrf-analyzer'), - 'UNVALIDATED_REDIRECT_ANALYZER': require('./unvalidated-redirect-analyzer'), - 'WEAK_CIPHER_ANALYZER': require('./weak-cipher-analyzer'), - 'WEAK_HASH_ANALYZER': require('./weak-hash-analyzer'), - 'XCONTENTTYPE_HEADER_MISSING_ANALYZER': require('./xcontenttype-header-missing-analyzer') + CODE_INJECTION_ANALYZER: require('./code-injection-analyzer'), + COMMAND_INJECTION_ANALYZER: require('./command-injection-analyzer'), + HARCODED_PASSWORD_ANALYZER: require('./hardcoded-password-analyzer'), + HARCODED_SECRET_ANALYZER: require('./hardcoded-secret-analyzer'), + HEADER_INJECTION_ANALYZER: require('./header-injection-analyzer'), + HSTS_HEADER_MISSING_ANALYZER: require('./hsts-header-missing-analyzer'), + INSECURE_COOKIE_ANALYZER: require('./insecure-cookie-analyzer'), + LDAP_ANALYZER: require('./ldap-injection-analyzer'), + NO_HTTPONLY_COOKIE_ANALYZER: require('./no-httponly-cookie-analyzer'), + NO_SAMESITE_COOKIE_ANALYZER: require('./no-samesite-cookie-analyzer'), + NOSQL_MONGODB_INJECTION: require('./nosql-injection-mongodb-analyzer'), + PATH_TRAVERSAL_ANALYZER: require('./path-traversal-analyzer'), + SQL_INJECTION_ANALYZER: require('./sql-injection-analyzer'), + SSRF: require('./ssrf-analyzer'), + UNVALIDATED_REDIRECT_ANALYZER: require('./unvalidated-redirect-analyzer'), + WEAK_CIPHER_ANALYZER: require('./weak-cipher-analyzer'), + WEAK_HASH_ANALYZER: require('./weak-hash-analyzer'), + WEAK_RANDOMNESS_ANALYZER: require('./weak-randomness-analyzer'), + XCONTENTTYPE_HEADER_MISSING_ANALYZER: require('./xcontenttype-header-missing-analyzer') } diff --git a/packages/dd-trace/src/appsec/iast/analyzers/code-injection-analyzer.js b/packages/dd-trace/src/appsec/iast/analyzers/code-injection-analyzer.js new file mode 100644 index 00000000000..f8937417e42 --- /dev/null +++ b/packages/dd-trace/src/appsec/iast/analyzers/code-injection-analyzer.js @@ -0,0 +1,16 @@ +'use strict' + +const InjectionAnalyzer = require('./injection-analyzer') +const { CODE_INJECTION } = require('../vulnerabilities') + +class CodeInjectionAnalyzer extends InjectionAnalyzer { + constructor () { + super(CODE_INJECTION) + } + + onConfigure () { + this.addSub('datadog:eval:call', ({ script }) => this.analyze(script)) + } +} + +module.exports = new CodeInjectionAnalyzer() diff --git a/packages/dd-trace/src/appsec/iast/analyzers/command-injection-analyzer.js b/packages/dd-trace/src/appsec/iast/analyzers/command-injection-analyzer.js index eccf8a3814b..fd2a230a2a8 100644 --- a/packages/dd-trace/src/appsec/iast/analyzers/command-injection-analyzer.js +++ b/packages/dd-trace/src/appsec/iast/analyzers/command-injection-analyzer.js @@ -8,7 +8,7 @@ class CommandInjectionAnalyzer extends InjectionAnalyzer { } onConfigure () { - this.addSub('datadog:child_process:execution:start', ({ command }) => this.analyze(command)) + this.addSub('tracing:datadog:child_process:execution:start', ({ command }) => this.analyze(command)) } } diff --git a/packages/dd-trace/src/appsec/iast/analyzers/cookie-analyzer.js b/packages/dd-trace/src/appsec/iast/analyzers/cookie-analyzer.js index 413653a7af2..2b125b88403 100644 --- a/packages/dd-trace/src/appsec/iast/analyzers/cookie-analyzer.js +++ b/packages/dd-trace/src/appsec/iast/analyzers/cookie-analyzer.js @@ -2,6 +2,7 @@ const Analyzer = require('./vulnerability-analyzer') const { getNodeModulesPaths } = require('../path-line') +const iastLog = require('../iast-log') const EXCLUDED_PATHS = getNodeModulesPaths('express/lib/response.js') @@ -11,7 +12,14 @@ class CookieAnalyzer extends Analyzer { this.propertyToBeSafe = propertyToBeSafe.toLowerCase() } - onConfigure () { + onConfigure (config) { + try { + this.cookieFilterRegExp = new RegExp(config.iast.cookieFilterPattern) + } catch { + iastLog.error('Invalid regex in cookieFilterPattern') + this.cookieFilterRegExp = /.{32,}/ + } + this.addSub( { channelName: 'datadog:iast:set-cookie', moduleName: 'http' }, (cookieInfo) => this.analyze(cookieInfo) @@ -28,12 +36,17 @@ class CookieAnalyzer extends Analyzer { } _createHashSource (type, evidence, location) { + if (typeof evidence.value === 'string' && evidence.value.match(this.cookieFilterRegExp)) { + return 'FILTERED_' + this._type + } + return `${type}:${evidence.value}` } _getExcludedPaths () { return EXCLUDED_PATHS } + _checkOCE (context, value) { if (value && value.location) { return true diff --git a/packages/dd-trace/src/appsec/iast/analyzers/hardcoded-base-analyzer.js b/packages/dd-trace/src/appsec/iast/analyzers/hardcoded-base-analyzer.js new file mode 100644 index 00000000000..36a4832ee74 --- /dev/null +++ b/packages/dd-trace/src/appsec/iast/analyzers/hardcoded-base-analyzer.js @@ -0,0 +1,71 @@ +'use strict' + +const Analyzer = require('./vulnerability-analyzer') +const { getRelativePath } = require('../path-line') + +class HardcodedBaseAnalyzer extends Analyzer { + constructor (type, allRules = [], valueOnlyRules = []) { + super(type) + + this.allRules = allRules + this.valueOnlyRules = valueOnlyRules + } + + onConfigure () { + this.addSub('datadog:secrets:result', (secrets) => { this.analyze(secrets) }) + } + + analyze (secrets) { + if (!secrets?.file || !secrets.literals) return + + const { allRules, valueOnlyRules } = this + + const matches = [] + for (const literal of secrets.literals) { + const { value, locations } = literal + if (!value || !locations) continue + + for (const location of locations) { + let match + if (location.ident) { + const fullValue = `${location.ident}=${value}` + match = allRules.find(rule => fullValue.match(rule.regex)) + } else { + match = valueOnlyRules.find(rule => value.match(rule.regex)) + } + + if (match) { + matches.push({ location, ruleId: match.id }) + } + } + } + + if (matches.length) { + const file = getRelativePath(secrets.file) + + matches + .forEach(match => this._report({ + file, + line: match.location.line, + column: match.location.column, + ident: match.location.ident, + data: match.ruleId + })) + } + } + + _getEvidence (value) { + return { value: `${value.data}` } + } + + _getLocation (value) { + return { + path: value.file, + line: value.line, + column: value.column, + isInternal: false + } + } +} + +module.exports = HardcodedBaseAnalyzer diff --git a/packages/dd-trace/src/appsec/iast/analyzers/hardcoded-password-analyzer.js b/packages/dd-trace/src/appsec/iast/analyzers/hardcoded-password-analyzer.js new file mode 100644 index 00000000000..509b292291f --- /dev/null +++ b/packages/dd-trace/src/appsec/iast/analyzers/hardcoded-password-analyzer.js @@ -0,0 +1,18 @@ +'use strict' + +const { HARDCODED_PASSWORD } = require('../vulnerabilities') +const HardcodedBaseAnalyzer = require('./hardcoded-base-analyzer') + +const allRules = require('./hardcoded-password-rules') + +class HardcodedPasswordAnalyzer extends HardcodedBaseAnalyzer { + constructor () { + super(HARDCODED_PASSWORD, allRules) + } + + _getEvidence (value) { + return { value: `${value.ident}` } + } +} + +module.exports = new HardcodedPasswordAnalyzer() diff --git a/packages/dd-trace/src/appsec/iast/analyzers/hardcoded-password-rules.js b/packages/dd-trace/src/appsec/iast/analyzers/hardcoded-password-rules.js new file mode 100644 index 00000000000..2e204b72830 --- /dev/null +++ b/packages/dd-trace/src/appsec/iast/analyzers/hardcoded-password-rules.js @@ -0,0 +1,12 @@ +/* eslint-disable max-len */ +'use strict' + +const { NameAndValue } = require('./hardcoded-rule-type') + +module.exports = [ + { + id: 'hardcoded-password', + regex: /(?:pwd|pswd|pass|secret)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([0-9a-z\-_.=]{10,150})(?:['"\s\x60;]|$)/i, + type: NameAndValue + } +] diff --git a/packages/dd-trace/src/appsec/iast/analyzers/hardcoded-rule-type.js b/packages/dd-trace/src/appsec/iast/analyzers/hardcoded-rule-type.js new file mode 100644 index 00000000000..001d336153d --- /dev/null +++ b/packages/dd-trace/src/appsec/iast/analyzers/hardcoded-rule-type.js @@ -0,0 +1,6 @@ +'use strict' + +module.exports = { + ValueOnly: 'ValueOnly', + NameAndValue: 'NameAndValue' +} diff --git a/packages/dd-trace/src/appsec/iast/analyzers/hardcoded-secret-analyzer.js b/packages/dd-trace/src/appsec/iast/analyzers/hardcoded-secret-analyzer.js index 6b1a6172e1f..e7c33536050 100644 --- a/packages/dd-trace/src/appsec/iast/analyzers/hardcoded-secret-analyzer.js +++ b/packages/dd-trace/src/appsec/iast/analyzers/hardcoded-secret-analyzer.js @@ -1,59 +1,14 @@ 'use strict' -const Analyzer = require('./vulnerability-analyzer') const { HARDCODED_SECRET } = require('../vulnerabilities') -const { getRelativePath } = require('../path-line') +const HardcodedBaseAnalyzer = require('./hardcoded-base-analyzer') +const { ValueOnly } = require('./hardcoded-rule-type') -const secretRules = require('./hardcoded-secrets-rules') +const allRules = require('./hardcoded-secret-rules') -class HardcodedSecretAnalyzer extends Analyzer { +class HardcodedSecretAnalyzer extends HardcodedBaseAnalyzer { constructor () { - super(HARDCODED_SECRET) - } - - onConfigure () { - this.addSub('datadog:secrets:result', (secrets) => { this.analyze(secrets) }) - } - - analyze (secrets) { - if (!secrets?.file || !secrets.literals) return - - const matches = secrets.literals - .filter(literal => literal.value && literal.locations?.length) - .map(literal => { - const match = secretRules.find(rule => literal.value.match(rule.regex)) - - return match ? { locations: literal.locations, ruleId: match.id } : undefined - }) - .filter(match => !!match) - - if (matches.length) { - const file = getRelativePath(secrets.file) - - matches.forEach(match => { - match.locations - .filter(location => location.line) - .forEach(location => this._report({ - file, - line: location.line, - column: location.column, - data: match.ruleId - })) - }) - } - } - - _getEvidence (value) { - return { value: `${value.data}` } - } - - _getLocation (value) { - return { - path: value.file, - line: value.line, - column: value.column, - isInternal: false - } + super(HARDCODED_SECRET, allRules, allRules.filter(rule => rule.type === ValueOnly)) } } diff --git a/packages/dd-trace/src/appsec/iast/analyzers/hardcoded-secret-rules.js b/packages/dd-trace/src/appsec/iast/analyzers/hardcoded-secret-rules.js new file mode 100644 index 00000000000..88ec3d54254 --- /dev/null +++ b/packages/dd-trace/src/appsec/iast/analyzers/hardcoded-secret-rules.js @@ -0,0 +1,742 @@ +/* eslint-disable max-len */ +'use strict' + +const { ValueOnly, NameAndValue } = require('./hardcoded-rule-type') + +module.exports = [ + { + id: 'adafruit-api-key', + regex: /(?:adafruit)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9_-]{32})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'adobe-client-id', + regex: /(?:adobe)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-f0-9]{32})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'adobe-client-secret', + regex: /\b((p8e-)[a-z0-9]{32})(?:['"\s\x60;]|$)/i, + type: ValueOnly + }, + { + id: 'age-secret-key', + regex: /AGE-SECRET-KEY-1[QPZRY9X8GF2TVDW0S3JN54KHCE6MUA7L]{58}/, + type: ValueOnly + }, + { + id: 'airtable-api-key', + regex: /(?:airtable)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{17})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'algolia-api-key', + regex: /(?:algolia)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{32})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'alibaba-access-key-id', + regex: /\b((LTAI)[a-z0-9]{20})(?:['"\s\x60;]|$)/i, + type: ValueOnly + }, + { + id: 'asana-client-id', + regex: /(?:asana)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([0-9]{16})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'asana-client-secret', + regex: /(?:asana)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{32})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'atlassian-api-token', + regex: /(?:atlassian|confluence|jira)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{24})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'authress-service-client-access-key', + regex: /\b((?:sc|ext|scauth|authress)_[a-z0-9]{5,30}\.[a-z0-9]{4,6}\.acc[_-][a-z0-9-]{10,32}\.[a-z0-9+/_=-]{30,120})(?:['"\s\x60;]|$)/i, + type: ValueOnly + }, + { + id: 'aws-access-token', + regex: /\b((A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16})(?:['"\s\x60;]|$)/, + type: ValueOnly + }, + { + id: 'beamer-api-token', + regex: /(?:beamer)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}(b_[a-z0-9=_-]{44})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'bitbucket-client-id', + regex: /(?:bitbucket)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{32})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'bitbucket-client-secret', + regex: /(?:bitbucket)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9=_-]{64})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'bittrex-access-key', + regex: /(?:bittrex)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{32})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'clojars-api-token', + regex: /(CLOJARS_)[a-z0-9]{60}/i, + type: ValueOnly + }, + { + id: 'codecov-access-token', + regex: /(?:codecov)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{32})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'coinbase-access-token', + regex: /(?:coinbase)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9_-]{64})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'confluent-access-token', + regex: /(?:confluent)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{16})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'confluent-secret-key', + regex: /(?:confluent)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{64})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'contentful-delivery-api-token', + regex: /(?:contentful)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9=_-]{43})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'databricks-api-token', + regex: /\b(dapi[a-h0-9]{32})(?:['"\s\x60;]|$)/i, + type: ValueOnly + }, + { + id: 'datadog-access-token', + regex: /(?:datadog)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{40})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'defined-networking-api-token', + regex: /(?:dnkey)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}(dnkey-[a-z0-9=_-]{26}-[a-z0-9=_-]{52})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'digitalocean-access-token', + regex: /\b(doo_v1_[a-f0-9]{64})(?:['"\s\x60;]|$)/i, + type: ValueOnly + }, + { + id: 'digitalocean-pat', + regex: /\b(dop_v1_[a-f0-9]{64})(?:['"\s\x60;]|$)/i, + type: ValueOnly + }, + { + id: 'digitalocean-refresh-token', + regex: /\b(dor_v1_[a-f0-9]{64})(?:['"\s\x60;]|$)/i, + type: ValueOnly + }, + { + id: 'discord-api-token', + regex: /(?:discord)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-f0-9]{64})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'discord-client-id', + regex: /(?:discord)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([0-9]{18})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'discord-client-secret', + regex: /(?:discord)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9=_-]{32})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'doppler-api-token', + regex: /(dp\.pt\.)[a-z0-9]{43}/i, + type: ValueOnly + }, + { + id: 'droneci-access-token', + regex: /(?:droneci)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{32})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'dropbox-api-token', + regex: /(?:dropbox)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{15})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'dropbox-long-lived-api-token', + regex: /(?:dropbox)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{11}(AAAAAAAAAA)[a-z0-9\-_=]{43})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'dropbox-short-lived-api-token', + regex: /(?:dropbox)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}(sl\.[a-z0-9\-=_]{135})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'duffel-api-token', + regex: /duffel_(test|live)_[a-z0-9_\-=]{43}/i, + type: ValueOnly + }, + { + id: 'dynatrace-api-token', + regex: /dt0c01\.[a-z0-9]{24}\.[a-z0-9]{64}/i, + type: ValueOnly + }, + { + id: 'easypost-api-token', + regex: /\bEZAK[a-z0-9]{54}/i, + type: ValueOnly + }, + { + id: 'etsy-access-token', + regex: /(?:etsy)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{24})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'facebook', + regex: /(?:facebook)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-f0-9]{32})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'fastly-api-token', + regex: /(?:fastly)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9=_-]{32})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'finicity-api-token', + regex: /(?:finicity)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-f0-9]{32})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'finicity-client-secret', + regex: /(?:finicity)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{20})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'finnhub-access-token', + regex: /(?:finnhub)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{20})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'flickr-access-token', + regex: /(?:flickr)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{32})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'flutterwave-public-key', + regex: /FLWPUBK_TEST-[a-h0-9]{32}-X/i, + type: ValueOnly + }, + { + id: 'frameio-api-token', + regex: /fio-u-[a-z0-9\-_=]{64}/i, + type: ValueOnly + }, + { + id: 'freshbooks-access-token', + regex: /(?:freshbooks)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{64})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'gcp-api-key', + regex: /\b(AIza[0-9a-z\-_]{35})(?:['"\s\x60;]|$)/i, + type: ValueOnly + }, + { + id: 'github-app-token', + regex: /(ghu|ghs)_[0-9a-zA-Z]{36}/, + type: ValueOnly + }, + { + id: 'github-fine-grained-pat', + regex: /github_pat_[0-9a-zA-Z_]{82}/, + type: ValueOnly + }, + { + id: 'github-oauth', + regex: /gho_[0-9a-zA-Z]{36}/, + type: ValueOnly + }, + { + id: 'github-pat', + regex: /ghp_[0-9a-zA-Z]{36}/, + type: ValueOnly + }, + { + id: 'gitlab-pat', + regex: /glpat-[0-9a-zA-Z\-_]{20}/, + type: ValueOnly + }, + { + id: 'gitlab-ptt', + regex: /glptt-[0-9a-f]{40}/, + type: ValueOnly + }, + { + id: 'gitlab-rrt', + regex: /GR1348941[0-9a-zA-Z\-_]{20}/, + type: ValueOnly + }, + { + id: 'gitter-access-token', + regex: /(?:gitter)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9_-]{40})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'gocardless-api-token', + regex: /(?:gocardless)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}(live_[a-z0-9\-_=]{40})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'grafana-api-key', + regex: /\b(eyJrIjoi[a-z0-9]{70,400}={0,2})(?:['"\s\x60;]|$)/i, + type: ValueOnly + }, + { + id: 'grafana-cloud-api-token', + regex: /\b(glc_[a-z0-9+/]{32,400}={0,2})(?:['"\s\x60;]|$)/i, + type: ValueOnly + }, + { + id: 'grafana-service-account-token', + regex: /\b(glsa_[a-z0-9]{32}_[a-f0-9]{8})(?:['"\s\x60;]|$)/i, + type: ValueOnly + }, + { + id: 'hashicorp-tf-api-token', + regex: /[a-z0-9]{14}\.atlasv1\.[a-z0-9\-_=]{60,70}/i, + type: ValueOnly + }, + { + id: 'heroku-api-key', + regex: /(?:heroku)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'hubspot-api-key', + regex: /(?:hubspot)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'intercom-api-key', + regex: /(?:intercom)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9=_-]{60})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'jfrog-api-key', + regex: /(?:jfrog|artifactory|bintray|xray)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{73})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'jwt', + regex: /\b(ey[a-zA-Z0-9]{17,}\.ey[a-zA-Z0-9/_-]{17,}\.(?:[a-zA-Z0-9/_-]{10,}={0,2})?)(?:['"\s\x60;]|$)/, + type: ValueOnly + }, + { + id: 'kraken-access-token', + regex: /(?:kraken)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9/=_+-]{80,90})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'kucoin-access-token', + regex: /(?:kucoin)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-f0-9]{24})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'launchdarkly-access-token', + regex: /(?:launchdarkly)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9=_-]{40})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'linear-api-key', + regex: /lin_api_[a-z0-9]{40}/i, + type: ValueOnly + }, + { + id: 'linkedin-client-secret', + regex: /(?:linkedin|linked-in)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{16})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'lob-pub-api-key', + regex: /(?:lob)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}((test|live)_pub_[a-f0-9]{31})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'mailchimp-api-key', + regex: /(?:mailchimp)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-f0-9]{32}-us20)(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'mailgun-private-api-token', + regex: /(?:mailgun)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}(key-[a-f0-9]{32})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'mailgun-pub-key', + regex: /(?:mailgun)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}(pubkey-[a-f0-9]{32})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'mailgun-signing-key', + regex: /(?:mailgun)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-h0-9]{32}-[a-h0-9]{8}-[a-h0-9]{8})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'mapbox-api-token', + regex: /(?:mapbox)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}(pk\.[a-z0-9]{60}\.[a-z0-9]{22})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'mattermost-access-token', + regex: /(?:mattermost)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{26})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'messagebird-api-token', + regex: /(?:messagebird|message-bird|message_bird)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{25})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'netlify-access-token', + regex: /(?:netlify)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9=_-]{40,46})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'new-relic-browser-api-token', + regex: /(?:new-relic|newrelic|new_relic)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}(NRJS-[a-f0-9]{19})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'new-relic-user-api-id', + regex: /(?:new-relic|newrelic|new_relic)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{64})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'new-relic-user-api-key', + regex: /(?:new-relic|newrelic|new_relic)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}(NRAK-[a-z0-9]{27})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'npm-access-token', + regex: /\b(npm_[a-z0-9]{36})(?:['"\s\x60;]|$)/i, + type: ValueOnly + }, + { + id: 'nytimes-access-token', + regex: /(?:nytimes|new-york-times,|newyorktimes)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9=_-]{32})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'okta-access-token', + regex: /(?:okta)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9=_-]{42})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'openai-api-key', + regex: /\b(sk-[a-z0-9]{20}T3BlbkFJ[a-z0-9]{20})(?:['"\s\x60;]|$)/i, + type: ValueOnly + }, + { + id: 'plaid-api-token', + regex: /(?:plaid)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}(access-(?:sandbox|development|production)-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'plaid-client-id', + regex: /(?:plaid)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{24})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'plaid-secret-key', + regex: /(?:plaid)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{30})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'planetscale-api-token', + regex: /\b(pscale_tkn_[a-z0-9=\-_.]{32,64})(?:['"\s\x60;]|$)/i, + type: ValueOnly + }, + { + id: 'planetscale-oauth-token', + regex: /\b(pscale_oauth_[a-z0-9=\-_.]{32,64})(?:['"\s\x60;]|$)/i, + type: ValueOnly + }, + { + id: 'planetscale-password', + regex: /\b(pscale_pw_[a-z0-9=\-_.]{32,64})(?:['"\s\x60;]|$)/i, + type: ValueOnly + }, + { + id: 'postman-api-token', + regex: /\b(PMAK-[a-f0-9]{24}-[a-f0-9]{34})(?:['"\s\x60;]|$)/i, + type: ValueOnly + }, + { + id: 'prefect-api-token', + regex: /\b(pnu_[a-z0-9]{36})(?:['"\s\x60;]|$)/i, + type: ValueOnly + }, + { + id: 'private-key', + regex: /-----BEGIN[ A-Z0-9_-]{0,100}PRIVATE KEY( BLOCK)?-----[\s\S]*KEY( BLOCK)?----/i, + type: ValueOnly + }, + { + id: 'pulumi-api-token', + regex: /\b(pul-[a-f0-9]{40})(?:['"\s\x60;]|$)/i, + type: ValueOnly + }, + { + id: 'pypi-upload-token', + regex: /pypi-AgEIcHlwaS5vcmc[A-Za-z0-9\-_]{50,1000}/, + type: ValueOnly + }, + { + id: 'rapidapi-access-token', + regex: /(?:rapidapi)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9_-]{50})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'readme-api-token', + regex: /\b(rdme_[a-z0-9]{70})(?:['"\s\x60;]|$)/i, + type: ValueOnly + }, + { + id: 'rubygems-api-token', + regex: /\b(rubygems_[a-f0-9]{48})(?:['"\s\x60;]|$)/i, + type: ValueOnly + }, + { + id: 'scalingo-api-token', + regex: /tk-us-[a-zA-Z0-9-_]{48}/, + type: ValueOnly + }, + { + id: 'sendbird-access-id', + regex: /(?:sendbird)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'sendbird-access-token', + regex: /(?:sendbird)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-f0-9]{40})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'sendgrid-api-token', + regex: /\b(SG\.[a-z0-9=_\-.]{66})(?:['"\s\x60;]|$)/i, + type: ValueOnly + }, + { + id: 'sendinblue-api-token', + regex: /\b(xkeysib-[a-f0-9]{64}-[a-z0-9]{16})(?:['"\s\x60;]|$)/i, + type: ValueOnly + }, + { + id: 'sentry-access-token', + regex: /(?:sentry)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-f0-9]{64})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'shippo-api-token', + regex: /\b(shippo_(live|test)_[a-f0-9]{40})(?:['"\s\x60;]|$)/i, + type: ValueOnly + }, + { + id: 'shopify-access-token', + regex: /shpat_[a-fA-F0-9]{32}/, + type: ValueOnly + }, + { + id: 'shopify-custom-access-token', + regex: /shpca_[a-fA-F0-9]{32}/, + type: ValueOnly + }, + { + id: 'shopify-private-app-access-token', + regex: /shppa_[a-fA-F0-9]{32}/, + type: ValueOnly + }, + { + id: 'shopify-shared-secret', + regex: /shpss_[a-fA-F0-9]{32}/, + type: ValueOnly + }, + { + id: 'sidekiq-secret', + regex: /(?:BUNDLE_ENTERPRISE__CONTRIBSYS__COM|BUNDLE_GEMS__CONTRIBSYS__COM)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-f0-9]{8}:[a-f0-9]{8})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'slack-app-token', + regex: /(xapp-\d-[A-Z0-9]+-\d+-[a-z0-9]+)/i, + type: ValueOnly + }, + { + id: 'slack-bot-token', + regex: /(xoxb-[0-9]{10,13}-[0-9]{10,13}[a-zA-Z0-9-]*)/, + type: ValueOnly + }, + { + id: 'slack-config-access-token', + regex: /(xoxe.xox[bp]-\d-[A-Z0-9]{163,166})/i, + type: ValueOnly + }, + { + id: 'slack-config-refresh-token', + regex: /(xoxe-\d-[A-Z0-9]{146})/i, + type: ValueOnly + }, + { + id: 'slack-legacy-bot-token', + regex: /(xoxb-[0-9]{8,14}-[a-zA-Z0-9]{18,26})/, + type: ValueOnly + }, + { + id: 'slack-legacy-token', + regex: /(xox[os]-\d+-\d+-\d+-[a-fA-F\d]+)/, + type: ValueOnly + }, + { + id: 'slack-legacy-workspace-token', + regex: /(xox[ar]-(?:\d-)?[0-9a-zA-Z]{8,48})/, + type: ValueOnly + }, + { + id: 'slack-user-token', + regex: /(xox[pe](?:-[0-9]{10,13}){3}-[a-zA-Z0-9-]{28,34})/, + type: ValueOnly + }, + { + id: 'slack-webhook-url', + regex: /(https?:\/\/)?hooks.slack.com\/(services|workflows)\/[A-Za-z0-9+/]{43,46}/, + type: ValueOnly + }, + { + id: 'snyk-api-token', + regex: /(?:snyk)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'square-access-token', + regex: /\b(sq0atp-[0-9a-z\-_]{22})(?:['"\s\x60;]|$)/i, + type: ValueOnly + }, + { + id: 'square-secret', + regex: /\b(sq0csp-[0-9a-z\-_]{43})(?:['"\s\x60;]|$)/i, + type: ValueOnly + }, + { + id: 'squarespace-access-token', + regex: /(?:squarespace)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'stripe-access-token', + regex: /(sk|pk)_(test|live)_[0-9a-z]{10,32}/i, + type: ValueOnly + }, + { + id: 'sumologic-access-token', + regex: /(?:sumo)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{64})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'telegram-bot-api-token', + regex: /(?:^|[^0-9])([0-9]{5,16}:A[a-z0-9_-]{34})(?:$|[^a-z0-9_-])/i, + type: ValueOnly + }, + { + id: 'travisci-access-token', + regex: /(?:travis)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{22})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'trello-access-token', + regex: /(?:trello)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z-0-9]{32})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'twilio-api-key', + regex: /SK[0-9a-fA-F]{32}/, + type: ValueOnly + }, + { + id: 'twitch-api-token', + regex: /(?:twitch)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{30})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'twitter-access-secret', + regex: /(?:twitter)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{45})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'twitter-access-token', + regex: /(?:twitter)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([0-9]{15,25}-[a-z0-9]{20,40})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'twitter-api-key', + regex: /(?:twitter)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{25})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'twitter-api-secret', + regex: /(?:twitter)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{50})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'twitter-bearer-token', + regex: /(?:twitter)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}(A{22}[a-z0-9%]{80,100})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'typeform-api-token', + regex: /(?:typeform)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}(tfp_[a-z0-9\-_.=]{59})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'vault-batch-token', + regex: /\b(hvb\.[a-z0-9_-]{138,212})(?:['"\s\x60;]|$)/i, + type: ValueOnly + }, + { + id: 'vault-service-token', + regex: /\b(hvs\.[a-z0-9_-]{90,100})(?:['"\s\x60;]|$)/i, + type: ValueOnly + }, + { + id: 'yandex-access-token', + regex: /(?:yandex)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}(t1\.[A-Z0-9a-z_-]+[=]{0,2}\.[A-Z0-9a-z_-]{86}[=]{0,2})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'yandex-api-key', + regex: /(?:yandex)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}(AQVN[a-z0-9_-]{35,38})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'yandex-aws-access-token', + regex: /(?:yandex)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}(YC[a-z0-9_-]{38})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'zendesk-secret-key', + regex: /(?:zendesk)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{40})(?:['"\s\x60;]|$)/i, + type: NameAndValue + } +] diff --git a/packages/dd-trace/src/appsec/iast/analyzers/hardcoded-secrets-rules.js b/packages/dd-trace/src/appsec/iast/analyzers/hardcoded-secrets-rules.js index b6069585e5c..88ec3d54254 100644 --- a/packages/dd-trace/src/appsec/iast/analyzers/hardcoded-secrets-rules.js +++ b/packages/dd-trace/src/appsec/iast/analyzers/hardcoded-secrets-rules.js @@ -1,269 +1,742 @@ /* eslint-disable max-len */ 'use strict' +const { ValueOnly, NameAndValue } = require('./hardcoded-rule-type') + module.exports = [ { - 'id': 'adobe-client-secret', - 'regex': /\b((p8e-)[a-z0-9]{32})(?:['"\s\x60;]|$)/i + id: 'adafruit-api-key', + regex: /(?:adafruit)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9_-]{32})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'adobe-client-id', + regex: /(?:adobe)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-f0-9]{32})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'adobe-client-secret', + regex: /\b((p8e-)[a-z0-9]{32})(?:['"\s\x60;]|$)/i, + type: ValueOnly + }, + { + id: 'age-secret-key', + regex: /AGE-SECRET-KEY-1[QPZRY9X8GF2TVDW0S3JN54KHCE6MUA7L]{58}/, + type: ValueOnly + }, + { + id: 'airtable-api-key', + regex: /(?:airtable)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{17})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'algolia-api-key', + regex: /(?:algolia)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{32})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'alibaba-access-key-id', + regex: /\b((LTAI)[a-z0-9]{20})(?:['"\s\x60;]|$)/i, + type: ValueOnly + }, + { + id: 'asana-client-id', + regex: /(?:asana)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([0-9]{16})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'asana-client-secret', + regex: /(?:asana)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{32})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'atlassian-api-token', + regex: /(?:atlassian|confluence|jira)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{24})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'authress-service-client-access-key', + regex: /\b((?:sc|ext|scauth|authress)_[a-z0-9]{5,30}\.[a-z0-9]{4,6}\.acc[_-][a-z0-9-]{10,32}\.[a-z0-9+/_=-]{30,120})(?:['"\s\x60;]|$)/i, + type: ValueOnly + }, + { + id: 'aws-access-token', + regex: /\b((A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16})(?:['"\s\x60;]|$)/, + type: ValueOnly + }, + { + id: 'beamer-api-token', + regex: /(?:beamer)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}(b_[a-z0-9=_-]{44})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'bitbucket-client-id', + regex: /(?:bitbucket)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{32})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'bitbucket-client-secret', + regex: /(?:bitbucket)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9=_-]{64})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'bittrex-access-key', + regex: /(?:bittrex)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{32})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'clojars-api-token', + regex: /(CLOJARS_)[a-z0-9]{60}/i, + type: ValueOnly + }, + { + id: 'codecov-access-token', + regex: /(?:codecov)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{32})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'coinbase-access-token', + regex: /(?:coinbase)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9_-]{64})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'confluent-access-token', + regex: /(?:confluent)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{16})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'confluent-secret-key', + regex: /(?:confluent)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{64})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'contentful-delivery-api-token', + regex: /(?:contentful)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9=_-]{43})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'databricks-api-token', + regex: /\b(dapi[a-h0-9]{32})(?:['"\s\x60;]|$)/i, + type: ValueOnly + }, + { + id: 'datadog-access-token', + regex: /(?:datadog)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{40})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'defined-networking-api-token', + regex: /(?:dnkey)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}(dnkey-[a-z0-9=_-]{26}-[a-z0-9=_-]{52})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'digitalocean-access-token', + regex: /\b(doo_v1_[a-f0-9]{64})(?:['"\s\x60;]|$)/i, + type: ValueOnly + }, + { + id: 'digitalocean-pat', + regex: /\b(dop_v1_[a-f0-9]{64})(?:['"\s\x60;]|$)/i, + type: ValueOnly + }, + { + id: 'digitalocean-refresh-token', + regex: /\b(dor_v1_[a-f0-9]{64})(?:['"\s\x60;]|$)/i, + type: ValueOnly + }, + { + id: 'discord-api-token', + regex: /(?:discord)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-f0-9]{64})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'discord-client-id', + regex: /(?:discord)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([0-9]{18})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'discord-client-secret', + regex: /(?:discord)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9=_-]{32})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'doppler-api-token', + regex: /(dp\.pt\.)[a-z0-9]{43}/i, + type: ValueOnly + }, + { + id: 'droneci-access-token', + regex: /(?:droneci)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{32})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'dropbox-api-token', + regex: /(?:dropbox)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{15})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'dropbox-long-lived-api-token', + regex: /(?:dropbox)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{11}(AAAAAAAAAA)[a-z0-9\-_=]{43})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'dropbox-short-lived-api-token', + regex: /(?:dropbox)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}(sl\.[a-z0-9\-=_]{135})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'duffel-api-token', + regex: /duffel_(test|live)_[a-z0-9_\-=]{43}/i, + type: ValueOnly + }, + { + id: 'dynatrace-api-token', + regex: /dt0c01\.[a-z0-9]{24}\.[a-z0-9]{64}/i, + type: ValueOnly + }, + { + id: 'easypost-api-token', + regex: /\bEZAK[a-z0-9]{54}/i, + type: ValueOnly + }, + { + id: 'etsy-access-token', + regex: /(?:etsy)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{24})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'facebook', + regex: /(?:facebook)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-f0-9]{32})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'fastly-api-token', + regex: /(?:fastly)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9=_-]{32})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'finicity-api-token', + regex: /(?:finicity)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-f0-9]{32})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'finicity-client-secret', + regex: /(?:finicity)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{20})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'finnhub-access-token', + regex: /(?:finnhub)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{20})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'flickr-access-token', + regex: /(?:flickr)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{32})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'flutterwave-public-key', + regex: /FLWPUBK_TEST-[a-h0-9]{32}-X/i, + type: ValueOnly + }, + { + id: 'frameio-api-token', + regex: /fio-u-[a-z0-9\-_=]{64}/i, + type: ValueOnly + }, + { + id: 'freshbooks-access-token', + regex: /(?:freshbooks)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{64})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'gcp-api-key', + regex: /\b(AIza[0-9a-z\-_]{35})(?:['"\s\x60;]|$)/i, + type: ValueOnly + }, + { + id: 'github-app-token', + regex: /(ghu|ghs)_[0-9a-zA-Z]{36}/, + type: ValueOnly + }, + { + id: 'github-fine-grained-pat', + regex: /github_pat_[0-9a-zA-Z_]{82}/, + type: ValueOnly + }, + { + id: 'github-oauth', + regex: /gho_[0-9a-zA-Z]{36}/, + type: ValueOnly + }, + { + id: 'github-pat', + regex: /ghp_[0-9a-zA-Z]{36}/, + type: ValueOnly + }, + { + id: 'gitlab-pat', + regex: /glpat-[0-9a-zA-Z\-_]{20}/, + type: ValueOnly + }, + { + id: 'gitlab-ptt', + regex: /glptt-[0-9a-f]{40}/, + type: ValueOnly + }, + { + id: 'gitlab-rrt', + regex: /GR1348941[0-9a-zA-Z\-_]{20}/, + type: ValueOnly + }, + { + id: 'gitter-access-token', + regex: /(?:gitter)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9_-]{40})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'gocardless-api-token', + regex: /(?:gocardless)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}(live_[a-z0-9\-_=]{40})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'grafana-api-key', + regex: /\b(eyJrIjoi[a-z0-9]{70,400}={0,2})(?:['"\s\x60;]|$)/i, + type: ValueOnly + }, + { + id: 'grafana-cloud-api-token', + regex: /\b(glc_[a-z0-9+/]{32,400}={0,2})(?:['"\s\x60;]|$)/i, + type: ValueOnly + }, + { + id: 'grafana-service-account-token', + regex: /\b(glsa_[a-z0-9]{32}_[a-f0-9]{8})(?:['"\s\x60;]|$)/i, + type: ValueOnly + }, + { + id: 'hashicorp-tf-api-token', + regex: /[a-z0-9]{14}\.atlasv1\.[a-z0-9\-_=]{60,70}/i, + type: ValueOnly + }, + { + id: 'heroku-api-key', + regex: /(?:heroku)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'hubspot-api-key', + regex: /(?:hubspot)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'intercom-api-key', + regex: /(?:intercom)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9=_-]{60})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'jfrog-api-key', + regex: /(?:jfrog|artifactory|bintray|xray)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{73})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'jwt', + regex: /\b(ey[a-zA-Z0-9]{17,}\.ey[a-zA-Z0-9/_-]{17,}\.(?:[a-zA-Z0-9/_-]{10,}={0,2})?)(?:['"\s\x60;]|$)/, + type: ValueOnly + }, + { + id: 'kraken-access-token', + regex: /(?:kraken)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9/=_+-]{80,90})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'kucoin-access-token', + regex: /(?:kucoin)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-f0-9]{24})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'launchdarkly-access-token', + regex: /(?:launchdarkly)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9=_-]{40})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'linear-api-key', + regex: /lin_api_[a-z0-9]{40}/i, + type: ValueOnly + }, + { + id: 'linkedin-client-secret', + regex: /(?:linkedin|linked-in)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{16})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'lob-pub-api-key', + regex: /(?:lob)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}((test|live)_pub_[a-f0-9]{31})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'mailchimp-api-key', + regex: /(?:mailchimp)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-f0-9]{32}-us20)(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'mailgun-private-api-token', + regex: /(?:mailgun)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}(key-[a-f0-9]{32})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'mailgun-pub-key', + regex: /(?:mailgun)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}(pubkey-[a-f0-9]{32})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'mailgun-signing-key', + regex: /(?:mailgun)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-h0-9]{32}-[a-h0-9]{8}-[a-h0-9]{8})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'mapbox-api-token', + regex: /(?:mapbox)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}(pk\.[a-z0-9]{60}\.[a-z0-9]{22})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'mattermost-access-token', + regex: /(?:mattermost)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{26})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'messagebird-api-token', + regex: /(?:messagebird|message-bird|message_bird)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{25})(?:['"\s\x60;]|$)/i, + type: NameAndValue + }, + { + id: 'netlify-access-token', + regex: /(?:netlify)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9=_-]{40,46})(?:['"\s\x60;]|$)/i, + type: NameAndValue }, { - 'id': 'age-secret-key', - 'regex': /AGE-SECRET-KEY-1[QPZRY9X8GF2TVDW0S3JN54KHCE6MUA7L]{58}/ + id: 'new-relic-browser-api-token', + regex: /(?:new-relic|newrelic|new_relic)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}(NRJS-[a-f0-9]{19})(?:['"\s\x60;]|$)/i, + type: NameAndValue }, { - 'id': 'alibaba-access-key-id', - 'regex': /\b((LTAI)[a-z0-9]{20})(?:['"\s\x60;]|$)/i + id: 'new-relic-user-api-id', + regex: /(?:new-relic|newrelic|new_relic)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{64})(?:['"\s\x60;]|$)/i, + type: NameAndValue }, { - 'id': 'authress-service-client-access-key', - 'regex': /\b((?:sc|ext|scauth|authress)_[a-z0-9]{5,30}\.[a-z0-9]{4,6}\.acc[_-][a-z0-9-]{10,32}\.[a-z0-9+/_=-]{30,120})(?:['"\s\x60;]|$)/i + id: 'new-relic-user-api-key', + regex: /(?:new-relic|newrelic|new_relic)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}(NRAK-[a-z0-9]{27})(?:['"\s\x60;]|$)/i, + type: NameAndValue }, { - 'id': 'aws-access-token', - 'regex': /\b((A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16})(?:['"\s\x60;]|$)/ + id: 'npm-access-token', + regex: /\b(npm_[a-z0-9]{36})(?:['"\s\x60;]|$)/i, + type: ValueOnly }, { - 'id': 'clojars-api-token', - 'regex': /(CLOJARS_)[a-z0-9]{60}/i + id: 'nytimes-access-token', + regex: /(?:nytimes|new-york-times,|newyorktimes)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9=_-]{32})(?:['"\s\x60;]|$)/i, + type: NameAndValue }, { - 'id': 'databricks-api-token', - 'regex': /\b(dapi[a-h0-9]{32})(?:['"\s\x60;]|$)/i + id: 'okta-access-token', + regex: /(?:okta)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9=_-]{42})(?:['"\s\x60;]|$)/i, + type: NameAndValue }, { - 'id': 'digitalocean-access-token', - 'regex': /\b(doo_v1_[a-f0-9]{64})(?:['"\s\x60;]|$)/i + id: 'openai-api-key', + regex: /\b(sk-[a-z0-9]{20}T3BlbkFJ[a-z0-9]{20})(?:['"\s\x60;]|$)/i, + type: ValueOnly }, { - 'id': 'digitalocean-pat', - 'regex': /\b(dop_v1_[a-f0-9]{64})(?:['"\s\x60;]|$)/i + id: 'plaid-api-token', + regex: /(?:plaid)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}(access-(?:sandbox|development|production)-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:['"\s\x60;]|$)/i, + type: NameAndValue }, { - 'id': 'digitalocean-refresh-token', - 'regex': /\b(dor_v1_[a-f0-9]{64})(?:['"\s\x60;]|$)/i + id: 'plaid-client-id', + regex: /(?:plaid)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{24})(?:['"\s\x60;]|$)/i, + type: NameAndValue }, { - 'id': 'doppler-api-token', - 'regex': /(dp\.pt\.)[a-z0-9]{43}/i + id: 'plaid-secret-key', + regex: /(?:plaid)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{30})(?:['"\s\x60;]|$)/i, + type: NameAndValue }, { - 'id': 'duffel-api-token', - 'regex': /duffel_(test|live)_[a-z0-9_\-=]{43}/i + id: 'planetscale-api-token', + regex: /\b(pscale_tkn_[a-z0-9=\-_.]{32,64})(?:['"\s\x60;]|$)/i, + type: ValueOnly }, { - 'id': 'dynatrace-api-token', - 'regex': /dt0c01\.[a-z0-9]{24}\.[a-z0-9]{64}/i + id: 'planetscale-oauth-token', + regex: /\b(pscale_oauth_[a-z0-9=\-_.]{32,64})(?:['"\s\x60;]|$)/i, + type: ValueOnly }, { - 'id': 'easypost-api-token', - 'regex': /\bEZAK[a-z0-9]{54}/i + id: 'planetscale-password', + regex: /\b(pscale_pw_[a-z0-9=\-_.]{32,64})(?:['"\s\x60;]|$)/i, + type: ValueOnly }, { - 'id': 'flutterwave-public-key', - 'regex': /FLWPUBK_TEST-[a-h0-9]{32}-X/i + id: 'postman-api-token', + regex: /\b(PMAK-[a-f0-9]{24}-[a-f0-9]{34})(?:['"\s\x60;]|$)/i, + type: ValueOnly }, { - 'id': 'frameio-api-token', - 'regex': /fio-u-[a-z0-9\-_=]{64}/i + id: 'prefect-api-token', + regex: /\b(pnu_[a-z0-9]{36})(?:['"\s\x60;]|$)/i, + type: ValueOnly }, { - 'id': 'gcp-api-key', - 'regex': /\b(AIza[0-9a-z\-_]{35})(?:['"\s\x60;]|$)/i + id: 'private-key', + regex: /-----BEGIN[ A-Z0-9_-]{0,100}PRIVATE KEY( BLOCK)?-----[\s\S]*KEY( BLOCK)?----/i, + type: ValueOnly }, { - 'id': 'github-app-token', - 'regex': /(ghu|ghs)_[0-9a-zA-Z]{36}/ + id: 'pulumi-api-token', + regex: /\b(pul-[a-f0-9]{40})(?:['"\s\x60;]|$)/i, + type: ValueOnly }, { - 'id': 'github-fine-grained-pat', - 'regex': /github_pat_[0-9a-zA-Z_]{82}/ + id: 'pypi-upload-token', + regex: /pypi-AgEIcHlwaS5vcmc[A-Za-z0-9\-_]{50,1000}/, + type: ValueOnly }, { - 'id': 'github-oauth', - 'regex': /gho_[0-9a-zA-Z]{36}/ + id: 'rapidapi-access-token', + regex: /(?:rapidapi)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9_-]{50})(?:['"\s\x60;]|$)/i, + type: NameAndValue }, { - 'id': 'github-pat', - 'regex': /ghp_[0-9a-zA-Z]{36}/ + id: 'readme-api-token', + regex: /\b(rdme_[a-z0-9]{70})(?:['"\s\x60;]|$)/i, + type: ValueOnly }, { - 'id': 'gitlab-pat', - 'regex': /glpat-[0-9a-zA-Z\-_]{20}/ + id: 'rubygems-api-token', + regex: /\b(rubygems_[a-f0-9]{48})(?:['"\s\x60;]|$)/i, + type: ValueOnly }, { - 'id': 'gitlab-ptt', - 'regex': /glptt-[0-9a-f]{40}/ + id: 'scalingo-api-token', + regex: /tk-us-[a-zA-Z0-9-_]{48}/, + type: ValueOnly }, { - 'id': 'gitlab-rrt', - 'regex': /GR1348941[0-9a-zA-Z\-_]{20}/ + id: 'sendbird-access-id', + regex: /(?:sendbird)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:['"\s\x60;]|$)/i, + type: NameAndValue }, { - 'id': 'grafana-api-key', - 'regex': /\b(eyJrIjoi[a-z0-9]{70,400}={0,2})(?:['"\s\x60;]|$)/i + id: 'sendbird-access-token', + regex: /(?:sendbird)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-f0-9]{40})(?:['"\s\x60;]|$)/i, + type: NameAndValue }, { - 'id': 'grafana-cloud-api-token', - 'regex': /\b(glc_[a-z0-9+/]{32,400}={0,2})(?:['"\s\x60;]|$)/i + id: 'sendgrid-api-token', + regex: /\b(SG\.[a-z0-9=_\-.]{66})(?:['"\s\x60;]|$)/i, + type: ValueOnly }, { - 'id': 'grafana-service-account-token', - 'regex': /\b(glsa_[a-z0-9]{32}_[a-f0-9]{8})(?:['"\s\x60;]|$)/i + id: 'sendinblue-api-token', + regex: /\b(xkeysib-[a-f0-9]{64}-[a-z0-9]{16})(?:['"\s\x60;]|$)/i, + type: ValueOnly }, { - 'id': 'hashicorp-tf-api-token', - 'regex': /[a-z0-9]{14}\.atlasv1\.[a-z0-9\-_=]{60,70}/i + id: 'sentry-access-token', + regex: /(?:sentry)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-f0-9]{64})(?:['"\s\x60;]|$)/i, + type: NameAndValue }, { - 'id': 'jwt', - 'regex': /\b(ey[a-zA-Z0-9]{17,}\.ey[a-zA-Z0-9/_-]{17,}\.(?:[a-zA-Z0-9/_-]{10,}={0,2})?)(?:['"\s\x60;]|$)/ + id: 'shippo-api-token', + regex: /\b(shippo_(live|test)_[a-f0-9]{40})(?:['"\s\x60;]|$)/i, + type: ValueOnly }, { - 'id': 'linear-api-key', - 'regex': /lin_api_[a-z0-9]{40}/i + id: 'shopify-access-token', + regex: /shpat_[a-fA-F0-9]{32}/, + type: ValueOnly }, { - 'id': 'npm-access-token', - 'regex': /\b(npm_[a-z0-9]{36})(?:['"\s\x60;]|$)/i + id: 'shopify-custom-access-token', + regex: /shpca_[a-fA-F0-9]{32}/, + type: ValueOnly }, { - 'id': 'openai-api-key', - 'regex': /\b(sk-[a-z0-9]{20}T3BlbkFJ[a-z0-9]{20})(?:['"\s\x60;]|$)/i + id: 'shopify-private-app-access-token', + regex: /shppa_[a-fA-F0-9]{32}/, + type: ValueOnly }, { - 'id': 'planetscale-api-token', - 'regex': /\b(pscale_tkn_[a-z0-9=\-_.]{32,64})(?:['"\s\x60;]|$)/i + id: 'shopify-shared-secret', + regex: /shpss_[a-fA-F0-9]{32}/, + type: ValueOnly }, { - 'id': 'planetscale-oauth-token', - 'regex': /\b(pscale_oauth_[a-z0-9=\-_.]{32,64})(?:['"\s\x60;]|$)/i + id: 'sidekiq-secret', + regex: /(?:BUNDLE_ENTERPRISE__CONTRIBSYS__COM|BUNDLE_GEMS__CONTRIBSYS__COM)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-f0-9]{8}:[a-f0-9]{8})(?:['"\s\x60;]|$)/i, + type: NameAndValue }, { - 'id': 'planetscale-password', - 'regex': /\b(pscale_pw_[a-z0-9=\-_.]{32,64})(?:['"\s\x60;]|$)/i + id: 'slack-app-token', + regex: /(xapp-\d-[A-Z0-9]+-\d+-[a-z0-9]+)/i, + type: ValueOnly }, { - 'id': 'postman-api-token', - 'regex': /\b(PMAK-[a-f0-9]{24}-[a-f0-9]{34})(?:['"\s\x60;]|$)/i + id: 'slack-bot-token', + regex: /(xoxb-[0-9]{10,13}-[0-9]{10,13}[a-zA-Z0-9-]*)/, + type: ValueOnly }, { - 'id': 'prefect-api-token', - 'regex': /\b(pnu_[a-z0-9]{36})(?:['"\s\x60;]|$)/i + id: 'slack-config-access-token', + regex: /(xoxe.xox[bp]-\d-[A-Z0-9]{163,166})/i, + type: ValueOnly }, { - 'id': 'private-key', - 'regex': /-----BEGIN[ A-Z0-9_-]{0,100}PRIVATE KEY( BLOCK)?-----[\s\S]*KEY( BLOCK)?----/i + id: 'slack-config-refresh-token', + regex: /(xoxe-\d-[A-Z0-9]{146})/i, + type: ValueOnly }, { - 'id': 'pulumi-api-token', - 'regex': /\b(pul-[a-f0-9]{40})(?:['"\s\x60;]|$)/i + id: 'slack-legacy-bot-token', + regex: /(xoxb-[0-9]{8,14}-[a-zA-Z0-9]{18,26})/, + type: ValueOnly }, { - 'id': 'pypi-upload-token', - 'regex': /pypi-AgEIcHlwaS5vcmc[A-Za-z0-9\-_]{50,1000}/ + id: 'slack-legacy-token', + regex: /(xox[os]-\d+-\d+-\d+-[a-fA-F\d]+)/, + type: ValueOnly }, { - 'id': 'readme-api-token', - 'regex': /\b(rdme_[a-z0-9]{70})(?:['"\s\x60;]|$)/i + id: 'slack-legacy-workspace-token', + regex: /(xox[ar]-(?:\d-)?[0-9a-zA-Z]{8,48})/, + type: ValueOnly }, { - 'id': 'rubygems-api-token', - 'regex': /\b(rubygems_[a-f0-9]{48})(?:['"\s\x60;]|$)/i + id: 'slack-user-token', + regex: /(xox[pe](?:-[0-9]{10,13}){3}-[a-zA-Z0-9-]{28,34})/, + type: ValueOnly }, { - 'id': 'scalingo-api-token', - 'regex': /tk-us-[a-zA-Z0-9-_]{48}/ + id: 'slack-webhook-url', + regex: /(https?:\/\/)?hooks.slack.com\/(services|workflows)\/[A-Za-z0-9+/]{43,46}/, + type: ValueOnly }, { - 'id': 'sendgrid-api-token', - 'regex': /\b(SG\.[a-z0-9=_\-.]{66})(?:['"\s\x60;]|$)/i + id: 'snyk-api-token', + regex: /(?:snyk)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:['"\s\x60;]|$)/i, + type: NameAndValue }, { - 'id': 'sendinblue-api-token', - 'regex': /\b(xkeysib-[a-f0-9]{64}-[a-z0-9]{16})(?:['"\s\x60;]|$)/i + id: 'square-access-token', + regex: /\b(sq0atp-[0-9a-z\-_]{22})(?:['"\s\x60;]|$)/i, + type: ValueOnly }, { - 'id': 'shippo-api-token', - 'regex': /\b(shippo_(live|test)_[a-f0-9]{40})(?:['"\s\x60;]|$)/i + id: 'square-secret', + regex: /\b(sq0csp-[0-9a-z\-_]{43})(?:['"\s\x60;]|$)/i, + type: ValueOnly }, { - 'id': 'shopify-access-token', - 'regex': /shpat_[a-fA-F0-9]{32}/ + id: 'squarespace-access-token', + regex: /(?:squarespace)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:['"\s\x60;]|$)/i, + type: NameAndValue }, { - 'id': 'shopify-custom-access-token', - 'regex': /shpca_[a-fA-F0-9]{32}/ + id: 'stripe-access-token', + regex: /(sk|pk)_(test|live)_[0-9a-z]{10,32}/i, + type: ValueOnly }, { - 'id': 'shopify-private-app-access-token', - 'regex': /shppa_[a-fA-F0-9]{32}/ + id: 'sumologic-access-token', + regex: /(?:sumo)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{64})(?:['"\s\x60;]|$)/i, + type: NameAndValue }, { - 'id': 'shopify-shared-secret', - 'regex': /shpss_[a-fA-F0-9]{32}/ + id: 'telegram-bot-api-token', + regex: /(?:^|[^0-9])([0-9]{5,16}:A[a-z0-9_-]{34})(?:$|[^a-z0-9_-])/i, + type: ValueOnly }, { - 'id': 'slack-app-token', - 'regex': /(xapp-\d-[A-Z0-9]+-\d+-[a-z0-9]+)/i + id: 'travisci-access-token', + regex: /(?:travis)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{22})(?:['"\s\x60;]|$)/i, + type: NameAndValue }, { - 'id': 'slack-bot-token', - 'regex': /(xoxb-[0-9]{10,13}-[0-9]{10,13}[a-zA-Z0-9-]*)/ + id: 'trello-access-token', + regex: /(?:trello)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z-0-9]{32})(?:['"\s\x60;]|$)/i, + type: NameAndValue }, { - 'id': 'slack-config-access-token', - 'regex': /(xoxe.xox[bp]-\d-[A-Z0-9]{163,166})/i + id: 'twilio-api-key', + regex: /SK[0-9a-fA-F]{32}/, + type: ValueOnly }, { - 'id': 'slack-config-refresh-token', - 'regex': /(xoxe-\d-[A-Z0-9]{146})/i + id: 'twitch-api-token', + regex: /(?:twitch)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{30})(?:['"\s\x60;]|$)/i, + type: NameAndValue }, { - 'id': 'slack-legacy-bot-token', - 'regex': /(xoxb-[0-9]{8,14}-[a-zA-Z0-9]{18,26})/ + id: 'twitter-access-secret', + regex: /(?:twitter)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{45})(?:['"\s\x60;]|$)/i, + type: NameAndValue }, { - 'id': 'slack-legacy-token', - 'regex': /(xox[os]-\d+-\d+-\d+-[a-fA-F\d]+)/ + id: 'twitter-access-token', + regex: /(?:twitter)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([0-9]{15,25}-[a-z0-9]{20,40})(?:['"\s\x60;]|$)/i, + type: NameAndValue }, { - 'id': 'slack-legacy-workspace-token', - 'regex': /(xox[ar]-(?:\d-)?[0-9a-zA-Z]{8,48})/ + id: 'twitter-api-key', + regex: /(?:twitter)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{25})(?:['"\s\x60;]|$)/i, + type: NameAndValue }, { - 'id': 'slack-user-token', - 'regex': /(xox[pe](?:-[0-9]{10,13}){3}-[a-zA-Z0-9-]{28,34})/ + id: 'twitter-api-secret', + regex: /(?:twitter)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{50})(?:['"\s\x60;]|$)/i, + type: NameAndValue }, { - 'id': 'slack-webhook-url', - 'regex': /(https?:\/\/)?hooks.slack.com\/(services|workflows)\/[A-Za-z0-9+/]{43,46}/ + id: 'twitter-bearer-token', + regex: /(?:twitter)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}(A{22}[a-z0-9%]{80,100})(?:['"\s\x60;]|$)/i, + type: NameAndValue }, { - 'id': 'square-access-token', - 'regex': /\b(sq0atp-[0-9a-z\-_]{22})(?:['"\s\x60;]|$)/i + id: 'typeform-api-token', + regex: /(?:typeform)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}(tfp_[a-z0-9\-_.=]{59})(?:['"\s\x60;]|$)/i, + type: NameAndValue }, { - 'id': 'square-secret', - 'regex': /\b(sq0csp-[0-9a-z\-_]{43})(?:['"\s\x60;]|$)/i + id: 'vault-batch-token', + regex: /\b(hvb\.[a-z0-9_-]{138,212})(?:['"\s\x60;]|$)/i, + type: ValueOnly }, { - 'id': 'stripe-access-token', - 'regex': /(sk|pk)_(test|live)_[0-9a-z]{10,32}/i + id: 'vault-service-token', + regex: /\b(hvs\.[a-z0-9_-]{90,100})(?:['"\s\x60;]|$)/i, + type: ValueOnly }, { - 'id': 'telegram-bot-api-token', - 'regex': /(?:^|[^0-9])([0-9]{5,16}:A[a-z0-9_-]{34})(?:$|[^a-z0-9_-])/i + id: 'yandex-access-token', + regex: /(?:yandex)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}(t1\.[A-Z0-9a-z_-]+[=]{0,2}\.[A-Z0-9a-z_-]{86}[=]{0,2})(?:['"\s\x60;]|$)/i, + type: NameAndValue }, { - 'id': 'twilio-api-key', - 'regex': /SK[0-9a-fA-F]{32}/ + id: 'yandex-api-key', + regex: /(?:yandex)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}(AQVN[a-z0-9_-]{35,38})(?:['"\s\x60;]|$)/i, + type: NameAndValue }, { - 'id': 'vault-batch-token', - 'regex': /\b(hvb\.[a-z0-9_-]{138,212})(?:['"\s\x60;]|$)/i + id: 'yandex-aws-access-token', + regex: /(?:yandex)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}(YC[a-z0-9_-]{38})(?:['"\s\x60;]|$)/i, + type: NameAndValue }, { - 'id': 'vault-service-token', - 'regex': /\b(hvs\.[a-z0-9_-]{90,100})(?:['"\s\x60;]|$)/i + id: 'zendesk-secret-key', + regex: /(?:zendesk)(?:[0-9a-z\-_\t.]{0,20})(?:[\s|']|[\s|""]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|""|\s|=|\x60){0,5}([a-z0-9]{40})(?:['"\s\x60;]|$)/i, + type: NameAndValue } ] diff --git a/packages/dd-trace/src/appsec/iast/analyzers/header-injection-analyzer.js b/packages/dd-trace/src/appsec/iast/analyzers/header-injection-analyzer.js index 73ac404f5a5..62330e87a07 100644 --- a/packages/dd-trace/src/appsec/iast/analyzers/header-injection-analyzer.js +++ b/packages/dd-trace/src/appsec/iast/analyzers/header-injection-analyzer.js @@ -44,9 +44,14 @@ class HeaderInjectionAnalyzer extends InjectionAnalyzer { if (this.isExcludedHeaderName(lowerCasedHeaderName) || typeof value !== 'string') return - return super._isVulnerable(value, iastContext) && - !(this.isCookieExclusion(lowerCasedHeaderName, value, iastContext) || - this.isAccessControlAllowOriginExclusion(lowerCasedHeaderName, value, iastContext)) + const ranges = getRanges(iastContext, value) + if (ranges?.length > 0) { + return !(this.isCookieExclusion(lowerCasedHeaderName, ranges) || + this.isSameHeaderExclusion(lowerCasedHeaderName, ranges) || + this.isAccessControlAllowExclusion(lowerCasedHeaderName, ranges)) + } + + return false } _getEvidence (headerInfo, iastContext) { @@ -70,24 +75,28 @@ class HeaderInjectionAnalyzer extends InjectionAnalyzer { return EXCLUDED_HEADER_NAMES.includes(name) } - isCookieExclusion (name, value, iastContext) { + isCookieExclusion (name, ranges) { if (name === 'set-cookie') { - return getRanges(iastContext, value) + return ranges .every(range => range.iinfo.type === HTTP_REQUEST_COOKIE_VALUE || range.iinfo.type === HTTP_REQUEST_COOKIE_NAME) } return false } - isAccessControlAllowOriginExclusion (name, value, iastContext) { - if (name === 'access-control-allow-origin') { - return getRanges(iastContext, value) + isAccessControlAllowExclusion (name, ranges) { + if (name?.startsWith('access-control-allow-')) { + return ranges .every(range => range.iinfo.type === HTTP_REQUEST_HEADER_VALUE) } return false } + isSameHeaderExclusion (name, ranges) { + return ranges.length === 1 && name === ranges[0].iinfo.parameterName?.toLowerCase() + } + _getExcludedPaths () { return EXCLUDED_PATHS } diff --git a/packages/dd-trace/src/appsec/iast/analyzers/hsts-header-missing-analyzer.js b/packages/dd-trace/src/appsec/iast/analyzers/hsts-header-missing-analyzer.js index a5196d3c92f..c5bbd19be99 100644 --- a/packages/dd-trace/src/appsec/iast/analyzers/hsts-header-missing-analyzer.js +++ b/packages/dd-trace/src/appsec/iast/analyzers/hsts-header-missing-analyzer.js @@ -9,6 +9,7 @@ class HstsHeaderMissingAnalyzer extends MissingHeaderAnalyzer { constructor () { super(HSTS_HEADER_MISSING, HSTS_HEADER_NAME) } + _isVulnerableFromRequestAndResponse (req, res) { const headerValues = this._getHeaderValues(res, HSTS_HEADER_NAME) return this._isHttpsProtocol(req) && ( diff --git a/packages/dd-trace/src/appsec/iast/analyzers/nosql-injection-mongodb-analyzer.js b/packages/dd-trace/src/appsec/iast/analyzers/nosql-injection-mongodb-analyzer.js index 83758045ece..e6d4ef3aa74 100644 --- a/packages/dd-trace/src/appsec/iast/analyzers/nosql-injection-mongodb-analyzer.js +++ b/packages/dd-trace/src/appsec/iast/analyzers/nosql-injection-mongodb-analyzer.js @@ -9,11 +9,11 @@ const { storage } = require('../../../../../datadog-core') const { getIastContext } = require('../iast-context') const { HTTP_REQUEST_PARAMETER, HTTP_REQUEST_BODY } = require('../taint-tracking/source-types') -const EXCLUDED_PATHS_FROM_STACK = getNodeModulesPaths('mongodb', 'mongoose') +const EXCLUDED_PATHS_FROM_STACK = getNodeModulesPaths('mongodb', 'mongoose', 'mquery') const MONGODB_NOSQL_SECURE_MARK = getNextSecureMark() -function iterateObjectStrings (target, fn, levelKeys = [], depth = 50, visited = new Set()) { - if (target && typeof target === 'object') { +function iterateObjectStrings (target, fn, levelKeys = [], depth = 20, visited = new Set()) { + if (target !== null && typeof target === 'object') { Object.keys(target).forEach((key) => { const nextLevelKeys = [...levelKeys, key] const val = target[key] @@ -37,34 +37,39 @@ class NosqlInjectionMongodbAnalyzer extends InjectionAnalyzer { onConfigure () { this.configureSanitizers() - this.addSub('datadog:mongodb:collection:filter:start', ({ filters }) => { + const onStart = ({ filters }) => { const store = storage.getStore() if (store && !store.nosqlAnalyzed && filters?.length) { filters.forEach(filter => { this.analyze({ filter }, store) }) } - }) - this.addSub('datadog:mongoose:model:filter:start', ({ filters }) => { - const store = storage.getStore() - if (!store) return + return store + } - if (filters?.length) { - filters.forEach(filter => { - this.analyze({ filter }, store) - }) + const onStartAndEnterWithStore = (message) => { + const store = onStart(message || {}) + if (store) { + storage.enterWith({ ...store, nosqlAnalyzed: true, nosqlParentStore: store }) } + } - storage.enterWith({ ...store, nosqlAnalyzed: true, mongooseParentStore: store }) - }) - - this.addSub('datadog:mongoose:model:filter:finish', () => { + const onFinish = () => { const store = storage.getStore() - if (store?.mongooseParentStore) { - storage.enterWith(store.mongooseParentStore) + if (store?.nosqlParentStore) { + storage.enterWith(store.nosqlParentStore) } - }) + } + + this.addSub('datadog:mongodb:collection:filter:start', onStart) + + this.addSub('datadog:mongoose:model:filter:start', onStartAndEnterWithStore) + this.addSub('datadog:mongoose:model:filter:finish', onFinish) + + this.addSub('datadog:mquery:filter:prepare', onStart) + this.addSub('tracing:datadog:mquery:filter:start', onStartAndEnterWithStore) + this.addSub('tracing:datadog:mquery:filter:asyncEnd', onFinish) } configureSanitizers () { diff --git a/packages/dd-trace/src/appsec/iast/analyzers/path-traversal-analyzer.js b/packages/dd-trace/src/appsec/iast/analyzers/path-traversal-analyzer.js index 83bf2a87085..625dbde9150 100644 --- a/packages/dd-trace/src/appsec/iast/analyzers/path-traversal-analyzer.js +++ b/packages/dd-trace/src/appsec/iast/analyzers/path-traversal-analyzer.js @@ -29,7 +29,14 @@ class PathTraversalAnalyzer extends InjectionAnalyzer { onConfigure () { this.addSub('apm:fs:operation:start', (obj) => { - if (ignoredOperations.includes(obj.operation)) return + const store = storage.getStore() + const outOfReqOrChild = !store?.fs?.root + + // we could filter out all the nested fs.operations based on store.fs.root + // but if we spect a store in the context to be present we are going to exclude + // all out_of_the_request fs.operations + // AppsecFsPlugin must be enabled + if (ignoredOperations.includes(obj.operation) || outOfReqOrChild) return const pathArguments = [] if (obj.dest) { diff --git a/packages/dd-trace/src/appsec/iast/analyzers/sql-injection-analyzer.js b/packages/dd-trace/src/appsec/iast/analyzers/sql-injection-analyzer.js index a857839e175..4d302ece1b6 100644 --- a/packages/dd-trace/src/appsec/iast/analyzers/sql-injection-analyzer.js +++ b/packages/dd-trace/src/appsec/iast/analyzers/sql-injection-analyzer.js @@ -4,8 +4,6 @@ const InjectionAnalyzer = require('./injection-analyzer') const { SQL_INJECTION } = require('../vulnerabilities') const { getRanges } = require('../taint-tracking/operations') const { storage } = require('../../../../../datadog-core') -const { getIastContext } = require('../iast-context') -const { addVulnerability } = require('../vulnerability-reporter') const { getNodeModulesPaths } = require('../path-line') const EXCLUDED_PATHS = getNodeModulesPaths('mysql', 'mysql2', 'sequelize', 'pg-pool', 'knex') @@ -16,9 +14,9 @@ class SqlInjectionAnalyzer extends InjectionAnalyzer { } onConfigure () { - this.addSub('apm:mysql:query:start', ({ sql }) => this.analyze(sql, 'MYSQL')) - this.addSub('apm:mysql2:query:start', ({ sql }) => this.analyze(sql, 'MYSQL')) - this.addSub('apm:pg:query:start', ({ query }) => this.analyze(query.text, 'POSTGRES')) + this.addSub('apm:mysql:query:start', ({ sql }) => this.analyze(sql, undefined, 'MYSQL')) + this.addSub('apm:mysql2:query:start', ({ sql }) => this.analyze(sql, undefined, 'MYSQL')) + this.addSub('apm:pg:query:start', ({ query }) => this.analyze(query.text, undefined, 'POSTGRES')) this.addSub( 'datadog:sequelize:query:start', @@ -42,7 +40,7 @@ class SqlInjectionAnalyzer extends InjectionAnalyzer { getStoreAndAnalyze (query, dialect) { const parentStore = storage.getStore() if (parentStore) { - this.analyze(query, dialect, parentStore) + this.analyze(query, parentStore, dialect) storage.enterWith({ ...parentStore, sqlAnalyzed: true, sqlParentStore: parentStore }) } @@ -60,29 +58,10 @@ class SqlInjectionAnalyzer extends InjectionAnalyzer { return { value, ranges, dialect } } - analyze (value, dialect, store = storage.getStore()) { + analyze (value, store, dialect) { + store = store || storage.getStore() if (!(store && store.sqlAnalyzed)) { - const iastContext = getIastContext(store) - if (this._isInvalidContext(store, iastContext)) return - this._reportIfVulnerable(value, iastContext, dialect) - } - } - - _reportIfVulnerable (value, context, dialect) { - if (this._isVulnerable(value, context) && this._checkOCE(context)) { - this._report(value, context, dialect) - return true - } - return false - } - - _report (value, context, dialect) { - const evidence = this._getEvidence(value, context, dialect) - const location = this._getLocation() - if (!this._isExcluded(location)) { - const spanId = context && context.rootSpan && context.rootSpan.context().toSpanId() - const vulnerability = this._createVulnerability(this._type, evidence, spanId, location) - addVulnerability(context, vulnerability) + super.analyze(value, store, dialect) } } diff --git a/packages/dd-trace/src/appsec/iast/analyzers/vulnerability-analyzer.js b/packages/dd-trace/src/appsec/iast/analyzers/vulnerability-analyzer.js index 1f52790300d..f79e7a44f71 100644 --- a/packages/dd-trace/src/appsec/iast/analyzers/vulnerability-analyzer.js +++ b/packages/dd-trace/src/appsec/iast/analyzers/vulnerability-analyzer.js @@ -22,8 +22,12 @@ class Analyzer extends SinkIastPlugin { return false } - _report (value, context) { - const evidence = this._getEvidence(value, context) + _report (value, context, meta) { + const evidence = this._getEvidence(value, context, meta) + this._reportEvidence(value, context, evidence) + } + + _reportEvidence (value, context, evidence) { const location = this._getLocation(value) if (!this._isExcluded(location)) { const locationSourceMap = this._replaceLocationFromSourceMap(location) @@ -33,9 +37,9 @@ class Analyzer extends SinkIastPlugin { } } - _reportIfVulnerable (value, context) { + _reportIfVulnerable (value, context, meta) { if (this._isVulnerable(value, context) && this._checkOCE(context, value)) { - this._report(value, context) + this._report(value, context, meta) return true } return false @@ -71,11 +75,11 @@ class Analyzer extends SinkIastPlugin { return store && !iastContext } - analyze (value, store = storage.getStore()) { + analyze (value, store = storage.getStore(), meta) { const iastContext = getIastContext(store) if (this._isInvalidContext(store, iastContext)) return - this._reportIfVulnerable(value, iastContext) + this._reportIfVulnerable(value, iastContext, meta) } analyzeAll (...values) { diff --git a/packages/dd-trace/src/appsec/iast/analyzers/weak-hash-analyzer.js b/packages/dd-trace/src/appsec/iast/analyzers/weak-hash-analyzer.js index 0cfb8252328..b7ae6681d00 100644 --- a/packages/dd-trace/src/appsec/iast/analyzers/weak-hash-analyzer.js +++ b/packages/dd-trace/src/appsec/iast/analyzers/weak-hash-analyzer.js @@ -20,11 +20,15 @@ const EXCLUDED_LOCATIONS = getNodeModulesPaths( 'pusher/lib/utils.js', 'redlock/dist/cjs', 'sqreen/lib/package-reader/index.js', - 'ws/lib/websocket-server.js' + 'ws/lib/websocket-server.js', + 'google-gax/build/src/grpc.js', + 'cookie-signature/index.js' ) const EXCLUDED_PATHS_FROM_STACK = [ - path.join('node_modules', 'object-hash', path.sep) + path.join('node_modules', 'object-hash', path.sep), + path.join('node_modules', 'aws-sdk', 'lib', 'util.js'), + path.join('node_modules', 'keygrip', path.sep) ] class WeakHashAnalyzer extends Analyzer { constructor () { @@ -43,6 +47,8 @@ class WeakHashAnalyzer extends Analyzer { } _isExcluded (location) { + if (!location) return false + return EXCLUDED_LOCATIONS.some(excludedLocation => { return location.path.includes(excludedLocation) }) diff --git a/packages/dd-trace/src/appsec/iast/analyzers/weak-randomness-analyzer.js b/packages/dd-trace/src/appsec/iast/analyzers/weak-randomness-analyzer.js new file mode 100644 index 00000000000..b150d172325 --- /dev/null +++ b/packages/dd-trace/src/appsec/iast/analyzers/weak-randomness-analyzer.js @@ -0,0 +1,19 @@ +'use strict' +const Analyzer = require('./vulnerability-analyzer') +const { WEAK_RANDOMNESS } = require('../vulnerabilities') + +class WeakRandomnessAnalyzer extends Analyzer { + constructor () { + super(WEAK_RANDOMNESS) + } + + onConfigure () { + this.addSub('datadog:random:call', ({ fn }) => this.analyze(fn)) + } + + _isVulnerable (fn) { + return fn === Math.random + } +} + +module.exports = new WeakRandomnessAnalyzer() diff --git a/packages/dd-trace/src/appsec/iast/context/context-plugin.js b/packages/dd-trace/src/appsec/iast/context/context-plugin.js new file mode 100644 index 00000000000..f074f1fd40f --- /dev/null +++ b/packages/dd-trace/src/appsec/iast/context/context-plugin.js @@ -0,0 +1,90 @@ +'use strict' + +const { storage } = require('../../../../../datadog-core') +const iastContextFunctions = require('../iast-context') +const overheadController = require('../overhead-controller') +const { IastPlugin } = require('../iast-plugin') +const { IAST_ENABLED_TAG_KEY } = require('../tags') +const { createTransaction, removeTransaction } = require('../taint-tracking/operations') +const vulnerabilityReporter = require('../vulnerability-reporter') +const { TagKey } = require('../telemetry/iast-metric') + +class IastContextPlugin extends IastPlugin { + startCtxOn (channelName, tag) { + super.addSub(channelName, (message) => this.startContext()) + + this._getAndRegisterSubscription({ + channelName, + tag, + tagKey: TagKey.SOURCE_TYPE + }) + } + + finishCtxOn (channelName) { + super.addSub(channelName, (message) => this.finishContext()) + } + + getRootSpan (store) { + return store?.span + } + + getTopContext () { + return {} + } + + newIastContext (rootSpan) { + return { rootSpan } + } + + addIastEnabledTag (isRequestAcquired, rootSpan) { + if (rootSpan?.addTags) { + rootSpan.addTags({ + [IAST_ENABLED_TAG_KEY]: isRequestAcquired ? 1 : 0 + }) + } + } + + startContext () { + let isRequestAcquired = false + let iastContext + + const store = storage.getStore() + if (store) { + const topContext = this.getTopContext() + const rootSpan = this.getRootSpan(store) + + isRequestAcquired = overheadController.acquireRequest(rootSpan) + if (isRequestAcquired) { + iastContext = iastContextFunctions.saveIastContext(store, topContext, this.newIastContext(rootSpan)) + createTransaction(rootSpan.context().toSpanId(), iastContext) + overheadController.initializeRequestContext(iastContext) + } + this.addIastEnabledTag(isRequestAcquired, rootSpan) + } + + return { + isRequestAcquired, + iastContext, + store + } + } + + finishContext () { + const store = storage.getStore() + if (store) { + const topContext = this.getTopContext() + const iastContext = iastContextFunctions.getIastContext(store, topContext) + const rootSpan = iastContext?.rootSpan + if (iastContext && rootSpan) { + vulnerabilityReporter.sendVulnerabilities(iastContext.vulnerabilities, rootSpan) + removeTransaction(iastContext) + } + + if (iastContextFunctions.cleanIastContext(store, topContext, iastContext)) { + overheadController.releaseRequest() + } + } + } +} + +module.exports = IastContextPlugin diff --git a/packages/dd-trace/src/appsec/iast/context/kafka-ctx-plugin.js b/packages/dd-trace/src/appsec/iast/context/kafka-ctx-plugin.js new file mode 100644 index 00000000000..a728082cad1 --- /dev/null +++ b/packages/dd-trace/src/appsec/iast/context/kafka-ctx-plugin.js @@ -0,0 +1,14 @@ +'use strict' + +const { KAFKA_MESSAGE_KEY, KAFKA_MESSAGE_VALUE } = require('../taint-tracking/source-types') +const IastContextPlugin = require('./context-plugin') + +class KafkaContextPlugin extends IastContextPlugin { + onConfigure () { + this.startCtxOn('dd-trace:kafkajs:consumer:afterStart', [KAFKA_MESSAGE_KEY, KAFKA_MESSAGE_VALUE]) + + this.finishCtxOn('dd-trace:kafkajs:consumer:beforeFinish') + } +} + +module.exports = new KafkaContextPlugin() diff --git a/packages/dd-trace/src/appsec/iast/iast-log.js b/packages/dd-trace/src/appsec/iast/iast-log.js index 25b33c3bf44..c126729f965 100644 --- a/packages/dd-trace/src/appsec/iast/iast-log.js +++ b/packages/dd-trace/src/appsec/iast/iast-log.js @@ -2,35 +2,9 @@ const dc = require('dc-polyfill') const log = require('../../log') -const { calculateDDBasePath } = require('../../util') const telemetryLog = dc.channel('datadog:telemetry:log') -const ddBasePath = calculateDDBasePath(__dirname) -const EOL = '\n' -const STACK_FRAME_LINE_REGEX = /^\s*at\s/gm - -function sanitize (logEntry, stack) { - if (!stack) return logEntry - - let stackLines = stack.split(EOL) - - const firstIndex = stackLines.findIndex(l => l.match(STACK_FRAME_LINE_REGEX)) - - const isDDCode = firstIndex > -1 && stackLines[firstIndex].includes(ddBasePath) - stackLines = stackLines - .filter((line, index) => (isDDCode && index < firstIndex) || line.includes(ddBasePath)) - .map(line => line.replace(ddBasePath, '')) - - logEntry.stack_trace = stackLines.join(EOL) - - if (!isDDCode) { - logEntry.message = 'omitted' - } - - return logEntry -} - function getTelemetryLog (data, level) { try { data = typeof data === 'function' ? data() : data @@ -42,18 +16,13 @@ function getTelemetryLog (data, level) { message = String(data.message || data) } - let logEntry = { + const logEntry = { message, level } - if (data.stack) { - logEntry = sanitize(logEntry, data.stack) - if (logEntry.stack_trace === '') { - return - } + logEntry.stack_trace = data.stack } - return logEntry } catch (e) { log.error(e) @@ -109,7 +78,8 @@ const iastLog = { errorAndPublish (data) { this.error(data) - return this.publish(data, 'ERROR') + // publish is done automatically by log.error() + return this } } diff --git a/packages/dd-trace/src/appsec/iast/iast-plugin.js b/packages/dd-trace/src/appsec/iast/iast-plugin.js index 2fe9f85bed6..5eb6e00410d 100644 --- a/packages/dd-trace/src/appsec/iast/iast-plugin.js +++ b/packages/dd-trace/src/appsec/iast/iast-plugin.js @@ -5,7 +5,8 @@ const { channel } = require('dc-polyfill') const iastLog = require('./iast-log') const Plugin = require('../../plugins/plugin') const iastTelemetry = require('./telemetry') -const { getInstrumentedMetric, getExecutedMetric, TagKey, EXECUTED_SOURCE } = require('./telemetry/iast-metric') +const { getInstrumentedMetric, getExecutedMetric, TagKey, EXECUTED_SOURCE, formatTags } = + require('./telemetry/iast-metric') const { storage } = require('../../../../datadog-core') const { getIastContext } = require('./iast-context') const instrumentations = require('../../../../datadog-instrumentations/src/helpers/instrumentations') @@ -20,25 +21,29 @@ const instrumentations = require('../../../../datadog-instrumentations/src/helpe * - tagKey can be only SOURCE_TYPE (Source) or VULNERABILITY_TYPE (Sink) */ class IastPluginSubscription { - constructor (moduleName, channelName, tag, tagKey = TagKey.VULNERABILITY_TYPE) { + constructor (moduleName, channelName, tagValues, tagKey = TagKey.VULNERABILITY_TYPE) { this.moduleName = moduleName this.channelName = channelName - this.tag = tag - this.tagKey = tagKey - this.executedMetric = getExecutedMetric(this.tagKey) - this.instrumentedMetric = getInstrumentedMetric(this.tagKey) + + tagValues = Array.isArray(tagValues) ? tagValues : [tagValues] + this.tags = formatTags(tagValues, tagKey) + + this.executedMetric = getExecutedMetric(tagKey) + this.instrumentedMetric = getInstrumentedMetric(tagKey) + this.moduleInstrumented = false } increaseInstrumented () { - if (this.moduleInstrumented) return + if (!this.moduleInstrumented) { + this.moduleInstrumented = true - this.moduleInstrumented = true - this.instrumentedMetric.inc(this.tag) + this.tags.forEach(tag => this.instrumentedMetric.inc(undefined, tag)) + } } increaseExecuted (iastContext) { - this.executedMetric.inc(this.tag, iastContext) + this.tags.forEach(tag => this.executedMetric.inc(iastContext, tag)) } matchesModuleInstrumented (name) { @@ -76,10 +81,16 @@ class IastPlugin extends Plugin { } } - _execHandlerAndIncMetric ({ handler, metric, tag, iastContext = getIastContext(storage.getStore()) }) { + _execHandlerAndIncMetric ({ handler, metric, tags, iastContext = getIastContext(storage.getStore()) }) { try { const result = handler() - iastTelemetry.isEnabled() && metric.inc(tag, iastContext) + if (iastTelemetry.isEnabled()) { + if (Array.isArray(tags)) { + tags.forEach(tag => metric.inc(iastContext, tag)) + } else { + metric.inc(iastContext, tags) + } + } return result } catch (e) { iastLog.errorAndPublish(e) @@ -101,6 +112,14 @@ class IastPlugin extends Plugin { } } + enable () { + this.configure(true) + } + + disable () { + this.configure(false) + } + onConfigure () {} configure (config) { @@ -108,7 +127,7 @@ class IastPlugin extends Plugin { config = { enabled: config } } if (config.enabled && !this.configured) { - this.onConfigure() + this.onConfigure(config.tracerConfig) this.configured = true } @@ -127,10 +146,13 @@ class IastPlugin extends Plugin { if (!channelName && !moduleName) return if (!moduleName) { - const firstSep = channelName.indexOf(':') + let firstSep = channelName.indexOf(':') if (firstSep === -1) { moduleName = channelName } else { + if (channelName.startsWith('tracing:')) { + firstSep = channelName.indexOf(':', 'tracing:'.length + 1) + } const lastSep = channelName.indexOf(':', firstSep + 1) moduleName = channelName.substring(firstSep + 1, lastSep !== -1 ? lastSep : channelName.length) } diff --git a/packages/dd-trace/src/appsec/iast/index.js b/packages/dd-trace/src/appsec/iast/index.js index 494c56c55a1..9330bfdbbb1 100644 --- a/packages/dd-trace/src/appsec/iast/index.js +++ b/packages/dd-trace/src/appsec/iast/index.js @@ -14,6 +14,7 @@ const { } = require('./taint-tracking') const { IAST_ENABLED_TAG_KEY } = require('./tags') const iastTelemetry = require('./telemetry') +const { enable: enableFsPlugin, disable: disableFsPlugin, IAST_MODULE } = require('../rasp/fs-plugin') // TODO Change to `apm:http:server:request:[start|close]` when the subscription // order of the callbacks can be enforce @@ -21,8 +22,13 @@ const requestStart = dc.channel('dd-trace:incomingHttpRequestStart') const requestClose = dc.channel('dd-trace:incomingHttpRequestEnd') const iastResponseEnd = dc.channel('datadog:iast:response-end') +let isEnabled = false + function enable (config, _tracer) { - iastTelemetry.configure(config, config.iast && config.iast.telemetryVerbosity) + if (isEnabled) return + + iastTelemetry.configure(config, config.iast?.telemetryVerbosity) + enableFsPlugin(IAST_MODULE) enableAllAnalyzers(config) enableTaintTracking(config.iast, iastTelemetry.verbosity) requestStart.subscribe(onIncomingHttpRequestStart) @@ -30,10 +36,17 @@ function enable (config, _tracer) { overheadController.configure(config.iast) overheadController.startGlobalContext() vulnerabilityReporter.start(config, _tracer) + + isEnabled = true } function disable () { + if (!isEnabled) return + + isEnabled = false + iastTelemetry.stop() + disableFsPlugin(IAST_MODULE) disableAllAnalyzers() disableTaintTracking() overheadController.finishGlobalContext() @@ -43,7 +56,7 @@ function disable () { } function onIncomingHttpRequestStart (data) { - if (data && data.req) { + if (data?.req) { const store = storage.getStore() if (store) { const topContext = web.getContext(data.req) @@ -68,11 +81,11 @@ function onIncomingHttpRequestStart (data) { } function onIncomingHttpRequestEnd (data) { - if (data && data.req) { + if (data?.req) { const store = storage.getStore() const topContext = web.getContext(data.req) const iastContext = iastContextFunctions.getIastContext(store, topContext) - if (iastContext && iastContext.rootSpan) { + if (iastContext?.rootSpan) { iastResponseEnd.publish(data) const vulnerabilities = iastContext.vulnerabilities diff --git a/packages/dd-trace/src/appsec/iast/overhead-controller.js b/packages/dd-trace/src/appsec/iast/overhead-controller.js index 75da257c836..42361608a34 100644 --- a/packages/dd-trace/src/appsec/iast/overhead-controller.js +++ b/packages/dd-trace/src/appsec/iast/overhead-controller.js @@ -52,9 +52,10 @@ function _resetGlobalContext () { } function acquireRequest (rootSpan) { - if (availableRequest > 0) { + if (availableRequest > 0 && rootSpan) { const sampling = config && typeof config.requestSampling === 'number' - ? config.requestSampling : 30 + ? config.requestSampling + : 30 if (rootSpan.context().toSpanId().slice(-2) <= sampling) { availableRequest-- return true diff --git a/packages/dd-trace/src/appsec/iast/path-line.js b/packages/dd-trace/src/appsec/iast/path-line.js index 11328ecdf15..bf7c3eb2d84 100644 --- a/packages/dd-trace/src/appsec/iast/path-line.js +++ b/packages/dd-trace/src/appsec/iast/path-line.js @@ -3,6 +3,7 @@ const path = require('path') const process = require('process') const { calculateDDBasePath } = require('../../util') +const { getCallSiteList } = require('../stack_trace') const pathLine = { getFirstNonDDPathAndLine, getNodeModulesPaths, @@ -24,21 +25,6 @@ const EXCLUDED_PATH_PREFIXES = [ 'async_hooks' ] -function getCallSiteInfo () { - const previousPrepareStackTrace = Error.prepareStackTrace - const previousStackTraceLimit = Error.stackTraceLimit - let callsiteList - Error.stackTraceLimit = 100 - Error.prepareStackTrace = function (_, callsites) { - callsiteList = callsites - } - const e = new Error() - e.stack - Error.prepareStackTrace = previousPrepareStackTrace - Error.stackTraceLimit = previousStackTraceLimit - return callsiteList -} - function getFirstNonDDPathAndLineFromCallsites (callsites, externallyExcludedPaths) { if (callsites) { for (let i = 0; i < callsites.length; i++) { @@ -88,7 +74,7 @@ function isExcluded (callsite, externallyExcludedPaths) { } function getFirstNonDDPathAndLine (externallyExcludedPaths) { - return getFirstNonDDPathAndLineFromCallsites(getCallSiteInfo(), externallyExcludedPaths) + return getFirstNonDDPathAndLineFromCallsites(getCallSiteList(), externallyExcludedPaths) } function getNodeModulesPaths (...paths) { diff --git a/packages/dd-trace/src/appsec/iast/taint-tracking/csi-methods.js b/packages/dd-trace/src/appsec/iast/taint-tracking/csi-methods.js index 39dd4378590..2133971afb9 100644 --- a/packages/dd-trace/src/appsec/iast/taint-tracking/csi-methods.js +++ b/packages/dd-trace/src/appsec/iast/taint-tracking/csi-methods.js @@ -2,14 +2,21 @@ const csiMethods = [ { src: 'concat' }, + { src: 'join' }, + { src: 'parse' }, { src: 'plusOperator', operator: true }, + { src: 'random' }, { src: 'replace' }, { src: 'slice' }, { src: 'substr' }, { src: 'substring' }, + { src: 'toLowerCase', dst: 'stringCase' }, + { src: 'toUpperCase', dst: 'stringCase' }, + { src: 'tplOperator', operator: true }, { src: 'trim' }, { src: 'trimEnd' }, - { src: 'trimStart', dst: 'trim' } + { src: 'trimStart', dst: 'trim' }, + { src: 'eval', allowedWithoutCallee: true } ] module.exports = { diff --git a/packages/dd-trace/src/appsec/iast/taint-tracking/index.js b/packages/dd-trace/src/appsec/iast/taint-tracking/index.js index 7d6003c9838..5c7109c4cda 100644 --- a/packages/dd-trace/src/appsec/iast/taint-tracking/index.js +++ b/packages/dd-trace/src/appsec/iast/taint-tracking/index.js @@ -10,21 +10,31 @@ const { } = require('./operations') const taintTrackingPlugin = require('./plugin') +const kafkaConsumerPlugin = require('./plugins/kafka') + +const kafkaContextPlugin = require('../context/kafka-ctx-plugin') module.exports = { enableTaintTracking (config, telemetryVerbosity) { enableRewriter(telemetryVerbosity) enableTaintOperations(telemetryVerbosity) taintTrackingPlugin.enable() + + kafkaContextPlugin.enable() + kafkaConsumerPlugin.enable() + setMaxTransactions(config.maxConcurrentRequests) }, disableTaintTracking () { disableRewriter() disableTaintOperations() taintTrackingPlugin.disable() + + kafkaContextPlugin.disable() + kafkaConsumerPlugin.disable() }, - setMaxTransactions: setMaxTransactions, - createTransaction: createTransaction, - removeTransaction: removeTransaction, + setMaxTransactions, + createTransaction, + removeTransaction, taintTrackingPlugin } diff --git a/packages/dd-trace/src/appsec/iast/taint-tracking/operations-taint-object.js b/packages/dd-trace/src/appsec/iast/taint-tracking/operations-taint-object.js new file mode 100644 index 00000000000..f678767394a --- /dev/null +++ b/packages/dd-trace/src/appsec/iast/taint-tracking/operations-taint-object.js @@ -0,0 +1,45 @@ +'use strict' + +const TaintedUtils = require('@datadog/native-iast-taint-tracking') +const { IAST_TRANSACTION_ID } = require('../iast-context') +const iastLog = require('../iast-log') + +function taintObject (iastContext, object, type) { + let result = object + const transactionId = iastContext?.[IAST_TRANSACTION_ID] + if (transactionId) { + const queue = [{ parent: null, property: null, value: object }] + const visited = new WeakSet() + + while (queue.length > 0) { + const { parent, property, value, key } = queue.pop() + if (value === null) { + continue + } + + try { + if (typeof value === 'string') { + const tainted = TaintedUtils.newTaintedString(transactionId, value, property, type) + if (!parent) { + result = tainted + } else { + parent[key] = tainted + } + } else if (typeof value === 'object' && !visited.has(value)) { + visited.add(value) + + for (const key of Object.keys(value)) { + queue.push({ parent: value, property: property ? `${property}.${key}` : key, value: value[key], key }) + } + } + } catch (e) { + iastLog.error(`Error visiting property : ${property}`).errorAndPublish(e) + } + } + } + return result +} + +module.exports = { + taintObject +} diff --git a/packages/dd-trace/src/appsec/iast/taint-tracking/operations.js b/packages/dd-trace/src/appsec/iast/taint-tracking/operations.js index 8240b358419..ce530b03702 100644 --- a/packages/dd-trace/src/appsec/iast/taint-tracking/operations.js +++ b/packages/dd-trace/src/appsec/iast/taint-tracking/operations.js @@ -1,12 +1,19 @@ 'use strict' +const dc = require('dc-polyfill') const TaintedUtils = require('@datadog/native-iast-taint-tracking') const { IAST_TRANSACTION_ID } = require('../iast-context') -const iastLog = require('../iast-log') const iastTelemetry = require('../telemetry') const { REQUEST_TAINTED } = require('../telemetry/iast-metric') const { isInfoAllowed } = require('../telemetry/verbosity') -const { getTaintTrackingImpl, getTaintTrackingNoop } = require('./taint-tracking-impl') +const { + getTaintTrackingImpl, + getTaintTrackingNoop, + lodashTaintTrackingHandler +} = require('./taint-tracking-impl') +const { taintObject } = require('./operations-taint-object') + +const lodashOperationCh = dc.channel('datadog:lodash:operation') function createTransaction (id, iastContext) { if (id && iastContext) { @@ -19,7 +26,7 @@ let onRemoveTransaction = (transactionId, iastContext) => {} function onRemoveTransactionInformationTelemetry (transactionId, iastContext) { const metrics = TaintedUtils.getMetrics(transactionId, iastTelemetry.verbosity) if (metrics?.requestCount) { - REQUEST_TAINTED.add(metrics.requestCount, null, iastContext) + REQUEST_TAINTED.inc(iastContext, metrics.requestCount) } } @@ -34,7 +41,7 @@ function removeTransaction (iastContext) { } function newTaintedString (iastContext, string, name, type) { - let result = string + let result const transactionId = iastContext?.[IAST_TRANSACTION_ID] if (transactionId) { result = TaintedUtils.newTaintedString(transactionId, string, name, type) @@ -44,56 +51,19 @@ function newTaintedString (iastContext, string, name, type) { return result } -function taintObject (iastContext, object, type, keyTainting, keyType) { - let result = object +function newTaintedObject (iastContext, obj, name, type) { + let result const transactionId = iastContext?.[IAST_TRANSACTION_ID] if (transactionId) { - const queue = [{ parent: null, property: null, value: object }] - const visited = new WeakSet() - - while (queue.length > 0) { - const { parent, property, value, key } = queue.pop() - if (value === null) { - continue - } - - try { - if (typeof value === 'string') { - const tainted = TaintedUtils.newTaintedString(transactionId, value, property, type) - if (!parent) { - result = tainted - } else { - if (keyTainting && key) { - const taintedProperty = TaintedUtils.newTaintedString(transactionId, key, property, keyType) - parent[taintedProperty] = tainted - } else { - parent[key] = tainted - } - } - } else if (typeof value === 'object' && !visited.has(value)) { - visited.add(value) - - const keys = Object.keys(value) - for (let i = 0; i < keys.length; i++) { - const key = keys[i] - queue.push({ parent: value, property: property ? `${property}.${key}` : key, value: value[key], key }) - } - - if (parent && keyTainting && key) { - const taintedProperty = TaintedUtils.newTaintedString(transactionId, key, property, keyType) - parent[taintedProperty] = value - } - } - } catch (e) { - iastLog.error(`Error visiting property : ${property}`).errorAndPublish(e) - } - } + result = TaintedUtils.newTaintedObject(transactionId, obj, name, type) + } else { + result = obj } return result } function isTainted (iastContext, string) { - let result = false + let result const transactionId = iastContext?.[IAST_TRANSACTION_ID] if (transactionId) { result = TaintedUtils.isTainted(transactionId, string) @@ -104,7 +74,7 @@ function isTainted (iastContext, string) { } function getRanges (iastContext, string) { - let result = [] + let result const transactionId = iastContext?.[IAST_TRANSACTION_ID] if (transactionId) { result = TaintedUtils.getRanges(transactionId, string) @@ -129,10 +99,12 @@ function enableTaintOperations (telemetryVerbosity) { } global._ddiast = getTaintTrackingImpl(telemetryVerbosity) + lodashOperationCh.subscribe(lodashTaintTrackingHandler) } function disableTaintOperations () { global._ddiast = getTaintTrackingNoop() + lodashOperationCh.unsubscribe(lodashTaintTrackingHandler) } function setMaxTransactions (transactions) { @@ -148,6 +120,7 @@ module.exports = { createTransaction, removeTransaction, newTaintedString, + newTaintedObject, taintObject, isTainted, getRanges, diff --git a/packages/dd-trace/src/appsec/iast/taint-tracking/plugin.js b/packages/dd-trace/src/appsec/iast/taint-tracking/plugin.js index 29cbb6526e1..48902323bec 100644 --- a/packages/dd-trace/src/appsec/iast/taint-tracking/plugin.js +++ b/packages/dd-trace/src/appsec/iast/taint-tracking/plugin.js @@ -3,7 +3,7 @@ const { SourceIastPlugin } = require('../iast-plugin') const { getIastContext } = require('../iast-context') const { storage } = require('../../../../../datadog-core') -const { taintObject, newTaintedString } = require('./operations') +const { taintObject, newTaintedString, getRanges } = require('./operations') const { HTTP_REQUEST_BODY, HTTP_REQUEST_COOKIE_VALUE, @@ -14,6 +14,10 @@ const { HTTP_REQUEST_PATH_PARAM, HTTP_REQUEST_URI } = require('./source-types') +const { EXECUTED_SOURCE } = require('../telemetry/iast-metric') + +const REQ_HEADER_TAGS = EXECUTED_SOURCE.formatTags(HTTP_REQUEST_HEADER_VALUE, HTTP_REQUEST_HEADER_NAME) +const REQ_URI_TAGS = EXECUTED_SOURCE.formatTags(HTTP_REQUEST_URI) class TaintTrackingPlugin extends SourceIastPlugin { constructor () { @@ -26,9 +30,9 @@ class TaintTrackingPlugin extends SourceIastPlugin { { channelName: 'datadog:body-parser:read:finish', tag: HTTP_REQUEST_BODY }, ({ req }) => { const iastContext = getIastContext(storage.getStore()) - if (iastContext && iastContext['body'] !== req.body) { + if (iastContext && iastContext.body !== req.body) { this._taintTrackingHandler(HTTP_REQUEST_BODY, req, 'body', iastContext) - iastContext['body'] = req.body + iastContext.body = req.body } } ) @@ -41,11 +45,11 @@ class TaintTrackingPlugin extends SourceIastPlugin { this.addSub( { channelName: 'apm:express:middleware:next', tag: HTTP_REQUEST_BODY }, ({ req }) => { - if (req && req.body && typeof req.body === 'object') { + if (req && req.body !== null && typeof req.body === 'object') { const iastContext = getIastContext(storage.getStore()) - if (iastContext && iastContext['body'] !== req.body) { + if (iastContext && iastContext.body !== req.body) { this._taintTrackingHandler(HTTP_REQUEST_BODY, req, 'body', iastContext) - iastContext['body'] = req.body + iastContext.body = req.body } } } @@ -59,12 +63,24 @@ class TaintTrackingPlugin extends SourceIastPlugin { this.addSub( { channelName: 'datadog:express:process_params:start', tag: HTTP_REQUEST_PATH_PARAM }, ({ req }) => { - if (req && req.params && typeof req.params === 'object') { + if (req && req.params !== null && typeof req.params === 'object') { this._taintTrackingHandler(HTTP_REQUEST_PATH_PARAM, req, 'params') } } ) + this.addSub( + { channelName: 'apm:graphql:resolve:start', tag: HTTP_REQUEST_BODY }, + (data) => { + const iastContext = getIastContext(storage.getStore()) + const source = data.context?.source + const ranges = source && getRanges(iastContext, source) + if (ranges?.length) { + this._taintTrackingHandler(ranges[0].iinfo.type, data.args, null, iastContext) + } + } + ) + // this is a special case to increment INSTRUMENTED_SOURCE metric for header this.addInstrumentedSource('http', [HTTP_REQUEST_HEADER_VALUE, HTTP_REQUEST_HEADER_NAME]) } @@ -79,13 +95,15 @@ class TaintTrackingPlugin extends SourceIastPlugin { _cookiesTaintTrackingHandler (target) { const iastContext = getIastContext(storage.getStore()) - taintObject(iastContext, target, HTTP_REQUEST_COOKIE_VALUE, true, HTTP_REQUEST_COOKIE_NAME) + // Prevent tainting cookie names since it leads to taint literal string with same value. + taintObject(iastContext, target, HTTP_REQUEST_COOKIE_VALUE) } taintHeaders (headers, iastContext) { + // Prevent tainting header names since it leads to taint literal string with same value. this.execSource({ - handler: () => taintObject(iastContext, headers, HTTP_REQUEST_HEADER_VALUE, true, HTTP_REQUEST_HEADER_NAME), - tag: [HTTP_REQUEST_HEADER_VALUE, HTTP_REQUEST_HEADER_NAME], + handler: () => taintObject(iastContext, headers, HTTP_REQUEST_HEADER_VALUE), + tags: REQ_HEADER_TAGS, iastContext }) } @@ -95,7 +113,7 @@ class TaintTrackingPlugin extends SourceIastPlugin { handler: function () { req.url = newTaintedString(iastContext, req.url, HTTP_REQUEST_URI, HTTP_REQUEST_URI) }, - tag: [HTTP_REQUEST_URI], + tags: REQ_URI_TAGS, iastContext }) } @@ -104,14 +122,6 @@ class TaintTrackingPlugin extends SourceIastPlugin { this.taintHeaders(req.headers, iastContext) this.taintUrl(req, iastContext) } - - enable () { - this.configure(true) - } - - disable () { - this.configure(false) - } } module.exports = new TaintTrackingPlugin() diff --git a/packages/dd-trace/src/appsec/iast/taint-tracking/plugins/kafka.js b/packages/dd-trace/src/appsec/iast/taint-tracking/plugins/kafka.js new file mode 100644 index 00000000000..1435978d03c --- /dev/null +++ b/packages/dd-trace/src/appsec/iast/taint-tracking/plugins/kafka.js @@ -0,0 +1,47 @@ +'use strict' + +const shimmer = require('../../../../../../datadog-shimmer') +const { storage } = require('../../../../../../datadog-core') +const { getIastContext } = require('../../iast-context') +const { KAFKA_MESSAGE_KEY, KAFKA_MESSAGE_VALUE } = require('../source-types') +const { newTaintedObject, newTaintedString } = require('../operations') +const { SourceIastPlugin } = require('../../iast-plugin') + +class KafkaConsumerIastPlugin extends SourceIastPlugin { + onConfigure () { + this.addSub({ channelName: 'dd-trace:kafkajs:consumer:afterStart', tag: [KAFKA_MESSAGE_KEY, KAFKA_MESSAGE_VALUE] }, + ({ message }) => this.taintKafkaMessage(message) + ) + } + + getToStringWrap (toString, iastContext, type) { + return function () { + const res = toString.apply(this, arguments) + return newTaintedString(iastContext, res, undefined, type) + } + } + + taintKafkaMessage (message) { + const iastContext = getIastContext(storage.getStore()) + + if (iastContext && message) { + const { key, value } = message + + if (key !== null && typeof key === 'object') { + shimmer.wrap(key, 'toString', + toString => this.getToStringWrap(toString, iastContext, KAFKA_MESSAGE_KEY)) + + newTaintedObject(iastContext, key, undefined, KAFKA_MESSAGE_KEY) + } + + if (value !== null && typeof value === 'object') { + shimmer.wrap(value, 'toString', + toString => this.getToStringWrap(toString, iastContext, KAFKA_MESSAGE_VALUE)) + + newTaintedObject(iastContext, value, undefined, KAFKA_MESSAGE_VALUE) + } + } + } +} + +module.exports = new KafkaConsumerIastPlugin() diff --git a/packages/dd-trace/src/appsec/iast/taint-tracking/rewriter-telemetry.js b/packages/dd-trace/src/appsec/iast/taint-tracking/rewriter-telemetry.js index 6643fc85826..d2279f39d26 100644 --- a/packages/dd-trace/src/appsec/iast/taint-tracking/rewriter-telemetry.js +++ b/packages/dd-trace/src/appsec/iast/taint-tracking/rewriter-telemetry.js @@ -14,7 +14,7 @@ const telemetryRewriter = { const metrics = response.metrics if (metrics && metrics.instrumentedPropagation) { - INSTRUMENTED_PROPAGATION.add(metrics.instrumentedPropagation) + INSTRUMENTED_PROPAGATION.inc(undefined, metrics.instrumentedPropagation) } return response diff --git a/packages/dd-trace/src/appsec/iast/taint-tracking/rewriter.js b/packages/dd-trace/src/appsec/iast/taint-tracking/rewriter.js index 71f6265b81d..cad8e5d6b18 100644 --- a/packages/dd-trace/src/appsec/iast/taint-tracking/rewriter.js +++ b/packages/dd-trace/src/appsec/iast/taint-tracking/rewriter.js @@ -12,6 +12,7 @@ const dc = require('dc-polyfill') const hardcodedSecretCh = dc.channel('datadog:secrets:result') let rewriter let getPrepareStackTrace +let kSymbolPrepareStackTrace let getRewriterOriginalPathAndLineFromSourceMap = function (path, line, column) { return { path, line, column } @@ -44,6 +45,7 @@ function getRewriter (telemetryVerbosity) { const iastRewriter = require('@datadog/native-iast-rewriter') const Rewriter = iastRewriter.Rewriter getPrepareStackTrace = iastRewriter.getPrepareStackTrace + kSymbolPrepareStackTrace = iastRewriter.kSymbolPrepareStackTrace const chainSourceMap = isFlagPresent('--enable-source-maps') const getOriginalPathAndLineFromSourceMap = iastRewriter.getOriginalPathAndLineFromSourceMap @@ -65,8 +67,9 @@ function getRewriter (telemetryVerbosity) { return rewriter } -let originalPrepareStackTrace = Error.prepareStackTrace +let originalPrepareStackTrace function getPrepareStackTraceAccessor () { + originalPrepareStackTrace = Error.prepareStackTrace let actual = getPrepareStackTrace(originalPrepareStackTrace) return { configurable: true, @@ -121,7 +124,16 @@ function enableRewriter (telemetryVerbosity) { function disableRewriter () { shimmer.unwrap(Module.prototype, '_compile') - Error.prepareStackTrace = originalPrepareStackTrace + + if (!Error.prepareStackTrace?.[kSymbolPrepareStackTrace]) return + + try { + delete Error.prepareStackTrace + + Error.prepareStackTrace = originalPrepareStackTrace + } catch (e) { + iastLog.warn(e) + } } function getOriginalPathAndLineFromSourceMap ({ path, line, column }) { diff --git a/packages/dd-trace/src/appsec/iast/taint-tracking/source-types.js b/packages/dd-trace/src/appsec/iast/taint-tracking/source-types.js index 2a3739515d8..f5c2ca2e8b0 100644 --- a/packages/dd-trace/src/appsec/iast/taint-tracking/source-types.js +++ b/packages/dd-trace/src/appsec/iast/taint-tracking/source-types.js @@ -9,5 +9,7 @@ module.exports = { HTTP_REQUEST_PARAMETER: 'http.request.parameter', HTTP_REQUEST_PATH: 'http.request.path', HTTP_REQUEST_PATH_PARAM: 'http.request.path.parameter', - HTTP_REQUEST_URI: 'http.request.uri' + HTTP_REQUEST_URI: 'http.request.uri', + KAFKA_MESSAGE_KEY: 'kafka.message.key', + KAFKA_MESSAGE_VALUE: 'kafka.message.value' } diff --git a/packages/dd-trace/src/appsec/iast/taint-tracking/taint-tracking-impl.js b/packages/dd-trace/src/appsec/iast/taint-tracking/taint-tracking-impl.js index 0ea3f41af78..5fa16d00d77 100644 --- a/packages/dd-trace/src/appsec/iast/taint-tracking/taint-tracking-impl.js +++ b/packages/dd-trace/src/appsec/iast/taint-tracking/taint-tracking-impl.js @@ -1,28 +1,41 @@ 'use strict' +const dc = require('dc-polyfill') const TaintedUtils = require('@datadog/native-iast-taint-tracking') const { storage } = require('../../../../../datadog-core') const iastContextFunctions = require('../iast-context') const iastLog = require('../iast-log') const { EXECUTED_PROPAGATION } = require('../telemetry/iast-metric') const { isDebugAllowed } = require('../telemetry/verbosity') +const { taintObject } = require('./operations-taint-object') + +const mathRandomCallCh = dc.channel('datadog:random:call') +const evalCallCh = dc.channel('datadog:eval:call') + +const JSON_VALUE = 'json.value' function noop (res) { return res } // NOTE: methods of this object must be synchronized with csi-methods.js file definitions! // Otherwise you may end up rewriting a method and not providing its rewritten implementation const TaintTrackingNoop = { - plusOperator: noop, concat: noop, + eval: noop, + join: noop, + parse: noop, + plusOperator: noop, + random: noop, replace: noop, slice: noop, substr: noop, substring: noop, + stringCase: noop, + tplOperator: noop, trim: noop, trimEnd: noop } function getTransactionId (iastContext) { - return iastContext && iastContext[iastContextFunctions.IAST_TRANSACTION_ID] + return iastContext?.[iastContextFunctions.IAST_TRANSACTION_ID] } function getContextDefault () { @@ -32,7 +45,7 @@ function getContextDefault () { function getContextDebug () { const iastContext = getContextDefault() - EXECUTED_PROPAGATION.inc(null, iastContext) + EXECUTED_PROPAGATION.inc(iastContext) return iastContext } @@ -99,18 +112,94 @@ function csiMethodsOverrides (getContext) { return TaintedUtils.concat(transactionId, res, op1, op2) } } catch (e) { - iastLog.error(`Error invoking CSI plusOperator`) + iastLog.error('Error invoking CSI plusOperator') .errorAndPublish(e) } return res }, + tplOperator: function (res, ...rest) { + try { + const iastContext = getContext() + const transactionId = getTransactionId(iastContext) + if (transactionId) { + return TaintedUtils.concat(transactionId, res, ...rest) + } + } catch (e) { + iastLog.error('Error invoking CSI tplOperator') + .errorAndPublish(e) + } + return res + }, + + stringCase: getCsiFn( + (transactionId, res, target) => TaintedUtils.stringCase(transactionId, res, target), + getContext, + String.prototype.toLowerCase, + String.prototype.toUpperCase + ), + trim: getCsiFn( (transactionId, res, target) => TaintedUtils.trim(transactionId, res, target), getContext, String.prototype.trim, String.prototype.trimStart - ) + ), + + random: function (res, fn) { + if (mathRandomCallCh.hasSubscribers) { + mathRandomCallCh.publish({ fn }) + } + return res + }, + + eval: function (res, fn, target, script) { + // eslint-disable-next-line no-eval + if (evalCallCh.hasSubscribers && fn === globalThis.eval) { + evalCallCh.publish({ script }) + } + + return res + }, + + parse: function (res, fn, target, json) { + if (fn === JSON.parse) { + try { + const iastContext = getContext() + const transactionId = getTransactionId(iastContext) + if (transactionId) { + const ranges = TaintedUtils.getRanges(transactionId, json) + + // TODO: first version. + // here we are losing the original source because taintObject always creates a new tainted + if (ranges?.length > 0) { + const range = ranges.find(range => range.iinfo?.type) + res = taintObject(iastContext, res, range?.iinfo.type || JSON_VALUE) + } + } + } catch (e) { + iastLog.error(e) + } + } + + return res + }, + + join: function (res, fn, target, separator) { + if (fn === Array.prototype.join) { + try { + const iastContext = getContext() + const transactionId = getTransactionId(iastContext) + if (transactionId) { + res = TaintedUtils.arrayJoin(transactionId, res, target, separator) + } + } catch (e) { + iastLog.error(e) + } + } + + return res + } } } @@ -138,7 +227,36 @@ function getTaintTrackingNoop () { return getTaintTrackingImpl(null, true) } +const lodashFns = { + join: TaintedUtils.arrayJoin, + toLower: TaintedUtils.stringCase, + toUpper: TaintedUtils.stringCase, + trim: TaintedUtils.trim, + trimEnd: TaintedUtils.trimEnd, + trimStart: TaintedUtils.trim + +} + +function getLodashTaintedUtilFn (lodashFn) { + return lodashFns[lodashFn] || ((transactionId, result) => result) +} + +function lodashTaintTrackingHandler (message) { + try { + if (!message.result) return + const context = getContextDefault() + const transactionId = getTransactionId(context) + if (transactionId) { + message.result = getLodashTaintedUtilFn(message.operation)(transactionId, message.result, ...message.arguments) + } + } catch (e) { + iastLog.error(`Error invoking CSI lodash ${message.operation}`) + .errorAndPublish(e) + } +} + module.exports = { getTaintTrackingImpl, - getTaintTrackingNoop + getTaintTrackingNoop, + lodashTaintTrackingHandler } diff --git a/packages/dd-trace/src/appsec/iast/telemetry/iast-metric.js b/packages/dd-trace/src/appsec/iast/telemetry/iast-metric.js index 675a7a7237b..2928e566829 100644 --- a/packages/dd-trace/src/appsec/iast/telemetry/iast-metric.js +++ b/packages/dd-trace/src/appsec/iast/telemetry/iast-metric.js @@ -19,6 +19,19 @@ const TagKey = { PROPAGATION_TYPE: 'propagation_type' } +function formatTags (tags, tagKey) { + return tags.map(tagValue => tagValue ? [`${tagKey}:${tagValue.toLowerCase()}`] : undefined) +} + +function getNamespace (scope, context) { + let namespace = globalNamespace + + if (scope === Scope.REQUEST) { + namespace = getNamespaceFromContext(context) || globalNamespace + } + return namespace +} + class IastMetric { constructor (name, scope, tagKey) { this.name = name @@ -26,30 +39,26 @@ class IastMetric { this.tagKey = tagKey } - getNamespace (context) { - return getNamespaceFromContext(context) || globalNamespace + formatTags (...tags) { + return formatTags(tags, this.tagKey) } - getTag (tagValue) { - return tagValue ? { [this.tagKey]: tagValue } : undefined + inc (context, tags, value = 1) { + const namespace = getNamespace(this.scope, context) + namespace.count(this.name, tags).inc(value) } +} - addValue (value, tagValue, context) { - this.getNamespace(context) - .count(this.name, this.getTag(tagValue)) - .inc(value) - } +class NoTaggedIastMetric extends IastMetric { + constructor (name, scope) { + super(name, scope) - add (value, tagValue, context) { - if (Array.isArray(tagValue)) { - tagValue.forEach(tag => this.addValue(value, tag, context)) - } else { - this.addValue(value, tagValue, context) - } + this.tags = [] } - inc (tagValue, context) { - this.add(1, tagValue, context) + inc (context, value = 1) { + const namespace = getNamespace(this.scope, context) + namespace.count(this.name, this.tags).inc(value) } } @@ -61,21 +70,18 @@ function getInstrumentedMetric (tagKey) { return tagKey === TagKey.VULNERABILITY_TYPE ? INSTRUMENTED_SINK : INSTRUMENTED_SOURCE } -const INSTRUMENTED_PROPAGATION = new IastMetric('instrumented.propagation', Scope.GLOBAL) +const INSTRUMENTED_PROPAGATION = new NoTaggedIastMetric('instrumented.propagation', Scope.GLOBAL) const INSTRUMENTED_SOURCE = new IastMetric('instrumented.source', Scope.GLOBAL, TagKey.SOURCE_TYPE) const INSTRUMENTED_SINK = new IastMetric('instrumented.sink', Scope.GLOBAL, TagKey.VULNERABILITY_TYPE) const EXECUTED_SOURCE = new IastMetric('executed.source', Scope.REQUEST, TagKey.SOURCE_TYPE) const EXECUTED_SINK = new IastMetric('executed.sink', Scope.REQUEST, TagKey.VULNERABILITY_TYPE) -const REQUEST_TAINTED = new IastMetric('request.tainted', Scope.REQUEST) +const REQUEST_TAINTED = new NoTaggedIastMetric('request.tainted', Scope.REQUEST) // DEBUG using metrics -const EXECUTED_PROPAGATION = new IastMetric('executed.propagation', Scope.REQUEST) -const EXECUTED_TAINTED = new IastMetric('executed.tainted', Scope.REQUEST) - -// DEBUG using distribution endpoint -const INSTRUMENTATION_TIME = new IastMetric('instrumentation.time', Scope.GLOBAL) +const EXECUTED_PROPAGATION = new NoTaggedIastMetric('executed.propagation', Scope.REQUEST) +const EXECUTED_TAINTED = new NoTaggedIastMetric('executed.tainted', Scope.REQUEST) module.exports = { INSTRUMENTED_PROPAGATION, @@ -89,13 +95,14 @@ module.exports = { REQUEST_TAINTED, - INSTRUMENTATION_TIME, - PropagationType, TagKey, IastMetric, + NoTaggedIastMetric, getExecutedMetric, - getInstrumentedMetric + getInstrumentedMetric, + + formatTags } diff --git a/packages/dd-trace/src/appsec/iast/telemetry/namespaces.js b/packages/dd-trace/src/appsec/iast/telemetry/namespaces.js index d00e52aef33..77a0db04604 100644 --- a/packages/dd-trace/src/appsec/iast/telemetry/namespaces.js +++ b/packages/dd-trace/src/appsec/iast/telemetry/namespaces.js @@ -2,21 +2,22 @@ const log = require('../../../log') const { Namespace } = require('../../../telemetry/metrics') -const { addMetricsToSpan, filterTags } = require('./span-tags') +const { addMetricsToSpan } = require('./span-tags') const { IAST_TRACE_METRIC_PREFIX } = require('../tags') +const iastLog = require('../iast-log') const DD_IAST_METRICS_NAMESPACE = Symbol('_dd.iast.request.metrics.namespace') function initRequestNamespace (context) { if (!context) return - const namespace = new Namespace('iast') + const namespace = new IastNamespace() context[DD_IAST_METRICS_NAMESPACE] = namespace return namespace } function getNamespaceFromContext (context) { - return context && context[DD_IAST_METRICS_NAMESPACE] + return context?.[DD_IAST_METRICS_NAMESPACE] } function finalizeRequestNamespace (context, rootSpan) { @@ -24,12 +25,11 @@ function finalizeRequestNamespace (context, rootSpan) { const namespace = getNamespaceFromContext(context) if (!namespace) return - const metrics = [...namespace.metrics.values()] - namespace.metrics.clear() + addMetricsToSpan(rootSpan, [...namespace.metrics.values()], IAST_TRACE_METRIC_PREFIX) - addMetricsToSpan(rootSpan, metrics, IAST_TRACE_METRIC_PREFIX) + merge(namespace) - merge(metrics) + namespace.clear() } catch (e) { log.error(e) } finally { @@ -39,28 +39,63 @@ function finalizeRequestNamespace (context, rootSpan) { } } -function merge (metrics) { - metrics.forEach(metric => metric.points.forEach(point => { - globalNamespace - .count(metric.metric, getTagsObject(metric.tags)) - .inc(point[1]) - })) -} +function merge (namespace) { + for (const [metricName, metricsByTagMap] of namespace.iastMetrics) { + for (const [tags, metric] of metricsByTagMap) { + const { type, points } = metric -function getTagsObject (tags) { - if (tags && tags.length > 0) { - return filterTags(tags) + if (points?.length && type === 'count') { + const gMetric = globalNamespace.getMetric(metricName, tags) + points.forEach(point => gMetric.inc(point[1])) + } + } } } class IastNamespace extends Namespace { - constructor () { + constructor (maxMetricTagsSize = 100) { super('iast') + + this.maxMetricTagsSize = maxMetricTagsSize + this.iastMetrics = new Map() } - reset () { - this.metrics.clear() + getIastMetrics (name) { + let metrics = this.iastMetrics.get(name) + if (!metrics) { + metrics = new Map() + this.iastMetrics.set(name, metrics) + } + + return metrics + } + + getMetric (name, tags, type = 'count') { + const metrics = this.getIastMetrics(name) + + let metric = metrics.get(tags) + if (!metric) { + metric = super[type](name, Array.isArray(tags) ? [...tags] : tags) + + if (metrics.size === this.maxMetricTagsSize) { + metrics.clear() + iastLog.warnAndPublish(`Tags cache max size reached for metric ${name}`) + } + + metrics.set(tags, metric) + } + + return metric + } + + count (name, tags) { + return this.getMetric(name, tags, 'count') + } + + clear () { + this.iastMetrics.clear() this.distributions.clear() + this.metrics.clear() } } @@ -72,5 +107,7 @@ module.exports = { finalizeRequestNamespace, globalNamespace, - DD_IAST_METRICS_NAMESPACE + DD_IAST_METRICS_NAMESPACE, + + IastNamespace } diff --git a/packages/dd-trace/src/appsec/iast/telemetry/span-tags.js b/packages/dd-trace/src/appsec/iast/telemetry/span-tags.js index 9d434bdb9ae..befcadf507c 100644 --- a/packages/dd-trace/src/appsec/iast/telemetry/span-tags.js +++ b/packages/dd-trace/src/appsec/iast/telemetry/span-tags.js @@ -1,11 +1,11 @@ 'use strict' function addMetricsToSpan (rootSpan, metrics, tagPrefix) { - if (!rootSpan || !rootSpan.addTags || !metrics) return + if (!rootSpan?.addTags || !metrics) return const flattenMap = new Map() metrics - .filter(data => data && data.metric) + .filter(data => data?.metric) .forEach(data => { const name = taggedMetricName(data) let total = flattenMap.get(name) @@ -27,19 +27,20 @@ function addMetricsToSpan (rootSpan, metrics, tagPrefix) { } function flatten (metricData) { - return metricData.points && metricData.points.map(point => point[1]).reduce((total, value) => total + value, 0) + const { points } = metricData + return points ? points.map(point => point[1]).reduce((total, value) => total + value, 0) : 0 } function taggedMetricName (data) { const metric = data.metric - const tags = data.tags && filterTags(data.tags) - return !tags || !tags.length + const tags = filterTags(data.tags) + return !tags?.length ? metric : `${metric}.${processTagValue(tags)}` } function filterTags (tags) { - return tags.filter(tag => !tag.startsWith('lib_language') && !tag.startsWith('version')) + return tags?.filter(tag => !tag.startsWith('version')) } function processTagValue (tags) { diff --git a/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/evidence-redaction/sensitive-analyzers/code-injection-sensitive-analyzer.js b/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/evidence-redaction/sensitive-analyzers/code-injection-sensitive-analyzer.js new file mode 100644 index 00000000000..ebced31c398 --- /dev/null +++ b/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/evidence-redaction/sensitive-analyzers/code-injection-sensitive-analyzer.js @@ -0,0 +1,25 @@ +'use strict' + +module.exports = function extractSensitiveRanges (evidence) { + const newRanges = [] + if (evidence.ranges[0].start > 0) { + newRanges.push({ + start: 0, + end: evidence.ranges[0].start + }) + } + + for (let i = 0; i < evidence.ranges.length; i++) { + const currentRange = evidence.ranges[i] + const nextRange = evidence.ranges[i + 1] + + const start = currentRange.end + const end = nextRange?.start || evidence.value.length + + if (start < end) { + newRanges.push({ start, end }) + } + } + + return newRanges +} diff --git a/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/evidence-redaction/sensitive-analyzers/hardcoded-password-analyzer.js b/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/evidence-redaction/sensitive-analyzers/hardcoded-password-analyzer.js new file mode 100644 index 00000000000..72361fb6d59 --- /dev/null +++ b/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/evidence-redaction/sensitive-analyzers/hardcoded-password-analyzer.js @@ -0,0 +1,13 @@ +'use strict' + +module.exports = function extractSensitiveRanges (evidence, valuePattern) { + const { value } = evidence + if (valuePattern.test(value)) { + return [{ + start: 0, + end: value.length + }] + } + + return [] +} diff --git a/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/evidence-redaction/sensitive-analyzers/sql-sensitive-analyzer.js b/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/evidence-redaction/sensitive-analyzers/sql-sensitive-analyzer.js index 88056f93168..15580b11869 100644 --- a/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/evidence-redaction/sensitive-analyzers/sql-sensitive-analyzer.js +++ b/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/evidence-redaction/sensitive-analyzers/sql-sensitive-analyzer.js @@ -68,7 +68,7 @@ module.exports = function extractSensitiveRanges (evidence) { try { let pattern = patterns[evidence.dialect] if (!pattern) { - pattern = patterns['ANSI'] + pattern = patterns.ANSI } pattern.lastIndex = 0 const tokens = [] diff --git a/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/evidence-redaction/sensitive-handler.js b/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/evidence-redaction/sensitive-handler.js index b2ae03e4bb6..39117dc5a34 100644 --- a/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/evidence-redaction/sensitive-handler.js +++ b/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/evidence-redaction/sensitive-handler.js @@ -5,7 +5,9 @@ const vulnerabilities = require('../../vulnerabilities') const { contains, intersects, remove } = require('./range-utils') +const codeInjectionSensitiveAnalyzer = require('./sensitive-analyzers/code-injection-sensitive-analyzer') const commandSensitiveAnalyzer = require('./sensitive-analyzers/command-sensitive-analyzer') +const hardcodedPasswordAnalyzer = require('./sensitive-analyzers/hardcoded-password-analyzer') const headerSensitiveAnalyzer = require('./sensitive-analyzers/header-sensitive-analyzer') const jsonSensitiveAnalyzer = require('./sensitive-analyzers/json-sensitive-analyzer') const ldapSensitiveAnalyzer = require('./sensitive-analyzers/ldap-sensitive-analyzer') @@ -22,6 +24,7 @@ class SensitiveHandler { this._valuePattern = new RegExp(DEFAULT_IAST_REDACTION_VALUE_PATTERN, 'gmi') this._sensitiveAnalyzers = new Map() + this._sensitiveAnalyzers.set(vulnerabilities.CODE_INJECTION, codeInjectionSensitiveAnalyzer) this._sensitiveAnalyzers.set(vulnerabilities.COMMAND_INJECTION, commandSensitiveAnalyzer) this._sensitiveAnalyzers.set(vulnerabilities.NOSQL_MONGODB_INJECTION, jsonSensitiveAnalyzer) this._sensitiveAnalyzers.set(vulnerabilities.LDAP_INJECTION, ldapSensitiveAnalyzer) @@ -31,6 +34,9 @@ class SensitiveHandler { this._sensitiveAnalyzers.set(vulnerabilities.HEADER_INJECTION, (evidence) => { return headerSensitiveAnalyzer(evidence, this._namePattern, this._valuePattern) }) + this._sensitiveAnalyzers.set(vulnerabilities.HARDCODED_PASSWORD, (evidence) => { + return hardcodedPasswordAnalyzer(evidence, this._valuePattern) + }) } isSensibleName (name) { @@ -51,7 +57,9 @@ class SensitiveHandler { const sensitiveAnalyzer = this._sensitiveAnalyzers.get(vulnerabilityType) if (sensitiveAnalyzer) { const sensitiveRanges = sensitiveAnalyzer(evidence) - return this.toRedactedJson(evidence, sensitiveRanges, sourcesIndexes, sources) + if (evidence.ranges || sensitiveRanges?.length) { + return this.toRedactedJson(evidence, sensitiveRanges, sourcesIndexes, sources) + } } return null } @@ -67,7 +75,7 @@ class SensitiveHandler { let nextTaintedIndex = 0 let sourceIndex - let nextTainted = ranges.shift() + let nextTainted = ranges?.shift() let nextSensitive = sensitive.shift() for (let i = 0; i < value.length; i++) { diff --git a/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/evidence-redaction/sensitive-regex.js b/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/evidence-redaction/sensitive-regex.js index f1ae7249a42..fe9d22f9c49 100644 --- a/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/evidence-redaction/sensitive-regex.js +++ b/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/evidence-redaction/sensitive-regex.js @@ -1,7 +1,7 @@ // eslint-disable-next-line max-len -const DEFAULT_IAST_REDACTION_NAME_PATTERN = '(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?|access_?|secret_?)key(?:_?id)?|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)' +const DEFAULT_IAST_REDACTION_NAME_PATTERN = '(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?|access_?|secret_?)key(?:_?id)?|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?|(?:sur|last)name|user(?:name)?|address|e?mail)' // eslint-disable-next-line max-len -const DEFAULT_IAST_REDACTION_VALUE_PATTERN = '(?:bearer\\s+[a-z0-9\\._\\-]+|glpat-[\\w\\-]{20}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L][\\w=\\-]+\\.ey[I-L][\\w=\\-]+(?:\\.[\\w.+/=\\-]+)?|(?:[\\-]{5}BEGIN[a-z\\s]+PRIVATE\\sKEY[\\-]{5}[^\\-]+[\\-]{5}END[a-z\\s]+PRIVATE\\sKEY[\\-]{5}|ssh-rsa\\s*[a-z0-9/\\.+]{100,}))' +const DEFAULT_IAST_REDACTION_VALUE_PATTERN = '(?:bearer\\s+[a-z0-9\\._\\-]+|glpat-[\\w\\-]{20}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L][\\w=\\-]+\\.ey[I-L][\\w=\\-]+(?:\\.[\\w.+/=\\-]+)?|(?:[\\-]{5}BEGIN[a-z\\s]+PRIVATE\\sKEY[\\-]{5}[^\\-]+[\\-]{5}END[a-z\\s]+PRIVATE\\sKEY[\\-]{5}|ssh-rsa\\s*[a-z0-9/\\.+]{100,})|[\\w\\.-]+@[a-zA-Z\\d\\.-]+\\.[a-zA-Z]{2,})' module.exports = { DEFAULT_IAST_REDACTION_NAME_PATTERN, diff --git a/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/index.js b/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/index.js index 9dfca76a9e6..d704743dde4 100644 --- a/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/index.js +++ b/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/index.js @@ -43,12 +43,18 @@ class VulnerabilityFormatter { const valueParts = [] let fromIndex = 0 + if (evidence.value == null) return { valueParts } + if (typeof evidence.value === 'object' && evidence.rangesToApply) { const { value, ranges } = stringifyWithRanges(evidence.value, evidence.rangesToApply) evidence.value = value evidence.ranges = ranges } + if (!evidence.ranges) { + return { value: evidence.value } + } + evidence.ranges.forEach((range, rangeIndex) => { if (fromIndex < range.start) { valueParts.push({ value: evidence.value.substring(fromIndex, range.start) }) @@ -65,12 +71,8 @@ class VulnerabilityFormatter { } formatEvidence (type, evidence, sourcesIndexes, sources) { - if (!evidence.ranges && !evidence.rangesToApply) { - if (typeof evidence.value === 'undefined') { - return undefined - } else { - return { value: evidence.value } - } + if (evidence.value === undefined) { + return undefined } return this._redactVulnearbilities diff --git a/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/utils.js b/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/utils.js index bf065ad3d65..959df790afd 100644 --- a/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/utils.js +++ b/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/utils.js @@ -13,15 +13,18 @@ const KEYS_REGEX_WITHOUT_SENSITIVE_RANGES = new RegExp(`"(${STRINGIFY_RANGE_KEY} const sensitiveValueRegex = new RegExp(DEFAULT_IAST_REDACTION_VALUE_PATTERN, 'gmi') -function iterateObject (target, fn, levelKeys = [], depth = 50) { +function iterateObject (target, fn, levelKeys = [], depth = 10, visited = new Set()) { Object.keys(target).forEach((key) => { const nextLevelKeys = [...levelKeys, key] const val = target[key] - fn(val, nextLevelKeys, target, key) + if (typeof val !== 'object' || !visited.has(val)) { + visited.add(val) + fn(val, nextLevelKeys, target, key) - if (val !== null && typeof val === 'object') { - iterateObject(val, fn, nextLevelKeys, depth - 1) + if (val !== null && typeof val === 'object' && depth > 0) { + iterateObject(val, fn, nextLevelKeys, depth - 1, visited) + } } }) } diff --git a/packages/dd-trace/src/appsec/iast/vulnerabilities.js b/packages/dd-trace/src/appsec/iast/vulnerabilities.js index a248b50c632..790ec6c5db9 100644 --- a/packages/dd-trace/src/appsec/iast/vulnerabilities.js +++ b/packages/dd-trace/src/appsec/iast/vulnerabilities.js @@ -1,5 +1,7 @@ module.exports = { COMMAND_INJECTION: 'COMMAND_INJECTION', + CODE_INJECTION: 'CODE_INJECTION', + HARDCODED_PASSWORD: 'HARDCODED_PASSWORD', HARDCODED_SECRET: 'HARDCODED_SECRET', HEADER_INJECTION: 'HEADER_INJECTION', HSTS_HEADER_MISSING: 'HSTS_HEADER_MISSING', @@ -14,5 +16,6 @@ module.exports = { UNVALIDATED_REDIRECT: 'UNVALIDATED_REDIRECT', WEAK_CIPHER: 'WEAK_CIPHER', WEAK_HASH: 'WEAK_HASH', + WEAK_RANDOMNESS: 'WEAK_RANDOMNESS', XCONTENTTYPE_HEADER_MISSING: 'XCONTENTTYPE_HEADER_MISSING' } diff --git a/packages/dd-trace/src/appsec/iast/vulnerability-reporter.js b/packages/dd-trace/src/appsec/iast/vulnerability-reporter.js index cd6bf5f1180..cc25d51b1e9 100644 --- a/packages/dd-trace/src/appsec/iast/vulnerability-reporter.js +++ b/packages/dd-trace/src/appsec/iast/vulnerability-reporter.js @@ -4,6 +4,7 @@ const { MANUAL_KEEP } = require('../../../../../ext/tags') const LRU = require('lru-cache') const vulnerabilitiesFormatter = require('./vulnerabilities-formatter') const { IAST_ENABLED_TAG_KEY, IAST_JSON_TAG_KEY } = require('./tags') +const standalone = require('../standalone') const VULNERABILITIES_KEY = 'vulnerabilities' const VULNERABILITY_HASHES_MAX_SIZE = 1000 @@ -57,6 +58,9 @@ function sendVulnerabilities (vulnerabilities, rootSpan) { tags[IAST_JSON_TAG_KEY] = JSON.stringify(jsonToSend) tags[MANUAL_KEEP] = 'true' span.addTags(tags) + + standalone.sample(span) + if (!rootSpan) span.finish() } } diff --git a/packages/dd-trace/src/appsec/index.js b/packages/dd-trace/src/appsec/index.js index 386918636cc..f3656e459e8 100644 --- a/packages/dd-trace/src/appsec/index.js +++ b/packages/dd-trace/src/appsec/index.js @@ -6,41 +6,46 @@ const remoteConfig = require('./remote_config') const { bodyParser, cookieParser, - graphqlFinishExecute, incomingHttpRequestStart, incomingHttpRequestEnd, passportVerify, queryParser, nextBodyParsed, - nextQueryParsed + nextQueryParsed, + expressProcessParams, + responseBody, + responseWriteHead, + responseSetHeader } = require('./channels') const waf = require('./waf') const addresses = require('./addresses') const Reporter = require('./reporter') const appsecTelemetry = require('./telemetry') +const apiSecuritySampler = require('./api_security_sampler') const web = require('../plugins/util/web') const { extractIp } = require('../plugins/util/ip_extractor') const { HTTP_CLIENT_IP } = require('../../../../ext/tags') -const { block, setTemplates } = require('./blocking') +const { isBlocked, block, setTemplates, getBlockingAction } = require('./blocking') const { passportTrackEvent } = require('./passport') const { storage } = require('../../../datadog-core') +const graphql = require('./graphql') +const rasp = require('./rasp') + +const responseAnalyzedSet = new WeakSet() let isEnabled = false let config -function sampleRequest ({ enabled, requestSampling }) { - if (!enabled || !requestSampling) { - return false - } - - return Math.random() <= requestSampling -} - function enable (_config) { if (isEnabled) return try { appsecTelemetry.enable(_config.telemetry) + graphql.enable() + + if (_config.appsec.rasp.enabled) { + rasp.enable(_config) + } setTemplates(_config) @@ -50,14 +55,19 @@ function enable (_config) { Reporter.setRateLimit(_config.appsec.rateLimit) + apiSecuritySampler.configure(_config.appsec) + + bodyParser.subscribe(onRequestBodyParsed) + cookieParser.subscribe(onRequestCookieParser) incomingHttpRequestStart.subscribe(incomingHttpStartTranslator) incomingHttpRequestEnd.subscribe(incomingHttpEndTranslator) - bodyParser.subscribe(onRequestBodyParsed) + queryParser.subscribe(onRequestQueryParsed) nextBodyParsed.subscribe(onRequestBodyParsed) nextQueryParsed.subscribe(onRequestQueryParsed) - queryParser.subscribe(onRequestQueryParsed) - cookieParser.subscribe(onRequestCookieParser) - graphqlFinishExecute.subscribe(onGraphqlFinishExecute) + expressProcessParams.subscribe(onRequestProcessParams) + responseBody.subscribe(onResponseBody) + responseWriteHead.subscribe(onResponseWriteHead) + responseSetHeader.subscribe(onResponseSetHeader) if (_config.appsec.eventTracking.enabled) { passportVerify.subscribe(onPassportVerify) @@ -73,6 +83,41 @@ function enable (_config) { } } +function onRequestBodyParsed ({ req, res, body, abortController }) { + if (body === undefined || body === null) return + + if (!req) { + const store = storage.getStore() + req = store?.req + } + + const rootSpan = web.root(req) + if (!rootSpan) return + + const results = waf.run({ + persistent: { + [addresses.HTTP_INCOMING_BODY]: body + } + }, req) + + handleResults(results, req, res, rootSpan, abortController) +} + +function onRequestCookieParser ({ req, res, abortController, cookies }) { + if (!cookies || typeof cookies !== 'object') return + + const rootSpan = web.root(req) + if (!rootSpan) return + + const results = waf.run({ + persistent: { + [addresses.HTTP_INCOMING_COOKIES]: cookies + } + }, req) + + handleResults(results, req, res, rootSpan, abortController) +} + function incomingHttpStartTranslator ({ req, res, abortController }) { const rootSpan = web.root(req) if (!rootSpan) return @@ -88,78 +133,62 @@ function incomingHttpStartTranslator ({ req, res, abortController }) { const requestHeaders = Object.assign({}, req.headers) delete requestHeaders.cookie - const payload = { + const persistent = { [addresses.HTTP_INCOMING_URL]: req.url, [addresses.HTTP_INCOMING_HEADERS]: requestHeaders, [addresses.HTTP_INCOMING_METHOD]: req.method } if (clientIp) { - payload[addresses.HTTP_CLIENT_IP] = clientIp + persistent[addresses.HTTP_CLIENT_IP] = clientIp } - if (sampleRequest(config.appsec.apiSecurity)) { - payload[addresses.WAF_CONTEXT_PROCESSOR] = { 'extract-schema': true } + if (apiSecuritySampler.sampleRequest(req)) { + persistent[addresses.WAF_CONTEXT_PROCESSOR] = { 'extract-schema': true } } - const actions = waf.run(payload, req) + const actions = waf.run({ persistent }, req) handleResults(actions, req, res, rootSpan, abortController) } function incomingHttpEndTranslator ({ req, res }) { - // TODO: this doesn't support headers sent with res.writeHead() - const responseHeaders = Object.assign({}, res.getHeaders()) - delete responseHeaders['set-cookie'] - - const payload = { - [addresses.HTTP_INCOMING_RESPONSE_CODE]: '' + res.statusCode, - [addresses.HTTP_INCOMING_RESPONSE_HEADERS]: responseHeaders - } + const persistent = {} // we need to keep this to support other body parsers // TODO: no need to analyze it if it was already done by the body-parser hook if (req.body !== undefined && req.body !== null) { - payload[addresses.HTTP_INCOMING_BODY] = req.body - } - - // TODO: temporary express instrumentation, will use express plugin later - if (req.params && typeof req.params === 'object') { - payload[addresses.HTTP_INCOMING_PARAMS] = req.params + persistent[addresses.HTTP_INCOMING_BODY] = req.body } // we need to keep this to support other cookie parsers - if (req.cookies && typeof req.cookies === 'object') { - payload[addresses.HTTP_INCOMING_COOKIES] = req.cookies + if (req.cookies !== null && typeof req.cookies === 'object') { + persistent[addresses.HTTP_INCOMING_COOKIES] = req.cookies } - if (req.query && typeof req.query === 'object') { - payload[addresses.HTTP_INCOMING_QUERY] = req.query + if (req.query !== null && typeof req.query === 'object') { + persistent[addresses.HTTP_INCOMING_QUERY] = req.query } - waf.run(payload, req) + if (Object.keys(persistent).length) { + waf.run({ persistent }, req) + } waf.disposeContext(req) Reporter.finishRequest(req, res) } -function onRequestBodyParsed ({ req, res, body, abortController }) { - if (body === undefined || body === null) return +function onPassportVerify ({ credentials, user }) { + const store = storage.getStore() + const rootSpan = store?.req && web.root(store.req) - if (!req) { - const store = storage.getStore() - req = store?.req + if (!rootSpan) { + log.warn('No rootSpan found in onPassportVerify') + return } - const rootSpan = web.root(req) - if (!rootSpan) return - - const results = waf.run({ - [addresses.HTTP_INCOMING_BODY]: body - }, req) - - handleResults(results, req, res, rootSpan, abortController) + passportTrackEvent(credentials, user, rootSpan, config.appsec.eventTracking.mode) } function onRequestQueryParsed ({ req, res, query, abortController }) { @@ -174,56 +203,83 @@ function onRequestQueryParsed ({ req, res, query, abortController }) { if (!rootSpan) return const results = waf.run({ - [addresses.HTTP_INCOMING_QUERY]: query + persistent: { + [addresses.HTTP_INCOMING_QUERY]: query + } }, req) handleResults(results, req, res, rootSpan, abortController) } -function onRequestCookieParser ({ req, res, abortController, cookies }) { - if (!cookies || typeof cookies !== 'object') return - +function onRequestProcessParams ({ req, res, abortController, params }) { const rootSpan = web.root(req) if (!rootSpan) return + if (!params || typeof params !== 'object' || !Object.keys(params).length) return + const results = waf.run({ - [addresses.HTTP_INCOMING_COOKIES]: cookies + persistent: { + [addresses.HTTP_INCOMING_PARAMS]: params + } }, req) handleResults(results, req, res, rootSpan, abortController) } -function onPassportVerify ({ credentials, user }) { - const store = storage.getStore() - const rootSpan = store && store.req && web.root(store.req) +function onResponseBody ({ req, body }) { + if (!body || typeof body !== 'object') return + if (!apiSecuritySampler.isSampled(req)) return - if (!rootSpan) { - log.warn('No rootSpan found in onPassportVerify') + // we don't support blocking at this point, so no results needed + waf.run({ + persistent: { + [addresses.HTTP_INCOMING_RESPONSE_BODY]: body + } + }, req) +} + +function onResponseWriteHead ({ req, res, abortController, statusCode, responseHeaders }) { + // avoid "write after end" error + if (isBlocked(res)) { + abortController?.abort() return } - passportTrackEvent(credentials, user, rootSpan, config.appsec.eventTracking.mode) -} + // avoid double waf call + if (responseAnalyzedSet.has(res)) { + return + } -function onGraphqlFinishExecute ({ context }) { - const store = storage.getStore() - const req = store?.req + const rootSpan = web.root(req) + if (!rootSpan) return - if (!req) return + responseHeaders = Object.assign({}, responseHeaders) + delete responseHeaders['set-cookie'] - const resolvers = context?.resolvers + const results = waf.run({ + persistent: { + [addresses.HTTP_INCOMING_RESPONSE_CODE]: '' + statusCode, + [addresses.HTTP_INCOMING_RESPONSE_HEADERS]: responseHeaders + } + }, req) - if (!resolvers || typeof resolvers !== 'object') return + responseAnalyzedSet.add(res) + + handleResults(results, req, res, rootSpan, abortController) +} - // Don't collect blocking result because it only works in monitor mode. - waf.run({ [addresses.HTTP_INCOMING_GRAPHQL_RESOLVERS]: resolvers }, req) +function onResponseSetHeader ({ res, abortController }) { + if (isBlocked(res)) { + abortController?.abort() + } } function handleResults (actions, req, res, rootSpan, abortController) { if (!actions || !req || !res || !rootSpan || !abortController) return - if (actions.includes('block')) { - block(req, res, rootSpan, abortController) + const blockingAction = getBlockingAction(actions) + if (blockingAction) { + block(req, res, rootSpan, abortController, blockingAction) } } @@ -234,17 +290,26 @@ function disable () { RuleManager.clearAllRules() appsecTelemetry.disable() + graphql.disable() + rasp.disable() remoteConfig.disableWafUpdate() + apiSecuritySampler.disable() + // Channel#unsubscribe() is undefined for non active channels if (bodyParser.hasSubscribers) bodyParser.unsubscribe(onRequestBodyParsed) - if (graphqlFinishExecute.hasSubscribers) graphqlFinishExecute.unsubscribe(onGraphqlFinishExecute) + if (cookieParser.hasSubscribers) cookieParser.unsubscribe(onRequestCookieParser) if (incomingHttpRequestStart.hasSubscribers) incomingHttpRequestStart.unsubscribe(incomingHttpStartTranslator) if (incomingHttpRequestEnd.hasSubscribers) incomingHttpRequestEnd.unsubscribe(incomingHttpEndTranslator) - if (queryParser.hasSubscribers) queryParser.unsubscribe(onRequestQueryParsed) - if (cookieParser.hasSubscribers) cookieParser.unsubscribe(onRequestCookieParser) if (passportVerify.hasSubscribers) passportVerify.unsubscribe(onPassportVerify) + if (queryParser.hasSubscribers) queryParser.unsubscribe(onRequestQueryParsed) + if (nextBodyParsed.hasSubscribers) nextBodyParsed.unsubscribe(onRequestBodyParsed) + if (nextQueryParsed.hasSubscribers) nextQueryParsed.unsubscribe(onRequestQueryParsed) + if (expressProcessParams.hasSubscribers) expressProcessParams.unsubscribe(onRequestProcessParams) + if (responseBody.hasSubscribers) responseBody.unsubscribe(onResponseBody) + if (responseWriteHead.hasSubscribers) responseWriteHead.unsubscribe(onResponseWriteHead) + if (responseSetHeader.hasSubscribers) responseSetHeader.unsubscribe(onResponseSetHeader) } module.exports = { diff --git a/packages/dd-trace/src/appsec/passport.js b/packages/dd-trace/src/appsec/passport.js index 83460c6b07b..2093b7b1fdc 100644 --- a/packages/dd-trace/src/appsec/passport.js +++ b/packages/dd-trace/src/appsec/passport.js @@ -13,7 +13,7 @@ const regexSdkEvent = new RegExp(SDK_USER_EVENT_PATTERN, 'i') function isSdkCalled (tags) { let called = false - if (tags && typeof tags === 'object') { + if (tags !== null && typeof tags === 'object') { called = Object.entries(tags).some(([key, value]) => regexSdkEvent.test(key) && value === 'true') } diff --git a/packages/dd-trace/src/appsec/rasp/fs-plugin.js b/packages/dd-trace/src/appsec/rasp/fs-plugin.js new file mode 100644 index 00000000000..a283b4f1a61 --- /dev/null +++ b/packages/dd-trace/src/appsec/rasp/fs-plugin.js @@ -0,0 +1,99 @@ +'use strict' + +const Plugin = require('../../plugins/plugin') +const { storage } = require('../../../../datadog-core') +const log = require('../../log') + +const RASP_MODULE = 'rasp' +const IAST_MODULE = 'iast' + +const enabledFor = { + [RASP_MODULE]: false, + [IAST_MODULE]: false +} + +let fsPlugin + +function enterWith (fsProps, store = storage.getStore()) { + if (store && !store.fs?.opExcluded) { + storage.enterWith({ + ...store, + fs: { + ...store.fs, + ...fsProps, + parentStore: store + } + }) + } +} + +class AppsecFsPlugin extends Plugin { + enable () { + this.addSub('apm:fs:operation:start', this._onFsOperationStart) + this.addSub('apm:fs:operation:finish', this._onFsOperationFinishOrRenderEnd) + this.addSub('tracing:datadog:express:response:render:start', this._onResponseRenderStart) + this.addSub('tracing:datadog:express:response:render:end', this._onFsOperationFinishOrRenderEnd) + + super.configure(true) + } + + disable () { + super.configure(false) + } + + _onFsOperationStart () { + const store = storage.getStore() + if (store) { + enterWith({ root: store.fs?.root === undefined }, store) + } + } + + _onResponseRenderStart () { + enterWith({ opExcluded: true }) + } + + _onFsOperationFinishOrRenderEnd () { + const store = storage.getStore() + if (store?.fs?.parentStore) { + storage.enterWith(store.fs.parentStore) + } + } +} + +function enable (mod) { + if (enabledFor[mod] !== false) return + + enabledFor[mod] = true + + if (!fsPlugin) { + fsPlugin = new AppsecFsPlugin() + fsPlugin.enable() + } + + log.info(`Enabled AppsecFsPlugin for ${mod}`) +} + +function disable (mod) { + if (!mod || !enabledFor[mod]) return + + enabledFor[mod] = false + + const allDisabled = Object.values(enabledFor).every(val => val === false) + if (allDisabled) { + fsPlugin?.disable() + + fsPlugin = undefined + } + + log.info(`Disabled AppsecFsPlugin for ${mod}`) +} + +module.exports = { + enable, + disable, + + AppsecFsPlugin, + + RASP_MODULE, + IAST_MODULE +} diff --git a/packages/dd-trace/src/appsec/rasp/index.js b/packages/dd-trace/src/appsec/rasp/index.js new file mode 100644 index 00000000000..d5a1312872a --- /dev/null +++ b/packages/dd-trace/src/appsec/rasp/index.js @@ -0,0 +1,117 @@ +'use strict' + +const web = require('../../plugins/util/web') +const { setUncaughtExceptionCaptureCallbackStart, expressMiddlewareError } = require('../channels') +const { block, isBlocked } = require('../blocking') +const ssrf = require('./ssrf') +const sqli = require('./sql_injection') +const lfi = require('./lfi') + +const { DatadogRaspAbortError } = require('./utils') + +function removeAllListeners (emitter, event) { + const listeners = emitter.listeners(event) + emitter.removeAllListeners(event) + + let cleaned = false + return function () { + if (cleaned === true) { + return + } + cleaned = true + + for (let i = 0; i < listeners.length; ++i) { + emitter.on(event, listeners[i]) + } + } +} + +function findDatadogRaspAbortError (err, deep = 10) { + if (err instanceof DatadogRaspAbortError) { + return err + } + + if (err?.cause && deep > 0) { + return findDatadogRaspAbortError(err.cause, deep - 1) + } +} + +function handleUncaughtExceptionMonitor (error) { + if (!blockOnDatadogRaspAbortError({ error })) return + + if (!process.hasUncaughtExceptionCaptureCallback()) { + const cleanUp = removeAllListeners(process, 'uncaughtException') + const handler = () => { + process.removeListener('uncaughtException', handler) + } + + setTimeout(() => { + process.removeListener('uncaughtException', handler) + cleanUp() + }) + + process.on('uncaughtException', handler) + } else { + // uncaughtException event is not executed when hasUncaughtExceptionCaptureCallback is true + let previousCb + const cb = ({ currentCallback, abortController }) => { + setUncaughtExceptionCaptureCallbackStart.unsubscribe(cb) + if (!currentCallback) { + abortController.abort() + return + } + + previousCb = currentCallback + } + + setUncaughtExceptionCaptureCallbackStart.subscribe(cb) + + process.setUncaughtExceptionCaptureCallback(null) + + // For some reason, previous callback was defined before the instrumentation + // We can not restore it, so we let the app decide + if (previousCb) { + process.setUncaughtExceptionCaptureCallback(() => { + process.setUncaughtExceptionCaptureCallback(null) + process.setUncaughtExceptionCaptureCallback(previousCb) + }) + } + } +} + +function blockOnDatadogRaspAbortError ({ error }) { + const abortError = findDatadogRaspAbortError(error) + if (!abortError) return false + + const { req, res, blockingAction } = abortError + if (!isBlocked(res)) { + block(req, res, web.root(req), null, blockingAction) + } + + return true +} + +function enable (config) { + ssrf.enable(config) + sqli.enable(config) + lfi.enable(config) + + process.on('uncaughtExceptionMonitor', handleUncaughtExceptionMonitor) + expressMiddlewareError.subscribe(blockOnDatadogRaspAbortError) +} + +function disable () { + ssrf.disable() + sqli.disable() + lfi.disable() + + process.off('uncaughtExceptionMonitor', handleUncaughtExceptionMonitor) + if (expressMiddlewareError.hasSubscribers) expressMiddlewareError.unsubscribe(blockOnDatadogRaspAbortError) +} + +module.exports = { + enable, + disable, + handleUncaughtExceptionMonitor, // exported only for testing purpose + blockOnDatadogRaspAbortError // exported only for testing purpose +} diff --git a/packages/dd-trace/src/appsec/rasp/lfi.js b/packages/dd-trace/src/appsec/rasp/lfi.js new file mode 100644 index 00000000000..1190734064d --- /dev/null +++ b/packages/dd-trace/src/appsec/rasp/lfi.js @@ -0,0 +1,112 @@ +'use strict' + +const { fsOperationStart, incomingHttpRequestStart } = require('../channels') +const { storage } = require('../../../../datadog-core') +const { enable: enableFsPlugin, disable: disableFsPlugin, RASP_MODULE } = require('./fs-plugin') +const { FS_OPERATION_PATH } = require('../addresses') +const waf = require('../waf') +const { RULE_TYPES, handleResult } = require('./utils') +const { isAbsolute } = require('path') + +let config +let enabled +let analyzeSubscribed + +function enable (_config) { + config = _config + + if (enabled) return + + enabled = true + + incomingHttpRequestStart.subscribe(onFirstReceivedRequest) +} + +function disable () { + if (fsOperationStart.hasSubscribers) fsOperationStart.unsubscribe(analyzeLfi) + if (incomingHttpRequestStart.hasSubscribers) incomingHttpRequestStart.unsubscribe(onFirstReceivedRequest) + + disableFsPlugin(RASP_MODULE) + + enabled = false + analyzeSubscribed = false +} + +function onFirstReceivedRequest () { + // nodejs unsubscribe during publish bug: https://github.com/nodejs/node/pull/55116 + process.nextTick(() => { + incomingHttpRequestStart.unsubscribe(onFirstReceivedRequest) + }) + + enableFsPlugin(RASP_MODULE) + + if (!analyzeSubscribed) { + fsOperationStart.subscribe(analyzeLfi) + analyzeSubscribed = true + } +} + +function analyzeLfi (ctx) { + const store = storage.getStore() + if (!store) return + + const { req, fs, res } = store + if (!req || !fs) return + + getPaths(ctx, fs).forEach(path => { + const persistent = { + [FS_OPERATION_PATH]: path + } + + const result = waf.run({ persistent }, req, RULE_TYPES.LFI) + handleResult(result, req, res, ctx.abortController, config) + }) +} + +function getPaths (ctx, fs) { + // these properties could have String, Buffer, URL, Integer or FileHandle types + const pathArguments = [ + ctx.dest, + ctx.existingPath, + ctx.file, + ctx.newPath, + ctx.oldPath, + ctx.path, + ctx.prefix, + ctx.src, + ctx.target + ] + + return pathArguments + .map(path => pathToStr(path)) + .filter(path => shouldAnalyze(path, fs)) +} + +function pathToStr (path) { + if (!path) return + + if (typeof path === 'string' || + path instanceof String || + path instanceof Buffer || + path instanceof URL) { + return path.toString() + } +} + +function shouldAnalyze (path, fs) { + if (!path) return + + const notExcludedRootOp = !fs.opExcluded && fs.root + return notExcludedRootOp && (isAbsolute(path) || path.includes('../') || shouldAnalyzeURLFile(path, fs)) +} + +function shouldAnalyzeURLFile (path, fs) { + if (path.startsWith('file://')) { + return shouldAnalyze(path.substring(7), fs) + } +} + +module.exports = { + enable, + disable +} diff --git a/packages/dd-trace/src/appsec/rasp/sql_injection.js b/packages/dd-trace/src/appsec/rasp/sql_injection.js new file mode 100644 index 00000000000..d4a165d8615 --- /dev/null +++ b/packages/dd-trace/src/appsec/rasp/sql_injection.js @@ -0,0 +1,106 @@ +'use strict' + +const { + pgQueryStart, + pgPoolQueryStart, + wafRunFinished, + mysql2OuterQueryStart +} = require('../channels') +const { storage } = require('../../../../datadog-core') +const addresses = require('../addresses') +const waf = require('../waf') +const { RULE_TYPES, handleResult } = require('./utils') + +const DB_SYSTEM_POSTGRES = 'postgresql' +const DB_SYSTEM_MYSQL = 'mysql' +const reqQueryMap = new WeakMap() // WeakMap> + +let config + +function enable (_config) { + config = _config + + pgQueryStart.subscribe(analyzePgSqlInjection) + pgPoolQueryStart.subscribe(analyzePgSqlInjection) + wafRunFinished.subscribe(clearQuerySet) + + mysql2OuterQueryStart.subscribe(analyzeMysql2SqlInjection) +} + +function disable () { + if (pgQueryStart.hasSubscribers) pgQueryStart.unsubscribe(analyzePgSqlInjection) + if (pgPoolQueryStart.hasSubscribers) pgPoolQueryStart.unsubscribe(analyzePgSqlInjection) + if (wafRunFinished.hasSubscribers) wafRunFinished.unsubscribe(clearQuerySet) + if (mysql2OuterQueryStart.hasSubscribers) mysql2OuterQueryStart.unsubscribe(analyzeMysql2SqlInjection) +} + +function analyzeMysql2SqlInjection (ctx) { + const query = ctx.sql + if (!query) return + + analyzeSqlInjection(query, DB_SYSTEM_MYSQL, ctx.abortController) +} + +function analyzePgSqlInjection (ctx) { + const query = ctx.query?.text + if (!query) return + + analyzeSqlInjection(query, DB_SYSTEM_POSTGRES, ctx.abortController) +} + +function analyzeSqlInjection (query, dbSystem, abortController) { + const store = storage.getStore() + if (!store) return + + const { req, res } = store + + if (!req) return + + let executedQueries = reqQueryMap.get(req) + if (executedQueries?.has(query)) return + + // Do not waste time checking same query twice + // This also will prevent double calls in pg.Pool internal queries + if (!executedQueries) { + executedQueries = new Set() + reqQueryMap.set(req, executedQueries) + } + executedQueries.add(query) + + const persistent = { + [addresses.DB_STATEMENT]: query, + [addresses.DB_SYSTEM]: dbSystem + } + + const result = waf.run({ persistent }, req, RULE_TYPES.SQL_INJECTION) + + handleResult(result, req, res, abortController, config) +} + +function hasInputAddress (payload) { + return hasAddressesObjectInputAddress(payload.ephemeral) || hasAddressesObjectInputAddress(payload.persistent) +} + +function hasAddressesObjectInputAddress (addressesObject) { + return addressesObject && Object.keys(addressesObject) + .some(address => address.startsWith('server.request') || address.startsWith('graphql.server')) +} + +function clearQuerySet ({ payload }) { + if (!payload) return + + const store = storage.getStore() + if (!store) return + + const { req } = store + if (!req) return + + const executedQueries = reqQueryMap.get(req) + if (!executedQueries) return + + if (hasInputAddress(payload)) { + executedQueries.clear() + } +} + +module.exports = { enable, disable } diff --git a/packages/dd-trace/src/appsec/rasp/ssrf.js b/packages/dd-trace/src/appsec/rasp/ssrf.js new file mode 100644 index 00000000000..38a3c150d74 --- /dev/null +++ b/packages/dd-trace/src/appsec/rasp/ssrf.js @@ -0,0 +1,38 @@ +'use strict' + +const { format } = require('url') +const { httpClientRequestStart } = require('../channels') +const { storage } = require('../../../../datadog-core') +const addresses = require('../addresses') +const waf = require('../waf') +const { RULE_TYPES, handleResult } = require('./utils') + +let config + +function enable (_config) { + config = _config + httpClientRequestStart.subscribe(analyzeSsrf) +} + +function disable () { + if (httpClientRequestStart.hasSubscribers) httpClientRequestStart.unsubscribe(analyzeSsrf) +} + +function analyzeSsrf (ctx) { + const store = storage.getStore() + const req = store?.req + const outgoingUrl = (ctx.args.options?.uri && format(ctx.args.options.uri)) ?? ctx.args.uri + + if (!req || !outgoingUrl) return + + const persistent = { + [addresses.HTTP_OUTGOING_URL]: outgoingUrl + } + + const result = waf.run({ persistent }, req, RULE_TYPES.SSRF) + + const res = store?.res + handleResult(result, req, res, ctx.abortController, config) +} + +module.exports = { enable, disable } diff --git a/packages/dd-trace/src/appsec/rasp/utils.js b/packages/dd-trace/src/appsec/rasp/utils.js new file mode 100644 index 00000000000..c4ee4f55c3f --- /dev/null +++ b/packages/dd-trace/src/appsec/rasp/utils.js @@ -0,0 +1,64 @@ +'use strict' + +const web = require('../../plugins/util/web') +const { reportStackTrace } = require('../stack_trace') +const { getBlockingAction } = require('../blocking') +const log = require('../../log') + +const abortOnUncaughtException = process.execArgv?.includes('--abort-on-uncaught-exception') + +if (abortOnUncaughtException) { + log.warn('The --abort-on-uncaught-exception flag is enabled. The RASP module will not block operations.') +} + +const RULE_TYPES = { + SSRF: 'ssrf', + SQL_INJECTION: 'sql_injection', + LFI: 'lfi' +} + +class DatadogRaspAbortError extends Error { + constructor (req, res, blockingAction) { + super('DatadogRaspAbortError') + this.name = 'DatadogRaspAbortError' + this.req = req + this.res = res + this.blockingAction = blockingAction + } +} + +function handleResult (actions, req, res, abortController, config) { + const generateStackTraceAction = actions?.generate_stack + if (generateStackTraceAction && config.appsec.stackTrace.enabled) { + const rootSpan = web.root(req) + reportStackTrace( + rootSpan, + generateStackTraceAction.stack_id, + config.appsec.stackTrace.maxDepth, + config.appsec.stackTrace.maxStackTraces + ) + } + + if (!abortController || abortOnUncaughtException) return + + const blockingAction = getBlockingAction(actions) + if (blockingAction) { + const rootSpan = web.root(req) + // Should block only in express + if (rootSpan?.context()._name === 'express.request') { + const abortError = new DatadogRaspAbortError(req, res, blockingAction) + abortController.abort(abortError) + + // TODO Delete this when support for node 16 is removed + if (!abortController.signal.reason) { + abortController.signal.reason = abortError + } + } + } +} + +module.exports = { + handleResult, + RULE_TYPES, + DatadogRaspAbortError +} diff --git a/packages/dd-trace/src/appsec/recommended.json b/packages/dd-trace/src/appsec/recommended.json index fc316459b63..158c33a8ccd 100644 --- a/packages/dd-trace/src/appsec/recommended.json +++ b/packages/dd-trace/src/appsec/recommended.json @@ -1,7 +1,7 @@ { "version": "2.2", "metadata": { - "rules_version": "1.9.0" + "rules_version": "1.13.1" }, "rules": [ { @@ -118,6 +118,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "list": [ @@ -138,7 +141,10 @@ "appscan_fingerprint", "w00tw00t.at.isc.sans.dfind", "w00tw00t.at.blackhats.romanian.anti-sec" - ] + ], + "options": { + "enforce_word_boundary": true + } }, "operator": "phrase_match" } @@ -346,6 +352,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "list": [ @@ -1772,7 +1781,10 @@ "windows\\win.ini", "default\\ntuser.dat", "/var/run/secrets/kubernetes.io/serviceaccount" - ] + ], + "options": { + "enforce_word_boundary": true + } }, "operator": "phrase_match" } @@ -1839,6 +1851,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "^(?i:file|ftps?)://.*?\\?+$", @@ -1881,8 +1896,14 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], + "options": { + "enforce_word_boundary": true + }, "list": [ "${cdpath}", "${dirstack}", @@ -1900,7 +1921,6 @@ "$ifs", "$oldpwd", "$ostype", - "$path", "$pwd", "dev/fd/", "dev/null", @@ -2391,6 +2411,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "^\\(\\s*\\)\\s+{", @@ -2456,7 +2479,10 @@ "settings.local.php", "local.xml", ".env" - ] + ], + "options": { + "enforce_word_boundary": true + } }, "operator": "phrase_match" } @@ -2547,8 +2573,14 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], + "options": { + "enforce_word_boundary": true + }, "list": [ "$globals", "$_cookie", @@ -2608,6 +2640,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "(?:HTTP_(?:ACCEPT(?:_(?:ENCODING|LANGUAGE|CHARSET))?|(?:X_FORWARDED_FO|REFERE)R|(?:USER_AGEN|HOS)T|CONNECTION|KEEP_ALIVE)|PATH_(?:TRANSLATED|INFO)|ORIG_PATH_INFO|QUERY_STRING|REQUEST_URI|AUTH_TYPE)", @@ -2650,6 +2685,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "php://(?:std(?:in|out|err)|(?:in|out)put|fd|memory|temp|filter)", @@ -2691,6 +2729,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "list": [ @@ -2738,7 +2779,10 @@ "wp_safe_remote_post", "wp_safe_remote_request", "zlib_decode" - ] + ], + "options": { + "enforce_word_boundary": true + } }, "operator": "phrase_match" } @@ -2775,6 +2819,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "\\b(?:s(?:e(?:t(?:_(?:e(?:xception|rror)_handler|magic_quotes_runtime|include_path)|defaultstub)|ssion_s(?:et_save_handler|tart))|qlite_(?:(?:(?:unbuffered|single|array)_)?query|create_(?:aggregate|function)|p?open|exec)|tr(?:eam_(?:context_create|socket_client)|ipc?slashes|rev)|implexml_load_(?:string|file)|ocket_c(?:onnect|reate)|h(?:ow_sourc|a1_fil)e|pl_autoload_register|ystem)|p(?:r(?:eg_(?:replace(?:_callback(?:_array)?)?|match(?:_all)?|split)|oc_(?:(?:terminat|clos|nic)e|get_status|open)|int_r)|o(?:six_(?:get(?:(?:e[gu]|g)id|login|pwnam)|mk(?:fifo|nod)|ttyname|kill)|pen)|hp(?:_(?:strip_whitespac|unam)e|version|info)|g_(?:(?:execut|prepar)e|connect|query)|a(?:rse_(?:ini_file|str)|ssthru)|utenv)|r(?:unkit_(?:function_(?:re(?:defin|nam)e|copy|add)|method_(?:re(?:defin|nam)e|copy|add)|constant_(?:redefine|add))|e(?:(?:gister_(?:shutdown|tick)|name)_function|ad(?:(?:gz)?file|_exif_data|dir))|awurl(?:de|en)code)|i(?:mage(?:createfrom(?:(?:jpe|pn)g|x[bp]m|wbmp|gif)|(?:jpe|pn)g|g(?:d2?|if)|2?wbmp|xbm)|s_(?:(?:(?:execut|write?|read)ab|fi)le|dir)|ni_(?:get(?:_all)?|set)|terator_apply|ptcembed)|g(?:et(?:_(?:c(?:urrent_use|fg_va)r|meta_tags)|my(?:[gpu]id|inode)|(?:lastmo|cw)d|imagesize|env)|z(?:(?:(?:defla|wri)t|encod|fil)e|compress|open|read)|lob)|a(?:rray_(?:u(?:intersect(?:_u?assoc)?|diff(?:_u?assoc)?)|intersect_u(?:assoc|key)|diff_u(?:assoc|key)|filter|reduce|map)|ssert(?:_options)?|tob)|h(?:tml(?:specialchars(?:_decode)?|_entity_decode|entities)|(?:ash(?:_(?:update|hmac))?|ighlight)_file|e(?:ader_register_callback|x2bin))|f(?:i(?:le(?:(?:[acm]tim|inod)e|(?:_exist|perm)s|group)?|nfo_open)|tp_(?:nb_(?:ge|pu)|connec|ge|pu)t|(?:unction_exis|pu)ts|write|open)|o(?:b_(?:get_(?:c(?:ontents|lean)|flush)|end_(?:clean|flush)|clean|flush|start)|dbc_(?:result(?:_all)?|exec(?:ute)?|connect)|pendir)|m(?:b_(?:ereg(?:_(?:replace(?:_callback)?|match)|i(?:_replace)?)?|parse_str)|(?:ove_uploaded|d5)_file|ethod_exists|ysql_query|kdir)|e(?:x(?:if_(?:t(?:humbnail|agname)|imagetype|read_data)|ec)|scapeshell(?:arg|cmd)|rror_reporting|val)|c(?:url_(?:file_create|exec|init)|onvert_uuencode|reate_function|hr)|u(?:n(?:serialize|pack)|rl(?:de|en)code|[ak]?sort)|b(?:(?:son_(?:de|en)|ase64_en)code|zopen|toa)|(?:json_(?:de|en)cod|debug_backtrac|tmpfil)e|var_dump)(?:\\s|/\\*.*\\*/|//.*|#.*|\\\"|')*\\((?:(?:\\s|/\\*.*\\*/|//.*|#.*)*(?:\\$\\w+|[A-Z\\d]\\w*|\\w+\\(.*\\)|\\\\?\"(?:[^\"]|\\\\\"|\"\"|\"\\+\")*\\\\?\"|\\\\?'(?:[^']|''|'\\+')*\\\\?')(?:\\s|/\\*.*\\*/|//.*|#.*)*(?:(?:::|\\.|->)(?:\\s|/\\*.*\\*/|//.*|#.*)*\\w+(?:\\(.*\\))?)?,)*(?:(?:\\s|/\\*.*\\*/|//.*|#.*)*(?:\\$\\w+|[A-Z\\d]\\w*|\\w+\\(.*\\)|\\\\?\"(?:[^\"]|\\\\\"|\"\"|\"\\+\")*\\\\?\"|\\\\?'(?:[^']|''|'\\+')*\\\\?')(?:\\s|/\\*.*\\*/|//.*|#.*)*(?:(?:::|\\.|->)(?:\\s|/\\*.*\\*/|//.*|#.*)*\\w+(?:\\(.*\\))?)?)?\\)", @@ -2820,6 +2867,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "[oOcC]:\\d+:\\\".+?\\\":\\d+:{[\\W\\w]*}", @@ -2861,6 +2911,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "(?:(?:bzip|ssh)2|z(?:lib|ip)|(?:ph|r)ar|expect|glob|ogg)://", @@ -2904,6 +2957,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "\\b(?:(?:l(?:(?:utimes|chmod)(?:Sync)?|(?:stat|ink)Sync)|w(?:rite(?:(?:File|v)(?:Sync)?|Sync)|atchFile)|u(?:n(?:watchFile|linkSync)|times(?:Sync)?)|s(?:(?:ymlink|tat)Sync|pawn(?:File|Sync))|ex(?:ec(?:File(?:Sync)?|Sync)|istsSync)|a(?:ppendFile|ccess)(?:Sync)?|(?:Caveat|Inode)s|open(?:dir)?Sync|new\\s+Function|Availability|\\beval)\\s*\\(|m(?:ain(?:Module\\s*(?:\\W*\\s*(?:constructor|require)|\\[)|\\s*(?:\\W*\\s*(?:constructor|require)|\\[))|kd(?:temp(?:Sync)?|irSync)\\s*\\(|odule\\.exports\\s*=)|c(?:(?:(?:h(?:mod|own)|lose)Sync|reate(?:Write|Read)Stream|p(?:Sync)?)\\s*\\(|o(?:nstructor\\s*(?:\\W*\\s*_load|\\[)|pyFile(?:Sync)?\\s*\\())|f(?:(?:(?:s(?:(?:yncS)?|tatS)|datas(?:yncS)?)ync|ch(?:mod|own)(?:Sync)?)\\s*\\(|u(?:nction\\s*\\(\\s*\\)\\s*{|times(?:Sync)?\\s*\\())|r(?:e(?:(?:ad(?:(?:File|link|dir)?Sync|v(?:Sync)?)|nameSync)\\s*\\(|quire\\s*(?:\\W*\\s*main|\\[))|m(?:Sync)?\\s*\\()|process\\s*(?:\\W*\\s*(?:mainModule|binding)|\\[)|t(?:his\\.constructor|runcateSync\\s*\\()|_(?:\\$\\$ND_FUNC\\$\\$_|_js_function)|global\\s*(?:\\W*\\s*process|\\[)|String\\s*\\.\\s*fromCharCode|binding\\s*\\[)", @@ -2942,10 +2998,10 @@ "address": "server.request.path_params" }, { - "address": "grpc.server.request.message" + "address": "graphql.server.all_resolvers" }, { - "address": "graphql.server.all_resolvers" + "address": "graphql.server.resolver" } ], "regex": "\\b(?:w(?:atch|rite)|(?:spaw|ope)n|exists|close|fork|read)\\s*\\(", @@ -2996,10 +3052,10 @@ "address": "server.request.path_params" }, { - "address": "grpc.server.request.message" + "address": "graphql.server.all_resolvers" }, { - "address": "graphql.server.all_resolvers" + "address": "graphql.server.resolver" } ], "regex": "]*>[\\s\\S]*?", @@ -3057,6 +3113,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "\\bon(?:d(?:r(?:ag(?:en(?:ter|d)|leave|start|over)?|op)|urationchange|blclick)|s(?:e(?:ek(?:ing|ed)|arch|lect)|u(?:spend|bmit)|talled|croll|how)|m(?:ouse(?:(?:lea|mo)ve|o(?:ver|ut)|enter|down|up)|essage)|p(?:a(?:ge(?:hide|show)|(?:st|us)e)|lay(?:ing)?|rogress|aste|ointer(?:cancel|down|enter|leave|move|out|over|rawupdate|up))|c(?:anplay(?:through)?|o(?:ntextmenu|py)|hange|lick|ut)|a(?:nimation(?:iteration|start|end)|(?:fterprin|bor)t|uxclick|fterscriptexecute)|t(?:o(?:uch(?:cancel|start|move|end)|ggle)|imeupdate)|f(?:ullscreen(?:change|error)|ocus(?:out|in)?|inish)|(?:(?:volume|hash)chang|o(?:ff|n)lin)e|b(?:efore(?:unload|print)|lur)|load(?:ed(?:meta)?data|start|end)?|r(?:es(?:ize|et)|atechange)|key(?:press|down|up)|w(?:aiting|heel)|in(?:valid|put)|e(?:nded|rror)|unload)[\\s\\x0B\\x09\\x0C\\x3B\\x2C\\x28\\x3B]*?=[^=]", @@ -3113,6 +3172,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "[a-z]+=(?:[^:=]+:.+;)*?[^:=]+:url\\(javascript", @@ -3169,6 +3231,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "(?:\\W|^)(?:javascript:(?:[\\s\\S]+[=\\x5c\\(\\[\\.<]|[\\s\\S]*?(?:\\bname\\b|\\x5c[ux]\\d)))|@\\W*?i\\W*?m\\W*?p\\W*?o\\W*?r\\W*?t\\W*?(?:/\\*[\\s\\S]*?)?(?:[\\\"']|\\W*?u\\W*?r\\W*?l[\\s\\S]*?\\()|[^-]*?-\\W*?m\\W*?o\\W*?z\\W*?-\\W*?b\\W*?i\\W*?n\\W*?d\\W*?i\\W*?n\\W*?g[^:]*?:\\W*?u\\W*?r\\W*?l[\\s\\S]*?\\(", @@ -3212,8 +3277,14 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], + "options": { + "enforce_word_boundary": true + }, "list": [ "document.cookie", "document.write", @@ -3260,6 +3331,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "(?i:<.*[:]?vmlframe.*?[\\s/+]*?src[\\s/+]*=)", @@ -3304,6 +3378,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "(?i:(?:j|&#x?0*(?:74|4A|106|6A);?)(?:\\t|\\n|\\r|&(?:#x?0*(?:9|13|10|A|D);?|tab;|newline;))*(?:a|&#x?0*(?:65|41|97|61);?)(?:\\t|\\n|\\r|&(?:#x?0*(?:9|13|10|A|D);?|tab;|newline;))*(?:v|&#x?0*(?:86|56|118|76);?)(?:\\t|\\n|\\r|&(?:#x?0*(?:9|13|10|A|D);?|tab;|newline;))*(?:a|&#x?0*(?:65|41|97|61);?)(?:\\t|\\n|\\r|&(?:#x?0*(?:9|13|10|A|D);?|tab;|newline;))*(?:s|&#x?0*(?:83|53|115|73);?)(?:\\t|\\n|\\r|&(?:#x?0*(?:9|13|10|A|D);?|tab;|newline;))*(?:c|&#x?0*(?:67|43|99|63);?)(?:\\t|\\n|\\r|&(?:#x?0*(?:9|13|10|A|D);?|tab;|newline;))*(?:r|&#x?0*(?:82|52|114|72);?)(?:\\t|\\n|\\r|&(?:#x?0*(?:9|13|10|A|D);?|tab;|newline;))*(?:i|&#x?0*(?:73|49|105|69);?)(?:\\t|\\n|\\r|&(?:#x?0*(?:9|13|10|A|D);?|tab;|newline;))*(?:p|&#x?0*(?:80|50|112|70);?)(?:\\t|\\n|\\r|&(?:#x?0*(?:9|13|10|A|D);?|tab;|newline;))*(?:t|&#x?0*(?:84|54|116|74);?)(?:\\t|\\n|\\r|&(?:#x?0*(?:9|13|10|A|D);?|tab;|newline;))*(?::|&(?:#x?0*(?:58|3A);?|colon;)).)", @@ -3348,6 +3425,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "(?i:(?:v|&#x?0*(?:86|56|118|76);?)(?:\\t|&(?:#x?0*(?:9|13|10|A|D);?|tab;|newline;))*(?:b|&#x?0*(?:66|42|98|62);?)(?:\\t|&(?:#x?0*(?:9|13|10|A|D);?|tab;|newline;))*(?:s|&#x?0*(?:83|53|115|73);?)(?:\\t|&(?:#x?0*(?:9|13|10|A|D);?|tab;|newline;))*(?:c|&#x?0*(?:67|43|99|63);?)(?:\\t|&(?:#x?0*(?:9|13|10|A|D);?|tab;|newline;))*(?:r|&#x?0*(?:82|52|114|72);?)(?:\\t|&(?:#x?0*(?:9|13|10|A|D);?|tab;|newline;))*(?:i|&#x?0*(?:73|49|105|69);?)(?:\\t|&(?:#x?0*(?:9|13|10|A|D);?|tab;|newline;))*(?:p|&#x?0*(?:80|50|112|70);?)(?:\\t|&(?:#x?0*(?:9|13|10|A|D);?|tab;|newline;))*(?:t|&#x?0*(?:84|54|116|74);?)(?:\\t|&(?:#x?0*(?:9|13|10|A|D);?|tab;|newline;))*(?::|&(?:#x?0*(?:58|3A);?|colon;)).)", @@ -3392,6 +3472,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "]", @@ -3608,6 +3700,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": ")|<.*\\+AD4-", @@ -3692,6 +3790,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "![!+ ]\\[\\]", @@ -3734,6 +3835,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "\\b(?i:eval|settimeout|setinterval|new\\s+Function|alert|prompt)[\\s+]*\\([^\\)]", @@ -3771,10 +3875,10 @@ "address": "server.request.path_params" }, { - "address": "grpc.server.request.message" + "address": "graphql.server.all_resolvers" }, { - "address": "graphql.server.all_resolvers" + "address": "graphql.server.resolver" } ] }, @@ -3814,6 +3918,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "(?i:sleep\\(\\s*?\\d*?\\s*?\\)|benchmark\\(.*?\\,.*?\\))", @@ -3856,6 +3963,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "(?:[\\\"'`](?:;*?\\s*?waitfor\\s+(?:delay|time)\\s+[\\\"'`]|;.*?:\\s*?goto)|alter\\s*?\\w+.*?cha(?:racte)?r\\s+set\\s+\\w+)", @@ -3896,6 +4006,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "(?i:merge.*?using\\s*?\\(|execute\\s*?immediate\\s*?[\\\"'`]|match\\s*?[\\w(?:),+-]+\\s*?against\\s*?\\()", @@ -3937,6 +4050,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "union.*?select.*?from", @@ -3978,6 +4094,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "(?:;\\s*?shutdown\\s*?(?:[#;{]|\\/\\*|--)|waitfor\\s*?delay\\s?[\\\"'`]+\\s?\\d|select\\s*?pg_sleep)", @@ -4018,6 +4137,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "(?i:(?:\\[?\\$(?:(?:s(?:lic|iz)|wher)e|e(?:lemMatch|xists|q)|n(?:o[rt]|in?|e)|l(?:ike|te?)|t(?:ext|ype)|a(?:ll|nd)|jsonSchema|between|regex|x?or|div|mod)\\]?)\\b)", @@ -4061,6 +4183,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "(?:^[\\W\\d]+\\s*?(?:alter\\s*(?:a(?:(?:pplication\\s*rol|ggregat)e|s(?:ymmetric\\s*ke|sembl)y|u(?:thorization|dit)|vailability\\s*group)|c(?:r(?:yptographic\\s*provider|edential)|o(?:l(?:latio|um)|nversio)n|ertificate|luster)|s(?:e(?:rv(?:ice|er)|curity|quence|ssion|arch)|y(?:mmetric\\s*key|nonym)|togroup|chema)|m(?:a(?:s(?:ter\\s*key|k)|terialized)|e(?:ssage\\s*type|thod)|odule)|l(?:o(?:g(?:file\\s*group|in)|ckdown)|a(?:ngua|r)ge|ibrary)|t(?:(?:abl(?:espac)?|yp)e|r(?:igger|usted)|hreshold|ext)|p(?:a(?:rtition|ckage)|ro(?:cedur|fil)e|ermission)|d(?:i(?:mension|skgroup)|atabase|efault|omain)|r(?:o(?:l(?:lback|e)|ute)|e(?:sourc|mot)e)|f(?:u(?:lltext|nction)|lashback|oreign)|e(?:xte(?:nsion|rnal)|(?:ndpoi|ve)nt)|in(?:dex(?:type)?|memory|stance)|b(?:roker\\s*priority|ufferpool)|x(?:ml\\s*schema|srobject)|w(?:ork(?:load)?|rapper)|hi(?:erarchy|stogram)|o(?:perator|utline)|(?:nicknam|queu)e|us(?:age|er)|group|java|view)|union\\s*(?:(?:distin|sele)ct|all))\\b|\\b(?:(?:(?:trunc|cre|upd)at|renam)e|(?:inser|selec)t|de(?:lete|sc)|alter|load)\\s+(?:group_concat|load_file|char)\\b\\s*\\(?|[\\s(]load_file\\s*?\\(|[\\\"'`]\\s+regexp\\W)", @@ -4101,6 +4226,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "(?i:/\\*[!+](?:[\\w\\s=_\\-(?:)]+)?\\*/)", @@ -4143,6 +4271,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "(?i:\\.cookie\\b.*?;\\W*?(?:expires|domain)\\W*?=|\\bhttp-equiv\\W+set-cookie\\b)", @@ -4188,6 +4319,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "java\\.lang\\.(?:runtime|processbuilder)", @@ -4233,6 +4367,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "(?:unmarshaller|base64data|java\\.).*(?:runtime|processbuilder)", @@ -4277,6 +4414,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "list": [ @@ -4312,6 +4452,7 @@ "java.lang.object", "java.lang.process", "java.lang.reflect", + "java.lang.runtime", "java.lang.string", "java.lang.stringbuilder", "java.lang.system", @@ -4321,7 +4462,10 @@ "org.apache.struts2", "org.omg.corba", "java.beans.xmldecode" - ] + ], + "options": { + "enforce_word_boundary": true + } }, "operator": "phrase_match" } @@ -4362,6 +4506,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "(?:class\\.module\\.classLoader\\.resources\\.context\\.parent\\.pipeline|springframework\\.context\\.support\\.FileSystemXmlApplicationContext)", @@ -4403,6 +4550,9 @@ { "address": "graphql.server.all_resolvers" }, + { + "address": "graphql.server.resolver" + }, { "address": "server.request.headers.no_cookies" } @@ -4443,10 +4593,10 @@ "address": "server.request.path_params" }, { - "address": "grpc.server.request.message" + "address": "graphql.server.all_resolvers" }, { - "address": "graphql.server.all_resolvers" + "address": "graphql.server.resolver" }, { "address": "server.request.headers.no_cookies" @@ -4493,6 +4643,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "[@#]ognl", @@ -4639,6 +4792,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "#(?:set|foreach|macro|parse|if)\\(.*\\)|<#assign.*>" @@ -4680,6 +4836,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "\\b(?:burpcollaborator\\.net|oastify\\.com)\\b" @@ -4721,6 +4880,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "\\bqualysperiscope\\.com\\b|\\.oscomm\\." @@ -4762,6 +4924,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "\\bprbly\\.win\\b" @@ -4802,6 +4967,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "\\b(?:webhook\\.site|\\.canarytokens\\.com|vii\\.one|act1on3\\.ru|gdsburp\\.com|arcticwolf\\.net|oob\\.li|htbiw\\.com|h4\\.vc|mochan\\.cloud|imshopping\\.com|bootstrapnodejs\\.com|mooo-ng\\.com|securitytrails\\.com|canyouhackit\\.io|7bae\\.xyz)\\b" @@ -4842,6 +5010,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "\\b(?:\\.ngrok\\.io|requestbin\\.com|requestbin\\.net)\\b" @@ -4883,6 +5054,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "\\bappspidered\\.rapid7\\." @@ -4924,6 +5098,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "\\b(?:interact\\.sh|oast\\.(?:pro|live|site|online|fun|me)|indusfacefinder\\.in|where\\.land|syhunt\\.net|tssrt\\.de|boardofcyber\\.io|assetnote-callback\\.com|praetorianlabs\\.dev|netspi\\.sh)\\b" @@ -4965,6 +5142,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "\\b(?:\\.|(?:\\\\|&#)(?:0*46|x0*2e);)?r87(?:\\.|(?:\\\\|&#)(?:0*46|x0*2e);)(?:me|com)\\b", @@ -5010,6 +5190,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "\\bwhsec(?:\\.|(?:\\\\|&#)(?:0*46|x0*2e);)us\\b", @@ -5055,6 +5238,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "\\b\\.nessus\\.org\\b", @@ -5100,6 +5286,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "\\bwatchtowr\\.com\\b", @@ -5145,6 +5334,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "\\bptst\\.io\\b", @@ -5158,6 +5350,40 @@ ], "transformers": [] }, + { + "id": "dog-920-001", + "name": "JWT authentication bypass", + "tags": { + "type": "http_protocol_violation", + "category": "attack_attempt", + "cwe": "287", + "capec": "1000/225/115", + "confidence": "0" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.cookies" + }, + { + "address": "server.request.headers.no_cookies", + "key_path": [ + "authorization" + ] + } + ], + "regex": "^(?:Bearer )?ey[A-Za-z0-9+_\\-/]*([QY][UW]x[Hn]Ij([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ICJ[Ou][Tb][02]5[Fl]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gOiAi[Tb][km]9[Ou][RZ][Q-Za-f]|[QY][UW]x[Hn]Ij([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]Ai[Tb][km]9[Ou][RZ][Q-Za-f]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]A6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ICJ[Ou][Tb][02]5[Fl]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ciID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ICJ[Ou][Tb][02]5[Fl]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]IDogI[km]5[Pv][Tb][km][U-X]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]A6ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]Ai[Tb][km]9[Ou][RZ][Q-Za-f]|[QY][UW]x[Hn]IiA6ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gI[km]5[Pv][Tb][km][U-X]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gI[km]5[Pv][Tb][km][U-X]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]IDoi[Tb][km]9[Ou][RZ][Q-Za-f]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]A6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]Ai[Tb][km]9[Ou][RZ][Q-Za-f]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]A6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ICJ[Ou][Tb][02]5[Fl]|[QY][UW]x[Hn]Ij([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gOiAi[Tb][km]9[Ou][RZ][Q-Za-f]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]Ai[Tb][km]9[Ou][RZ][Q-Za-f]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ciOiAi[Tb][km]9[Ou][RZ][Q-Za-f]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*IDogI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]IDogI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]yIgO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]Ai[Tb][km]9[Ou][RZ][Q-Za-f]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]A6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]A6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ciIDoi[Tb][km]9[Ou][RZ][Q-Za-f]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*IDogI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]yIgOiJ[Ou][Tb][02]5[Fl]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ciO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gI[km]5[Pv][Tb][km][U-X]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]IDoi[Tb][km]9[Ou][RZ][Q-Za-f]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ciID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]IDogI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]yI6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]yI6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ICJ[Ou][Tb][02]5[Fl]|[QY][UW]x[Hn]IiA6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ICJ[Ou][Tb][02]5[Fl]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]Ai[Tb][km]9[Ou][RZ][Q-Za-f]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*IDogI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]yIgO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ICJ[Ou][Tb][02]5[Fl]|[QY][UW]x[Hn]Ij([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]Ai[Tb][km]9[Ou][RZ][Q-Za-f]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]A6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]A6ICJ[Ou][Tb][02]5[Fl]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]Ai[Tb][km]9[Ou][RZ][Q-Za-f]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ciOiJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*IDoi[Tb][km]9[Ou][RZ][Q-Za-f]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gOiJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ciO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]yIgOiAi[Tb][km]9[Ou][RZ][Q-Za-f]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]IDoi[Tb][km]9[Ou][RZ][Q-Za-f]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ciID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]A6ICJ[Ou][Tb][02]5[Fl]|[QY][UW]x[Hn]IjogI[km]5[Pv][Tb][km][U-X]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ICJ[Ou][Tb][02]5[Fl]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]A6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]Ai[Tb][km]9[Ou][RZ][Q-Za-f]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]Ai[Tb][km]9[Ou][RZ][Q-Za-f]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ciO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]Ai[Tb][km]9[Ou][RZ][Q-Za-f]|[QY][UW]x[Hn]IiA6I[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ciID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]Ai[Tb][km]9[Ou][RZ][Q-Za-f]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]Ai[Tb][km]9[Ou][RZ][Q-Za-f]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]A6I[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]yI6I[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]yI6ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ciIDogI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gI[km]5[Pv][Tb][km][U-X]|[QY][UW]x[Hn]IiA6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]Ai[Tb][km]9[Ou][RZ][Q-Za-f]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]Ai[Tb][km]9[Ou][RZ][Q-Za-f]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]A6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ICJ[Ou][Tb][02]5[Fl]|[QY][UW]x[Hn]IiA6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*IDoi[Tb][km]9[Ou][RZ][Q-Za-f]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ciO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ICJ[Ou][Tb][02]5[Fl]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]A6I[km]5[Pv][Tb][km][U-X]|[QY][UW]x[Hn]IiA6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]A6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]A6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]yI6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]yIgO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gOiJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]A6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]Ai[Tb][km]9[Ou][RZ][Q-Za-f]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]Ai[Tb][km]9[Ou][RZ][Q-Za-f]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]yIgO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ICJ[Ou][Tb][02]5[Fl]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]A6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ICJ[Ou][Tb][02]5[Fl]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*IDoi[Tb][km]9[Ou][RZ][Q-Za-f]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]A6I[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gOiJ[Ou][Tb][02]5[Fl]|[QY][UW]x[Hn]Ijoi[Tb][km]9[Ou][RZ][Q-Za-f]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gOiAi[Tb][km]9[Ou][RZ][Q-Za-f]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]yI6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]Ai[Tb][km]9[Ou][RZ][Q-Za-f])[A-Za-z0-9+-/]*\\.[A-Za-z0-9+_\\-/]+\\.(?:[A-Za-z0-9+_\\-/]+)?$", + "options": { + "case_sensitive": true + } + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, { "id": "dog-931-001", "name": "RFI: URL Payload to well known RFI target", @@ -5186,6 +5412,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "^(?i:file|ftps?|https?).*/rfiinc\\.txt\\?+$", @@ -5230,6 +5459,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "(?:(?:['\"\\x60({|;&]|(?:^|['\"\\x60({|;&])(?:cmd(?:\\.exe)?\\s+(?:/\\w(?::\\w+)?\\s+)*))(?:ping|curl|wget|telnet)|\\bnslookup)[\\s,]", @@ -5265,6 +5497,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "(?:<\\?xml[^>]*>.*)]+SYSTEM\\s+[^>]+>", @@ -5318,6 +5553,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "<(?:iframe|esi:include)(?:(?:\\s|/)*\\w+=[\"'\\w]+)*(?:\\s|/)*src(?:doc)?=[\"']?(?:data:|javascript:|http:|dns:|//)[^\\s'\"]+['\"]?", @@ -5364,6 +5602,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "https?:\\/\\/(?:.*\\.)?(?:bxss\\.(?:in|me)|xss\\.ht|js\\.rip)", @@ -5404,6 +5645,9 @@ { "operator": "phrase_match", "parameters": { + "options": { + "enforce_word_boundary": true + }, "inputs": [ { "address": "server.request.uri.raw" @@ -5604,7 +5848,8 @@ "/website.php", "/stats.php", "/assets/plugins/mp3_id/mp3_id.php", - "/siteminderagent/forms/smpwservices.fcc" + "/siteminderagent/forms/smpwservices.fcc", + "/eval-stdin.php" ] } } @@ -5950,60 +6195,67 @@ "transformers": [] }, { - "id": "sqr-000-001", - "name": "SSRF: Try to access the credential manager of the main cloud services", + "id": "nfd-000-010", + "name": "Detect failed attempts to find API documentation", "tags": { - "type": "ssrf", + "type": "security_scanner", "category": "attack_attempt", - "cwe": "918", - "capec": "1000/225/115/664", - "confidence": "1" + "cwe": "200", + "capec": "1000/118/169", + "confidence": "0" }, "conditions": [ { + "operator": "match_regex", "parameters": { "inputs": [ { - "address": "server.request.query" - }, - { - "address": "server.request.body" - }, - { - "address": "server.request.path_params" - }, - { - "address": "grpc.server.request.message" - }, + "address": "server.response.status" + } + ], + "regex": "^404$", + "options": { + "case_sensitive": true + } + } + }, + { + "operator": "match_regex", + "parameters": { + "inputs": [ { - "address": "graphql.server.all_resolvers" + "address": "server.request.uri.raw" } ], - "regex": "(?i)^\\W*((http|ftp)s?://)?\\W*((::f{4}:)?(169|(0x)?0*a9|0+251)\\.?(254|(0x)?0*fe|0+376)[0-9a-fx\\.:]+|metadata\\.google\\.internal|metadata\\.goog)\\W*/", + "regex": "(?:/swagger\\b|/api[-/]docs?\\b)", "options": { - "min_length": 4 + "case_sensitive": false } - }, - "operator": "match_regex" + } } ], - "transformers": [ - "removeNulls" - ] + "transformers": [] }, { - "id": "sqr-000-002", - "name": "Server-side Javascript injection: Try to detect obvious JS injection", + "id": "rasp-930-100", + "name": "Local file inclusion exploit", "tags": { - "type": "js_code_injection", - "category": "attack_attempt", - "cwe": "94", - "capec": "1000/152/242" + "type": "lfi", + "category": "vulnerability_trigger", + "cwe": "22", + "capec": "1000/255/153/126", + "confidence": "0", + "module": "rasp" }, "conditions": [ { "parameters": { - "inputs": [ + "resource": [ + { + "address": "server.io.fs.file" + } + ], + "params": [ { "address": "server.request.query" }, @@ -6018,12 +6270,249 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } - ], - "regex": "require\\(['\"][\\w\\.]+['\"]\\)|process\\.\\w+\\([\\w\\.]*\\)|\\.toString\\(\\)", - "options": { - "min_length": 4 - } + ] + }, + "operator": "lfi_detector" + } + ], + "transformers": [], + "on_match": [ + "stack_trace" + ] + }, + { + "id": "rasp-932-100", + "name": "Command injection exploit", + "tags": { + "type": "command_injection", + "category": "vulnerability_trigger", + "cwe": "77", + "capec": "1000/152/248/88", + "confidence": "0", + "module": "rasp" + }, + "conditions": [ + { + "parameters": { + "resource": [ + { + "address": "server.sys.shell.cmd" + } + ], + "params": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + }, + { + "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" + } + ] + }, + "operator": "shi_detector" + } + ], + "transformers": [], + "on_match": [ + "stack_trace" + ] + }, + { + "id": "rasp-934-100", + "name": "Server-side request forgery exploit", + "enabled": false, + "tags": { + "type": "ssrf", + "category": "vulnerability_trigger", + "cwe": "918", + "capec": "1000/225/115/664", + "confidence": "0", + "module": "rasp" + }, + "conditions": [ + { + "parameters": { + "resource": [ + { + "address": "server.io.net.url" + } + ], + "params": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + }, + { + "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" + } + ] + }, + "operator": "ssrf_detector" + } + ], + "transformers": [], + "on_match": [ + "stack_trace" + ] + }, + { + "id": "rasp-942-100", + "name": "SQL injection exploit", + "enabled": false, + "tags": { + "type": "sql_injection", + "category": "vulnerability_trigger", + "cwe": "89", + "capec": "1000/152/248/66", + "confidence": "0", + "module": "rasp" + }, + "conditions": [ + { + "parameters": { + "resource": [ + { + "address": "server.db.statement" + } + ], + "params": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" + } + ], + "db_type": [ + { + "address": "server.db.system" + } + ] + }, + "operator": "sqli_detector" + } + ], + "transformers": [], + "on_match": [ + "stack_trace" + ] + }, + { + "id": "sqr-000-001", + "name": "SSRF: Try to access the credential manager of the main cloud services", + "tags": { + "type": "ssrf", + "category": "attack_attempt", + "cwe": "918", + "capec": "1000/225/115/664", + "confidence": "1" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + }, + { + "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" + } + ], + "regex": "(?i)^\\W*((http|ftp)s?://)?\\W*((::f{4}:)?(169|(0x)?0*a9|0+251)\\.?(254|(0x)?0*fe|0+376)[0-9a-fx\\.:]+|metadata\\.google\\.internal|metadata\\.goog)\\W*/", + "options": { + "min_length": 4 + } + }, + "operator": "match_regex" + } + ], + "transformers": [ + "removeNulls" + ] + }, + { + "id": "sqr-000-002", + "name": "Server-side Javascript injection: Try to detect obvious JS injection", + "tags": { + "type": "js_code_injection", + "category": "attack_attempt", + "cwe": "94", + "capec": "1000/152/242" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + }, + { + "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" + } + ], + "regex": "require\\(['\"][\\w\\.]+['\"]\\)|process\\.\\w+\\([\\w\\.]*\\)|\\.toString\\(\\)", + "options": { + "min_length": 4 + } }, "operator": "match_regex" } @@ -6063,6 +6552,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "(?i)[&|]\\s*type\\s+%\\w+%\\\\+\\w+\\.ini\\s*[&|]" @@ -6103,6 +6595,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "(?i)[&|]\\s*cat\\s*\\/etc\\/[\\w\\.\\/]*passwd\\s*[&|]" @@ -6145,6 +6640,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "(?i)[&|]\\s*timeout\\s+/t\\s+\\d+\\s*[&|]" @@ -6182,6 +6680,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "http(s?):\\/\\/([A-Za-z0-9\\.\\-\\_]+|\\[[A-Fa-f0-9\\:]+\\]|):5986\\/wsman", @@ -6222,6 +6723,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "^(jar:)?(http|https):\\/\\/([0-9oq]{1,5}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}|[0-9]{1,10})(:[0-9]{1,5})?(\\/[^:@]*)?$" @@ -6261,6 +6765,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "^(jar:)?(http|https):\\/\\/((\\[)?[:0-9a-f\\.x]{2,}(\\])?)(:[0-9]{1,5})?(\\/[^:@]*)?$" @@ -6303,6 +6810,9 @@ }, { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" } ], "regex": "(http|https):\\/\\/(?:.*\\.)?(?:burpcollaborator\\.net|localtest\\.me|mail\\.ebc\\.apple\\.com|bugbounty\\.dod\\.network|.*\\.[nx]ip\\.io|oastify\\.com|oast\\.(?:pro|live|site|online|fun|me)|sslip\\.io|requestbin\\.com|requestbin\\.net|hookbin\\.com|webhook\\.site|canarytokens\\.com|interact\\.sh|ngrok\\.io|bugbounty\\.click|prbly\\.win|qualysperiscope\\.com|vii\\.one|act1on3\\.ru)" @@ -6339,10 +6849,10 @@ "address": "server.request.headers.no_cookies" }, { - "address": "grpc.server.request.message" + "address": "graphql.server.all_resolvers" }, { - "address": "graphql.server.all_resolvers" + "address": "graphql.server.resolver" } ], "regex": "^(jar:)?((file|netdoc):\\/\\/[\\\\\\/]+|(dict|gopher|ldap|sftp|tftp):\\/\\/.*:[0-9]{1,5})" @@ -6384,10 +6894,10 @@ "address": "server.request.headers.no_cookies" }, { - "address": "grpc.server.request.message" + "address": "graphql.server.all_resolvers" }, { - "address": "graphql.server.all_resolvers" + "address": "graphql.server.resolver" } ], "regex": "\\${[^j]*j[^n]*n[^d]*d[^i]*i[^:]*:[^}]*}" @@ -7923,5 +8433,1349 @@ ], "transformers": [] } + ], + "processors": [ + { + "id": "http-endpoint-fingerprint", + "generator": "http_endpoint_fingerprint", + "conditions": [ + { + "operator": "exists", + "parameters": { + "inputs": [ + { + "address": "waf.context.event" + }, + { + "address": "server.business_logic.users.login.failure" + }, + { + "address": "server.business_logic.users.login.success" + } + ] + } + } + ], + "parameters": { + "mappings": [ + { + "method": [ + { + "address": "server.request.method" + } + ], + "uri_raw": [ + { + "address": "server.request.uri.raw" + } + ], + "body": [ + { + "address": "server.request.body" + } + ], + "query": [ + { + "address": "server.request.query" + } + ], + "output": "_dd.appsec.fp.http.endpoint" + } + ] + }, + "evaluate": false, + "output": true + }, + { + "id": "extract-content", + "generator": "extract_schema", + "conditions": [ + { + "operator": "equals", + "parameters": { + "inputs": [ + { + "address": "waf.context.processor", + "key_path": [ + "extract-schema" + ] + } + ], + "type": "boolean", + "value": true + } + } + ], + "parameters": { + "mappings": [ + { + "inputs": [ + { + "address": "server.request.body" + } + ], + "output": "_dd.appsec.s.req.body" + }, + { + "inputs": [ + { + "address": "server.request.cookies" + } + ], + "output": "_dd.appsec.s.req.cookies" + }, + { + "inputs": [ + { + "address": "server.request.query" + } + ], + "output": "_dd.appsec.s.req.query" + }, + { + "inputs": [ + { + "address": "server.request.path_params" + } + ], + "output": "_dd.appsec.s.req.params" + }, + { + "inputs": [ + { + "address": "server.response.body" + } + ], + "output": "_dd.appsec.s.res.body" + }, + { + "inputs": [ + { + "address": "graphql.server.all_resolvers" + } + ], + "output": "_dd.appsec.s.graphql.all_resolvers" + }, + { + "inputs": [ + { + "address": "graphql.server.resolver" + } + ], + "output": "_dd.appsec.s.graphql.resolver" + } + ], + "scanners": [ + { + "tags": { + "category": "payment" + } + }, + { + "tags": { + "category": "pii" + } + } + ] + }, + "evaluate": false, + "output": true + }, + { + "id": "extract-headers", + "generator": "extract_schema", + "conditions": [ + { + "operator": "equals", + "parameters": { + "inputs": [ + { + "address": "waf.context.processor", + "key_path": [ + "extract-schema" + ] + } + ], + "type": "boolean", + "value": true + } + } + ], + "parameters": { + "mappings": [ + { + "inputs": [ + { + "address": "server.request.headers.no_cookies" + } + ], + "output": "_dd.appsec.s.req.headers" + }, + { + "inputs": [ + { + "address": "server.response.headers.no_cookies" + } + ], + "output": "_dd.appsec.s.res.headers" + } + ], + "scanners": [ + { + "tags": { + "category": "credentials" + } + }, + { + "tags": { + "category": "pii" + } + } + ] + }, + "evaluate": false, + "output": true + }, + { + "id": "http-header-fingerprint", + "generator": "http_header_fingerprint", + "conditions": [ + { + "operator": "exists", + "parameters": { + "inputs": [ + { + "address": "waf.context.event" + }, + { + "address": "server.business_logic.users.login.failure" + }, + { + "address": "server.business_logic.users.login.success" + } + ] + } + } + ], + "parameters": { + "mappings": [ + { + "headers": [ + { + "address": "server.request.headers.no_cookies" + } + ], + "output": "_dd.appsec.fp.http.header" + } + ] + }, + "evaluate": false, + "output": true + }, + { + "id": "http-network-fingerprint", + "generator": "http_network_fingerprint", + "conditions": [ + { + "operator": "exists", + "parameters": { + "inputs": [ + { + "address": "waf.context.event" + }, + { + "address": "server.business_logic.users.login.failure" + }, + { + "address": "server.business_logic.users.login.success" + } + ] + } + } + ], + "parameters": { + "mappings": [ + { + "headers": [ + { + "address": "server.request.headers.no_cookies" + } + ], + "output": "_dd.appsec.fp.http.network" + } + ] + }, + "evaluate": false, + "output": true + }, + { + "id": "session-fingerprint", + "generator": "session_fingerprint", + "conditions": [ + { + "operator": "exists", + "parameters": { + "inputs": [ + { + "address": "waf.context.event" + }, + { + "address": "server.business_logic.users.login.failure" + }, + { + "address": "server.business_logic.users.login.success" + } + ] + } + } + ], + "parameters": { + "mappings": [ + { + "cookies": [ + { + "address": "server.request.cookies" + } + ], + "session_id": [ + { + "address": "usr.session_id" + } + ], + "user_id": [ + { + "address": "usr.id" + } + ], + "output": "_dd.appsec.fp.session" + } + ] + }, + "evaluate": false, + "output": true + } + ], + "scanners": [ + { + "id": "406f8606-52c4-4663-8db9-df70f9e8766c", + "name": "ZIP Code", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:zip|postal)\\b", + "options": { + "case_sensitive": false, + "min_length": 3 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "^[0-9]{5}(?:-[0-9]{4})?$", + "options": { + "case_sensitive": true, + "min_length": 5 + } + } + }, + "tags": { + "type": "zipcode", + "category": "address" + } + }, + { + "id": "JU1sRk3mSzqSUJn6GrVn7g", + "name": "American Express Card Scanner (4+4+4+3 digits)", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:card|cc|credit|debit|payment|amex|visa|mastercard|maestro|discover|jcb|diner|amex|visa|mastercard|maestro|discover|jcb|diner)\\b", + "options": { + "case_sensitive": false, + "min_length": 3 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "\\b3[47]\\d{2}(?:(?:\\s\\d{4}\\s\\d{4}\\s\\d{3})|(?:\\,\\d{4}\\,\\d{4}\\,\\d{3})|(?:-\\d{4}-\\d{4}-\\d{3})|(?:\\.\\d{4}\\.\\d{4}\\.\\d{3}))\\b", + "options": { + "case_sensitive": false, + "min_length": 16 + } + } + }, + "tags": { + "type": "card", + "card_type": "amex", + "category": "payment" + } + }, + { + "id": "edmH513UTQWcRiQ9UnzHlw-mod", + "name": "American Express Card Scanner (4+6|5+5|6 digits)", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:card|cc|credit|debit|payment|amex|visa|mastercard|maestro|discover|jcb|diner)\\b", + "options": { + "case_sensitive": false, + "min_length": 3 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "\\b3[47]\\d{2}(?:(?:\\s\\d{5,6}\\s\\d{5,6})|(?:\\.\\d{5,6}\\.\\d{5,6})|(?:-\\d{5,6}-\\d{5,6})|(?:,\\d{5,6},\\d{5,6}))\\b", + "options": { + "case_sensitive": false, + "min_length": 17 + } + } + }, + "tags": { + "type": "card", + "card_type": "amex", + "category": "payment" + } + }, + { + "id": "e6K4h_7qTLaMiAbaNXoSZA", + "name": "American Express Card Scanner (8+7 digits)", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:card|cc|credit|debit|payment|amex|visa|mastercard|maestro|discover|jcb|diner)\\b", + "options": { + "case_sensitive": false, + "min_length": 3 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "\\b3[47]\\d{6}(?:(?:\\s\\d{7})|(?:\\,\\d{7})|(?:-\\d{7})|(?:\\.\\d{7}))\\b", + "options": { + "case_sensitive": false, + "min_length": 16 + } + } + }, + "tags": { + "type": "card", + "card_type": "amex", + "category": "payment" + } + }, + { + "id": "K2rZflWzRhGM9HiTc6whyQ", + "name": "American Express Card Scanner (1x15 digits)", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:card|cc|credit|debit|payment|amex|visa|mastercard|maestro|discover|jcb|diner)\\b", + "options": { + "case_sensitive": false, + "min_length": 3 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "\\b3[47]\\d{13}\\b", + "options": { + "case_sensitive": false, + "min_length": 15 + } + } + }, + "tags": { + "type": "card", + "card_type": "amex", + "category": "payment" + } + }, + { + "id": "9d7756e343cefa22a5c098e1092590f806eb5446", + "name": "Basic Authentication Scanner", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "\\bauthorization\\b", + "options": { + "case_sensitive": false, + "min_length": 13 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "^basic\\s+[A-Za-z0-9+/=]+", + "options": { + "case_sensitive": false, + "min_length": 7 + } + } + }, + "tags": { + "type": "basic_auth", + "category": "credentials" + } + }, + { + "id": "mZy8XjZLReC9smpERXWnnw", + "name": "Bearer Authentication Scanner", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "\\bauthorization\\b", + "options": { + "case_sensitive": false, + "min_length": 13 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "^bearer\\s+[-a-z0-9._~+/]{4,}", + "options": { + "case_sensitive": false, + "min_length": 11 + } + } + }, + "tags": { + "type": "bearer_token", + "category": "credentials" + } + }, + { + "id": "450239afc250a19799b6c03dc0e16fd6a4b2a1af", + "name": "Canadian Social Insurance Number Scanner", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:social[\\s_]?(?:insurance(?:\\s+number)?)?|SIN|Canadian[\\s_]?(?:social[\\s_]?(?:insurance)?|insurance[\\s_]?number)?)\\b", + "options": { + "case_sensitive": false, + "min_length": 3 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "\\b\\d{3}-\\d{3}-\\d{3}\\b", + "options": { + "case_sensitive": false, + "min_length": 11 + } + } + }, + "tags": { + "type": "canadian_sin", + "category": "pii" + } + }, + { + "id": "87a879ff33693b46c8a614d8211f5a2c289beca0", + "name": "Digest Authentication Scanner", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "\\bauthorization\\b", + "options": { + "case_sensitive": false, + "min_length": 13 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "^digest\\s+", + "options": { + "case_sensitive": false, + "min_length": 7 + } + } + }, + "tags": { + "type": "digest_auth", + "category": "credentials" + } + }, + { + "id": "qWumeP1GQUa_E4ffAnT-Yg", + "name": "American Express Card Scanner (1x14 digits)", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:card|cc|credit|debit|payment|amex|visa|mastercard|maestro|discover|jcb|diner)\\b", + "options": { + "case_sensitive": false, + "min_length": 3 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "(?:30[0-59]\\d|3[689]\\d{2})(?:\\d{10})", + "options": { + "case_sensitive": false, + "min_length": 14 + } + } + }, + "tags": { + "type": "card", + "card_type": "diners", + "category": "payment" + } + }, + { + "id": "NlTWWM5LS6W0GSqBLuvtRw", + "name": "Diners Card Scanner (4+4+4+2 digits)", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:card|cc|credit|debit|payment|amex|visa|mastercard|maestro|discover|jcb|diner)\\b", + "options": { + "case_sensitive": false, + "min_length": 3 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:30[0-59]\\d|3[689]\\d{2})(?:(?:\\s\\d{4}\\s\\d{4}\\s\\d{2})|(?:\\,\\d{4}\\,\\d{4}\\,\\d{2})|(?:-\\d{4}-\\d{4}-\\d{2})|(?:\\.\\d{4}\\.\\d{4}\\.\\d{2}))\\b", + "options": { + "case_sensitive": false, + "min_length": 17 + } + } + }, + "tags": { + "type": "card", + "card_type": "diners", + "category": "payment" + } + }, + { + "id": "Xr5VdbQSTXitYGGiTfxBpw", + "name": "Diners Card Scanner (4+6+4 digits)", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:card|cc|credit|debit|payment|amex|visa|mastercard|maestro|discover|jcb|diner)\\b", + "options": { + "case_sensitive": false, + "min_length": 3 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:30[0-59]\\d|3[689]\\d{2})(?:(?:\\s\\d{6}\\s\\d{4})|(?:\\.\\d{6}\\.\\d{4})|(?:-\\d{6}-\\d{4})|(?:,\\d{6},\\d{4}))\\b", + "options": { + "case_sensitive": false, + "min_length": 16 + } + } + }, + "tags": { + "type": "card", + "card_type": "diners", + "category": "payment" + } + }, + { + "id": "gAbunN_WQNytxu54DjcbAA-mod", + "name": "Diners Card Scanner (8+6 digits)", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:card|cc|credit|debit|payment|amex|visa|mastercard|maestro|discover|jcb|diner)\\b", + "options": { + "case_sensitive": false, + "min_length": 3 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:30[0-59]\\d{5}|3[689]\\d{6})\\s?(?:(?:\\s\\d{6})|(?:\\,\\d{6})|(?:-\\d{6})|(?:\\.\\d{6}))\\b", + "options": { + "case_sensitive": false, + "min_length": 14 + } + } + }, + "tags": { + "type": "card", + "card_type": "diners", + "category": "payment" + } + }, + { + "id": "9cs4qCfEQBeX17U7AepOvQ", + "name": "MasterCard Scanner (2x8 digits)", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:card|cc|credit|debit|payment|amex|visa|mastercard|maestro|discover|jcb|diner)\\b", + "options": { + "case_sensitive": false, + "min_length": 3 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:6221(?:2[6-9]|[3-9][0-9])\\d{2}(?:,\\d{8}|\\s\\d{8}|-\\d{8}|\\.\\d{8})|6229(?:[01][0-9]|2[0-5])\\d{2}(?:,\\d{8}|\\s\\d{8}|-\\d{8}|\\.\\d{8})|(?:6011|65\\d{2}|64[4-9]\\d|622[2-8])\\d{4}(?:,\\d{8}|\\s\\d{8}|-\\d{8}|\\.\\d{8}))\\b", + "options": { + "case_sensitive": false, + "min_length": 16 + } + } + }, + "tags": { + "type": "card", + "card_type": "discover", + "category": "payment" + } + }, + { + "id": "YBIDWJIvQWW_TFOyU0CGJg", + "name": "Discover Card Scanner (4x4 digits)", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:card|cc|credit|debit|payment|amex|visa|mastercard|maestro|discover|jcb|diner)\\b", + "options": { + "case_sensitive": false, + "min_length": 3 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:(?:(?:6221(?:2[6-9]|[3-9][0-9])\\d{2}(?:,\\d{4}){2})|(?:6221\\s(?:2[6-9]|[3-9][0-9])\\d{2}(?:\\s\\d{4}){2})|(?:6221\\.(?:2[6-9]|[3-9][0-9])\\d{2}(?:\\.\\d{4}){2})|(?:6221-(?:2[6-9]|[3-9][0-9])\\d{2}(?:-\\d{4}){2}))|(?:(?:6229(?:[01][0-9]|2[0-5])\\d{2}(?:,\\d{4}){2})|(?:6229\\s(?:[01][0-9]|2[0-5])\\d{2}(?:\\s\\d{4}){2})|(?:6229\\.(?:[01][0-9]|2[0-5])\\d{2}(?:\\.\\d{4}){2})|(?:6229-(?:[01][0-9]|2[0-5])\\d{2}(?:-\\d{4}){2}))|(?:(?:6011|65\\d{2}|64[4-9]\\d|622[2-8])(?:(?:\\s\\d{4}){3}|(?:\\.\\d{4}){3}|(?:-\\d{4}){3}|(?:,\\d{4}){3})))\\b", + "options": { + "case_sensitive": false, + "min_length": 16 + } + } + }, + "tags": { + "type": "card", + "card_type": "discover", + "category": "payment" + } + }, + { + "id": "12cpbjtVTMaMutFhh9sojQ", + "name": "Discover Card Scanner (1x16 digits)", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:card|cc|credit|debit|payment|amex|visa|mastercard|maestro|discover|jcb|diner)\\b", + "options": { + "case_sensitive": false, + "min_length": 3 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:6221(?:2[6-9]|[3-9][0-9])\\d{10}|6229(?:[01][0-9]|2[0-5])\\d{10}|(?:6011|65\\d{2}|64[4-9]\\d|622[2-8])\\d{12})\\b", + "options": { + "case_sensitive": false, + "min_length": 16 + } + } + }, + "tags": { + "type": "card", + "card_type": "discover", + "category": "payment" + } + }, + { + "id": "PuXiVTCkTHOtj0Yad1ppsw", + "name": "Standard E-mail Address", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:(?:e[-\\s]?)?mail|address|sender|\\bto\\b|from|recipient)\\b", + "options": { + "case_sensitive": false, + "min_length": 2 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "\\b[\\w!#$%&'*+/=?`{|}~^-]+(?:\\.[\\w!#$%&'*+/=?`{|}~^-]+)*(%40|@)(?:[a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,6}\\b", + "options": { + "case_sensitive": false, + "min_length": 5 + } + } + }, + "tags": { + "type": "email", + "category": "pii" + } + }, + { + "id": "8VS2RKxzR8a_95L5fuwaXQ", + "name": "IBAN", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:iban|account|sender|receiver)\\b", + "options": { + "case_sensitive": false, + "min_length": 3 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:NO\\d{2}(?:[ \\-]?\\d{4}){2}[ \\-]?\\d{3}|BE\\d{2}(?:[ \\-]?\\d{4}){3}|(?:DK|FO|FI|GL|SD)\\d{2}(?:[ \\-]?\\d{4}){3}[ \\-]?\\d{2}|NL\\d{2}[ \\-]?[A-Z]{4}(?:[ \\-]?\\d{4}){2}[ \\-]?\\d{2}|MK\\d{2}[ \\-]?\\d{3}[A-Z0-9](?:[ \\-]?[A-Z0-9]{4}){2}[ \\-]?[A-Z0-9]\\d{2}|SI\\d{17}|(?:AT|BA|EE|LT|XK)\\d{18}|(?:LU|KZ|EE|LT)\\d{5}[A-Z0-9]{13}|LV\\d{2}[A-Z]{4}[A-Z0-9]{13}|(?:LI|CH)\\d{2}[ \\-]?\\d{4}[ \\-]?\\d[A-Z0-9]{3}(?:[ \\-]?[A-Z0-9]{4}){2}[ \\-]?[A-Z0-9]|HR\\d{2}(?:[ \\-]?\\d{4}){4}[ \\-]?\\d|GE\\d{2}[ \\-]?[A-Z0-9]{2}\\d{2}\\d{14}|VA\\d{20}|BG\\d{2}[A-Z]{4}\\d{6}[A-Z0-9]{8}|BH\\d{2}[A-Z]{4}[A-Z0-9]{14}|GB\\d{2}[A-Z]{4}(?:[ \\-]?\\d{4}){3}[ \\-]?\\d{2}|IE\\d{2}[ \\-]?[A-Z0-9]{4}(?:[ \\-]?\\d{4}){3}[ \\-]?\\d{2}|(?:CR|DE|ME|RS)\\d{2}(?:[ \\-]?\\d{4}){4}[ \\-]?\\d{2}|(?:AE|TL|IL)\\d{2}(?:[ \\-]?\\d{4}){4}[ \\-]?\\d{3}|GI\\d{2}[ \\-]?[A-Z]{4}(?:[ \\-]?[A-Z0-9]{4}){3}[ \\-]?[A-Z0-9]{3}|IQ\\d{2}[ \\-]?[A-Z]{4}(?:[ \\-]?\\d{4}){3}[ \\-]?\\d{3}|MD\\d{2}(?:[ \\-]?[A-Z0-9]{4}){5}|SA\\d{2}[ \\-]?\\d{2}[A-Z0-9]{2}(?:[ \\-]?[A-Z0-9]{4}){4}|RO\\d{2}[ \\-]?[A-Z]{4}(?:[ \\-]?[A-Z0-9]{4}){4}|(?:PK|VG)\\d{2}[ \\-]?[A-Z0-9]{4}(?:[ \\-]?\\d{4}){4}|AD\\d{2}(?:[ \\-]?\\d{4}){2}(?:[ \\-]?[A-Z0-9]{4}){3}|(?:CZ|SK|ES|SE|TN)\\d{2}(?:[ \\-]?\\d{4}){5}|(?:LY|PT|ST)\\d{2}(?:[ \\-]?\\d{4}){5}[ \\-]?\\d|TR\\d{2}[ \\-]?\\d{4}[ \\-]?\\d[A-Z0-9]{3}(?:[ \\-]?[A-Z0-9]{4}){3}[ \\-]?[A-Z0-9]{2}|IS\\d{2}(?:[ \\-]?\\d{4}){5}[ \\-]?\\d{2}|(?:IT|SM)\\d{2}[ \\-]?[A-Z]\\d{3}[ \\-]?\\d{4}[ \\-]?\\d{3}[A-Z0-9](?:[ \\-]?[A-Z0-9]{4}){2}[ \\-]?[A-Z0-9]{3}|GR\\d{2}[ \\-]?\\d{4}[ \\-]?\\d{3}[A-Z0-9](?:[ \\-]?[A-Z0-9]{4}){3}[A-Z0-9]{3}|(?:FR|MC)\\d{2}(?:[ \\-]?\\d{4}){2}[ \\-]?\\d{2}[A-Z0-9]{2}(?:[ \\-]?[A-Z0-9]{4}){2}[ \\-]?[A-Z0-9]\\d{2}|MR\\d{2}(?:[ \\-]?\\d{4}){5}[ \\-]?\\d{3}|(?:SV|DO)\\d{2}[ \\-]?[A-Z]{4}(?:[ \\-]?\\d{4}){5}|BY\\d{2}[ \\-]?[A-Z]{4}[ \\-]?\\d{4}(?:[ \\-]?[A-Z0-9]{4}){4}|GT\\d{2}(?:[ \\-]?[A-Z0-9]{4}){6}|AZ\\d{2}[ \\-]?[A-Z0-9]{4}(?:[ \\-]?\\d{5}){4}|LB\\d{2}[ \\-]?\\d{4}(?:[ \\-]?[A-Z0-9]{5}){4}|(?:AL|CY)\\d{2}(?:[ \\-]?\\d{4}){2}(?:[ \\-]?[A-Z0-9]{4}){4}|(?:HU|PL)\\d{2}(?:[ \\-]?\\d{4}){6}|QA\\d{2}[ \\-]?[A-Z]{4}(?:[ \\-]?[A-Z0-9]{4}){5}[ \\-]?[A-Z0-9]|PS\\d{2}[ \\-]?[A-Z0-9]{4}(?:[ \\-]?\\d{4}){5}[ \\-]?\\d|UA\\d{2}[ \\-]?\\d{4}[ \\-]?\\d{2}[A-Z0-9]{2}(?:[ \\-]?[A-Z0-9]{4}){4}[ \\-]?[A-Z0-9]|BR\\d{2}(?:[ \\-]?\\d{4}){5}[ \\-]?\\d{3}[A-Z0-9][ \\-]?[A-Z0-9]|EG\\d{2}(?:[ \\-]?\\d{4}){6}\\d|MU\\d{2}[ \\-]?[A-Z]{4}(?:[ \\-]?\\d{4}){4}\\d{3}[A-Z][ \\-]?[A-Z]{2}|(?:KW|JO)\\d{2}[ \\-]?[A-Z]{4}(?:[ \\-]?[A-Z0-9]{4}){5}[ \\-]?[A-Z0-9]{2}|MT\\d{2}[ \\-]?[A-Z]{4}[ \\-]?\\d{4}[ \\-]?\\d[A-Z0-9]{3}(?:[ \\-]?[A-Z0-9]{3}){4}[ \\-]?[A-Z0-9]{3}|SC\\d{2}[ \\-]?[A-Z]{4}(?:[ \\-]?\\d{4}){5}[ \\-]?[A-Z]{3}|LC\\d{2}[ \\-]?[A-Z]{4}(?:[ \\-]?[A-Z0-9]{4}){6})\\b", + "options": { + "case_sensitive": false, + "min_length": 15 + } + } + }, + "tags": { + "type": "iban", + "category": "payment" + } + }, + { + "id": "h6WJcecQTwqvN9KeEtwDvg", + "name": "JCB Card Scanner (1x16 digits)", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:card|cc|credit|debit|payment|amex|visa|mastercard|maestro|discover|jcb|diner)\\b", + "options": { + "case_sensitive": false, + "min_length": 3 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "\\b35(?:2[89]|[3-9][0-9])(?:\\d{12})\\b", + "options": { + "case_sensitive": false, + "min_length": 16 + } + } + }, + "tags": { + "type": "card", + "card_type": "jcb", + "category": "payment" + } + }, + { + "id": "gcEaMu_VSJ2-bGCEkgyC0w", + "name": "JCB Card Scanner (2x8 digits)", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:card|cc|credit|debit|payment|amex|visa|mastercard|maestro|discover|jcb|diner)\\b", + "options": { + "case_sensitive": false, + "min_length": 3 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "\\b35(?:2[89]|[3-9][0-9])\\d{4}(?:(?:,\\d{8})|(?:-\\d{8})|(?:\\s\\d{8})|(?:\\.\\d{8}))\\b", + "options": { + "case_sensitive": false, + "min_length": 17 + } + } + }, + "tags": { + "type": "card", + "card_type": "jcb", + "category": "payment" + } + }, + { + "id": "imTliuhXT5GAeRNhqChXQQ", + "name": "JCB Card Scanner (4x4 digits)", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:card|cc|credit|debit|payment|amex|visa|mastercard|maestro|discover|jcb|diner)\\b", + "options": { + "case_sensitive": false, + "min_length": 3 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "\\b35(?:2[89]|[3-9][0-9])(?:(?:\\s\\d{4}){3}|(?:\\.\\d{4}){3}|(?:-\\d{4}){3}|(?:,\\d{4}){3})\\b", + "options": { + "case_sensitive": false, + "min_length": 16 + } + } + }, + "tags": { + "type": "card", + "card_type": "jcb", + "category": "payment" + } + }, + { + "id": "9osY3xc9Q7ONAV0zw9Uz4A", + "name": "JSON Web Token", + "value": { + "operator": "match_regex", + "parameters": { + "regex": "\\bey[I-L][\\w=-]+\\.ey[I-L][\\w=-]+(\\.[\\w.+\\/=-]+)?\\b", + "options": { + "case_sensitive": false, + "min_length": 20 + } + } + }, + "tags": { + "type": "json_web_token", + "category": "credentials" + } + }, + { + "id": "d1Q9D3YMRxuVKf6CZInJPw", + "name": "Maestro Card Scanner (1x16 digits)", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:card|cc|credit|debit|payment|amex|visa|mastercard|maestro|discover|jcb|diner)\\b", + "options": { + "case_sensitive": false, + "min_length": 3 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:5[06-9]\\d{2}|6\\d{3})(?:\\d{12})\\b", + "options": { + "case_sensitive": false, + "min_length": 16 + } + } + }, + "tags": { + "type": "card", + "card_type": "maestro", + "category": "payment" + } + }, + { + "id": "M3YIQKKjRVmoeQuM3pjzrw", + "name": "Maestro Card Scanner (2x8 digits)", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:card|cc|credit|debit|payment|amex|visa|mastercard|maestro|discover|jcb|diner)\\b", + "options": { + "case_sensitive": false, + "min_length": 3 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:5[06-9]\\d{6}|6\\d{7})(?:\\s\\d{8}|\\.\\d{8}|-\\d{8}|,\\d{8})\\b", + "options": { + "case_sensitive": false, + "min_length": 17 + } + } + }, + "tags": { + "type": "card", + "card_type": "maestro", + "category": "payment" + } + }, + { + "id": "hRxiQBlSSVKcjh5U7LZYLA", + "name": "Maestro Card Scanner (4x4 digits)", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:card|cc|credit|debit|payment|amex|visa|mastercard|maestro|discover|jcb|diner)\\b", + "options": { + "case_sensitive": false, + "min_length": 3 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:5[06-9]\\d{2}|6\\d{3})(?:(?:\\s\\d{4}){3}|(?:\\.\\d{4}){3}|(?:-\\d{4}){3}|(?:,\\d{4}){3})\\b", + "options": { + "case_sensitive": false, + "min_length": 16 + } + } + }, + "tags": { + "type": "card", + "card_type": "maestro", + "category": "payment" + } + }, + { + "id": "NwhIYNS4STqZys37WlaIKA", + "name": "MasterCard Scanner (2x8 digits)", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:card|cc|credit|debit|payment|amex|visa|mastercard|maestro|discover|jcb|diner)\\b", + "options": { + "case_sensitive": false, + "min_length": 3 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:(?:5[1-5]\\d{2})|(?:222[1-9])|(?:22[3-9]\\d)|(?:2[3-6]\\d{2})|(?:27[0-1]\\d)|(?:2720))(?:(?:\\d{4}(?:(?:,\\d{8})|(?:-\\d{8})|(?:\\s\\d{8})|(?:\\.\\d{8}))))\\b", + "options": { + "case_sensitive": false, + "min_length": 16 + } + } + }, + "tags": { + "type": "card", + "card_type": "mastercard", + "category": "payment" + } + }, + { + "id": "axxJkyjhRTOuhjwlsA35Vw", + "name": "MasterCard Scanner (4x4 digits)", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:card|cc|credit|debit|payment|amex|visa|mastercard|maestro|discover|jcb|diner)\\b", + "options": { + "case_sensitive": false, + "min_length": 3 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:(?:5[1-5]\\d{2})|(?:222[1-9])|(?:22[3-9]\\d)|(?:2[3-6]\\d{2})|(?:27[0-1]\\d)|(?:2720))(?:(?:\\s\\d{4}){3}|(?:\\.\\d{4}){3}|(?:-\\d{4}){3}|(?:,\\d{4}){3})\\b", + "options": { + "case_sensitive": false, + "min_length": 16 + } + } + }, + "tags": { + "type": "card", + "card_type": "mastercard", + "category": "payment" + } + }, + { + "id": "76EhmoK3TPqJcpM-fK0pLw", + "name": "MasterCard Scanner (1x16 digits)", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:card|cc|credit|debit|payment|amex|visa|mastercard|maestro|discover|jcb|diner)\\b", + "options": { + "case_sensitive": false, + "min_length": 3 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:(?:5[1-5]\\d{2})|(?:222[1-9])|(?:22[3-9]\\d)|(?:2[3-6]\\d{2})|(?:27[0-1]\\d)|(?:2720))(?:\\d{12})\\b", + "options": { + "case_sensitive": false, + "min_length": 16 + } + } + }, + "tags": { + "type": "card", + "card_type": "mastercard", + "category": "payment" + } + }, + { + "id": "18b608bd7a764bff5b2344c0", + "name": "Phone number", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "\\bphone|number|mobile\\b", + "options": { + "case_sensitive": false, + "min_length": 3 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "^(?:\\(\\+\\d{1,3}\\)|\\+\\d{1,3}|00\\d{1,3})?[-\\s\\.]?(?:\\(\\d{3}\\)[-\\s\\.]?)?(?:\\d[-\\s\\.]?){6,10}$", + "options": { + "case_sensitive": false, + "min_length": 6 + } + } + }, + "tags": { + "type": "phone", + "category": "pii" + } + }, + { + "id": "de0899e0cbaaa812bb624cf04c912071012f616d-mod", + "name": "UK National Insurance Number Scanner", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "^nin$|\\binsurance\\b", + "options": { + "case_sensitive": false, + "min_length": 3 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "\\b[A-Z]{2}[\\s-]?\\d{6}[\\s-]?[A-Z]?\\b", + "options": { + "case_sensitive": false, + "min_length": 8 + } + } + }, + "tags": { + "type": "uk_nin", + "category": "pii" + } + }, + { + "id": "d962f7ddb3f55041e39195a60ff79d4814a7c331", + "name": "US Passport Scanner", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "\\bpassport\\b", + "options": { + "case_sensitive": false, + "min_length": 8 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "\\b[0-9A-Z]{9}\\b|\\b[0-9]{6}[A-Z][0-9]{2}\\b", + "options": { + "case_sensitive": false, + "min_length": 8 + } + } + }, + "tags": { + "type": "passport_number", + "category": "pii" + } + }, + { + "id": "7771fc3b-b205-4b93-bcef-28608c5c1b54", + "name": "United States Social Security Number Scanner", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:SSN|(?:(?:social)?[\\s_]?(?:security)?[\\s_]?(?:number)?)?)\\b", + "options": { + "case_sensitive": false, + "min_length": 3 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "\\b\\d{3}[-\\s\\.]{1}\\d{2}[-\\s\\.]{1}\\d{4}\\b", + "options": { + "case_sensitive": false, + "min_length": 11 + } + } + }, + "tags": { + "type": "us_ssn", + "category": "pii" + } + }, + { + "id": "ac6d683cbac77f6e399a14990793dd8fd0fca333", + "name": "US Vehicle Identification Number Scanner", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:vehicle[_\\s-]*identification[_\\s-]*number|vin)\\b", + "options": { + "case_sensitive": false, + "min_length": 3 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "\\b[A-HJ-NPR-Z0-9]{17}\\b", + "options": { + "case_sensitive": false, + "min_length": 17 + } + } + }, + "tags": { + "type": "vin", + "category": "pii" + } + }, + { + "id": "wJIgOygRQhKkR69b_9XbRQ", + "name": "Visa Card Scanner (2x8 digits)", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:card|cc|credit|debit|payment|amex|visa|mastercard|maestro|discover|jcb|diner)\\b", + "options": { + "case_sensitive": false, + "min_length": 3 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "\\b4\\d{3}(?:(?:\\d{4}(?:(?:,\\d{8})|(?:-\\d{8})|(?:\\s\\d{8})|(?:\\.\\d{8}))))\\b", + "options": { + "case_sensitive": false, + "min_length": 16 + } + } + }, + "tags": { + "type": "card", + "card_type": "visa", + "category": "payment" + } + }, + { + "id": "0o71SJxXQNK7Q6gMbBesFQ", + "name": "Visa Card Scanner (4x4 digits)", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:card|cc|credit|debit|payment|amex|visa|mastercard|maestro|discover|jcb|diner)\\b", + "options": { + "case_sensitive": false, + "min_length": 3 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "\\b4\\d{3}(?:(?:,\\d{4}){3}|(?:\\s\\d{4}){3}|(?:\\.\\d{4}){3}|(?:-\\d{4}){3})\\b", + "options": { + "case_sensitive": false, + "min_length": 16 + } + } + }, + "tags": { + "type": "card", + "card_type": "visa", + "category": "payment" + } + }, + { + "id": "QrHD6AfgQm6z-j0wStxTvA", + "name": "Visa Card Scanner (1x15 & 1x16 & 1x19 digits)", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:card|cc|credit|debit|payment|amex|visa|mastercard|maestro|discover|jcb|diner)\\b", + "options": { + "case_sensitive": false, + "min_length": 3 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "4[0-9]{12}(?:[0-9]{3})?", + "options": { + "case_sensitive": false, + "min_length": 13 + } + } + }, + "tags": { + "type": "card", + "card_type": "visa", + "category": "payment" + } + } ] } diff --git a/packages/dd-trace/src/appsec/remote_config/capabilities.js b/packages/dd-trace/src/appsec/remote_config/capabilities.js index 94141438db5..3eda140a986 100644 --- a/packages/dd-trace/src/appsec/remote_config/capabilities.js +++ b/packages/dd-trace/src/appsec/remote_config/capabilities.js @@ -6,8 +6,22 @@ module.exports = { ASM_DD_RULES: 1n << 3n, ASM_EXCLUSIONS: 1n << 4n, ASM_REQUEST_BLOCKING: 1n << 5n, + ASM_RESPONSE_BLOCKING: 1n << 6n, ASM_USER_BLOCKING: 1n << 7n, ASM_CUSTOM_RULES: 1n << 8n, ASM_CUSTOM_BLOCKING_RESPONSE: 1n << 9n, - ASM_TRUSTED_IPS: 1n << 10n + ASM_TRUSTED_IPS: 1n << 10n, + ASM_API_SECURITY_SAMPLE_RATE: 1n << 11n, + APM_TRACING_SAMPLE_RATE: 1n << 12n, + APM_TRACING_LOGS_INJECTION: 1n << 13n, + APM_TRACING_HTTP_HEADER_TAGS: 1n << 14n, + APM_TRACING_CUSTOM_TAGS: 1n << 15n, + APM_TRACING_ENABLED: 1n << 19n, + ASM_RASP_SQLI: 1n << 21n, + ASM_RASP_LFI: 1n << 22n, + ASM_RASP_SSRF: 1n << 23n, + APM_TRACING_SAMPLE_RULES: 1n << 29n, + ASM_ENDPOINT_FINGERPRINT: 1n << 32n, + ASM_NETWORK_FINGERPRINT: 1n << 34n, + ASM_HEADER_FINGERPRINT: 1n << 35n } diff --git a/packages/dd-trace/src/appsec/remote_config/index.js b/packages/dd-trace/src/appsec/remote_config/index.js index f7c6118598a..2b7eea57c82 100644 --- a/packages/dd-trace/src/appsec/remote_config/index.js +++ b/packages/dd-trace/src/appsec/remote_config/index.js @@ -1,40 +1,67 @@ 'use strict' +const Activation = require('../activation') + const RemoteConfigManager = require('./manager') const RemoteConfigCapabilities = require('./capabilities') +const apiSecuritySampler = require('../api_security_sampler') let rc -function enable (config) { +function enable (config, appsec) { rc = new RemoteConfigManager(config) + rc.updateCapabilities(RemoteConfigCapabilities.APM_TRACING_CUSTOM_TAGS, true) + rc.updateCapabilities(RemoteConfigCapabilities.APM_TRACING_HTTP_HEADER_TAGS, true) + rc.updateCapabilities(RemoteConfigCapabilities.APM_TRACING_LOGS_INJECTION, true) + rc.updateCapabilities(RemoteConfigCapabilities.APM_TRACING_SAMPLE_RATE, true) + rc.updateCapabilities(RemoteConfigCapabilities.APM_TRACING_ENABLED, true) + rc.updateCapabilities(RemoteConfigCapabilities.APM_TRACING_SAMPLE_RULES, true) + + const activation = Activation.fromConfig(config) - if (config.appsec.enabled === undefined) { // only activate ASM_FEATURES when conf is not set locally - rc.updateCapabilities(RemoteConfigCapabilities.ASM_ACTIVATION, true) + if (activation !== Activation.DISABLED) { + if (activation === Activation.ONECLICK) { + rc.updateCapabilities(RemoteConfigCapabilities.ASM_ACTIVATION, true) + } - rc.on('ASM_FEATURES', (action, conf) => { - if (conf && conf.asm && typeof conf.asm.enabled === 'boolean') { - let shouldEnable + if (config.appsec.apiSecurity?.enabled) { + rc.updateCapabilities(RemoteConfigCapabilities.ASM_API_SECURITY_SAMPLE_RATE, true) + } - if (action === 'apply' || action === 'modify') { - shouldEnable = conf.asm.enabled // take control - } else { - shouldEnable = config.appsec.enabled // give back control to local config - } + rc.setProductHandler('ASM_FEATURES', (action, rcConfig) => { + if (!rcConfig) return - if (shouldEnable) { - require('..').enable(config) - } else { - require('..').disable() - } + if (activation === Activation.ONECLICK) { + enableOrDisableAppsec(action, rcConfig, config, appsec) } + + apiSecuritySampler.setRequestSampling(rcConfig.api_security?.request_sample_rate) }) } return rc } +function enableOrDisableAppsec (action, rcConfig, config, appsec) { + if (typeof rcConfig.asm?.enabled === 'boolean') { + let shouldEnable + + if (action === 'apply' || action === 'modify') { + shouldEnable = rcConfig.asm.enabled // take control + } else { + shouldEnable = config.appsec.enabled // give back control to local config + } + + if (shouldEnable) { + appsec.enable(config) + } else { + appsec.disable() + } + } +} + function enableWafUpdate (appsecConfig) { - if (rc && appsecConfig && !appsecConfig.customRulesProvided) { + if (rc && appsecConfig && !appsecConfig.rules) { // dirty require to make startup faster for serverless const RuleManager = require('../rule_manager') @@ -44,13 +71,24 @@ function enableWafUpdate (appsecConfig) { rc.updateCapabilities(RemoteConfigCapabilities.ASM_DD_RULES, true) rc.updateCapabilities(RemoteConfigCapabilities.ASM_EXCLUSIONS, true) rc.updateCapabilities(RemoteConfigCapabilities.ASM_REQUEST_BLOCKING, true) + rc.updateCapabilities(RemoteConfigCapabilities.ASM_RESPONSE_BLOCKING, true) rc.updateCapabilities(RemoteConfigCapabilities.ASM_CUSTOM_RULES, true) rc.updateCapabilities(RemoteConfigCapabilities.ASM_CUSTOM_BLOCKING_RESPONSE, true) rc.updateCapabilities(RemoteConfigCapabilities.ASM_TRUSTED_IPS, true) + rc.updateCapabilities(RemoteConfigCapabilities.ASM_ENDPOINT_FINGERPRINT, true) + rc.updateCapabilities(RemoteConfigCapabilities.ASM_NETWORK_FINGERPRINT, true) + rc.updateCapabilities(RemoteConfigCapabilities.ASM_HEADER_FINGERPRINT, true) - rc.on('ASM_DATA', noop) - rc.on('ASM_DD', noop) - rc.on('ASM', noop) + if (appsecConfig.rasp?.enabled) { + rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_SQLI, true) + rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_SSRF, true) + rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_LFI, true) + } + + // TODO: delete noop handlers and kPreUpdate and replace with batched handlers + rc.setProductHandler('ASM_DATA', noop) + rc.setProductHandler('ASM_DD', noop) + rc.setProductHandler('ASM', noop) rc.on(RemoteConfigManager.kPreUpdate, RuleManager.updateWafFromRC) } @@ -65,13 +103,21 @@ function disableWafUpdate () { rc.updateCapabilities(RemoteConfigCapabilities.ASM_DD_RULES, false) rc.updateCapabilities(RemoteConfigCapabilities.ASM_EXCLUSIONS, false) rc.updateCapabilities(RemoteConfigCapabilities.ASM_REQUEST_BLOCKING, false) + rc.updateCapabilities(RemoteConfigCapabilities.ASM_RESPONSE_BLOCKING, false) rc.updateCapabilities(RemoteConfigCapabilities.ASM_CUSTOM_RULES, false) rc.updateCapabilities(RemoteConfigCapabilities.ASM_CUSTOM_BLOCKING_RESPONSE, false) rc.updateCapabilities(RemoteConfigCapabilities.ASM_TRUSTED_IPS, false) + rc.updateCapabilities(RemoteConfigCapabilities.ASM_ENDPOINT_FINGERPRINT, false) + rc.updateCapabilities(RemoteConfigCapabilities.ASM_NETWORK_FINGERPRINT, false) + rc.updateCapabilities(RemoteConfigCapabilities.ASM_HEADER_FINGERPRINT, false) + + rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_SQLI, false) + rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_SSRF, false) + rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_LFI, false) - rc.off('ASM_DATA', noop) - rc.off('ASM_DD', noop) - rc.off('ASM', noop) + rc.removeProductHandler('ASM_DATA') + rc.removeProductHandler('ASM_DD') + rc.removeProductHandler('ASM') rc.off(RemoteConfigManager.kPreUpdate, RuleManager.updateWafFromRC) } diff --git a/packages/dd-trace/src/appsec/remote_config/manager.js b/packages/dd-trace/src/appsec/remote_config/manager.js index 2b9074dfcce..8f2aa44cea2 100644 --- a/packages/dd-trace/src/appsec/remote_config/manager.js +++ b/packages/dd-trace/src/appsec/remote_config/manager.js @@ -15,6 +15,7 @@ const clientId = uuid() const DEFAULT_CAPABILITY = Buffer.alloc(1).toString('base64') // 0x00 const kPreUpdate = Symbol('kPreUpdate') +const kSupportsAckCallback = Symbol('kSupportsAckCallback') // There MUST NOT exist separate instances of RC clients in a tracer making separate ClientGetConfigsRequest // with their own separated Client.ClientState. @@ -25,26 +26,33 @@ class RemoteConfigManager extends EventEmitter { super() const pollInterval = Math.floor(config.remoteConfig.pollInterval * 1000) - const url = config.url || new URL(format({ + + this.url = config.url || new URL(format({ protocol: 'http:', hostname: config.hostname || 'localhost', port: config.port })) - this.scheduler = new Scheduler((cb) => this.poll(cb), pollInterval) + this._handlers = new Map() + const appliedConfigs = this.appliedConfigs = new Map() - this.requestOptions = { - url, - method: 'POST', - path: '/v0.7/config' - } + this.scheduler = new Scheduler((cb) => this.poll(cb), pollInterval) this.state = { client: { - state: { // updated by `parseConfig()` + state: { // updated by `parseConfig()` and `poll()` root_version: 1, targets_version: 0, - config_states: [], + // Use getter so `apply_*` can be updated async and still affect the content of `config_states` + get config_states () { + return Array.from(appliedConfigs.values()).map((conf) => ({ + id: conf.id, + version: conf.version, + product: conf.product, + apply_state: conf.apply_state, + apply_error: conf.apply_error + })) + }, has_error: false, error: '', backend_client_state: '' @@ -65,8 +73,6 @@ class RemoteConfigManager extends EventEmitter { }, cached_target_files: [] // updated by `parseConfig()` } - - this.appliedConfigs = new Map() } updateCapabilities (mask, value) { @@ -87,32 +93,24 @@ class RemoteConfigManager extends EventEmitter { this.state.client.capabilities = Buffer.from(str, 'hex').toString('base64') } - on (event, listener) { - super.on(event, listener) - + setProductHandler (product, handler) { + this._handlers.set(product, handler) this.updateProducts() - - if (this.state.client.products.length) { + if (this.state.client.products.length === 1) { this.scheduler.start() } - - return this } - off (event, listener) { - super.off(event, listener) - + removeProductHandler (product) { + this._handlers.delete(product) this.updateProducts() - - if (!this.state.client.products.length) { + if (this.state.client.products.length === 0) { this.scheduler.stop() } - - return this } updateProducts () { - this.state.client.products = this.eventNames().filter(e => typeof e === 'string') + this.state.client.products = Array.from(this._handlers.keys()) } getPayload () { @@ -122,7 +120,16 @@ class RemoteConfigManager extends EventEmitter { } poll (cb) { - request(this.getPayload(), this.requestOptions, (err, data, statusCode) => { + const options = { + url: this.url, + method: 'POST', + path: '/v0.7/config', + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + + request(this.getPayload(), options, (err, data, statusCode) => { // 404 means RC is disabled, ignore it if (statusCode === 404) return cb() @@ -224,24 +231,11 @@ class RemoteConfigManager extends EventEmitter { this.dispatch(toApply, 'apply') this.dispatch(toModify, 'modify') - this.state.client.state.config_states = [] - this.state.cached_target_files = [] - - for (const conf of this.appliedConfigs.values()) { - this.state.client.state.config_states.push({ - id: conf.id, - version: conf.version, - product: conf.product, - apply_state: conf.apply_state, - apply_error: conf.apply_error - }) - - this.state.cached_target_files.push({ - path: conf.path, - length: conf.length, - hashes: Object.entries(conf.hashes).map((entry) => ({ algorithm: entry[0], hash: entry[1] })) - }) - } + this.state.cached_target_files = Array.from(this.appliedConfigs.values()).map((conf) => ({ + path: conf.path, + length: conf.length, + hashes: Object.entries(conf.hashes).map((entry) => ({ algorithm: entry[0], hash: entry[1] })) + })) } } @@ -250,20 +244,7 @@ class RemoteConfigManager extends EventEmitter { // TODO: we need a way to tell if unapply configs were handled by kPreUpdate or not, because they're always // emitted unlike the apply and modify configs - // in case the item was already handled by kPreUpdate - if (item.apply_state === UNACKNOWLEDGED || action === 'unapply') { - try { - // TODO: do we want to pass old and new config ? - const hadListeners = this.emit(item.product, action, item.file, item.id) - - if (hadListeners) { - item.apply_state = ACKNOWLEDGED - } - } catch (err) { - item.apply_state = ERROR - item.apply_error = err.toString() - } - } + this._callHandlerFor(action, item) if (action === 'unapply') { this.appliedConfigs.delete(item.path) @@ -272,6 +253,49 @@ class RemoteConfigManager extends EventEmitter { } } } + + _callHandlerFor (action, item) { + // in case the item was already handled by kPreUpdate + if (item.apply_state !== UNACKNOWLEDGED && action !== 'unapply') return + + const handler = this._handlers.get(item.product) + + if (!handler) return + + try { + if (supportsAckCallback(handler)) { + // If the handler accepts an `ack` callback, expect that to be called and set `apply_state` accordinly + // TODO: do we want to pass old and new config ? + handler(action, item.file, item.id, (err) => { + if (err) { + item.apply_state = ERROR + item.apply_error = err.toString() + } else if (item.apply_state !== ERROR) { + item.apply_state = ACKNOWLEDGED + } + }) + } else { + // If the handler doesn't accept an `ack` callback, assume `apply_state` is `ACKNOWLEDGED`, + // unless it returns a promise, in which case we wait for the promise to be resolved or rejected. + // TODO: do we want to pass old and new config ? + const result = handler(action, item.file, item.id) + if (result instanceof Promise) { + result.then( + () => { item.apply_state = ACKNOWLEDGED }, + (err) => { + item.apply_state = ERROR + item.apply_error = err.toString() + } + ) + } else { + item.apply_state = ACKNOWLEDGED + } + } + } catch (err) { + item.apply_state = ERROR + item.apply_error = err.toString() + } + } } function fromBase64JSON (str) { @@ -295,4 +319,22 @@ function parseConfigPath (configPath) { } } +function supportsAckCallback (handler) { + if (kSupportsAckCallback in handler) return handler[kSupportsAckCallback] + + const numOfArgs = handler.length + let result = false + + if (numOfArgs >= 4) { + result = true + } else if (numOfArgs !== 0) { + const source = handler.toString() + result = source.slice(0, source.indexOf(')')).includes('...') + } + + handler[kSupportsAckCallback] = result + + return result +} + module.exports = RemoteConfigManager diff --git a/packages/dd-trace/src/appsec/reporter.js b/packages/dd-trace/src/appsec/reporter.js index d22613c749a..dd2bde9fb06 100644 --- a/packages/dd-trace/src/appsec/reporter.js +++ b/packages/dd-trace/src/appsec/reporter.js @@ -7,34 +7,55 @@ const { ipHeaderList } = require('../plugins/util/ip_extractor') const { incrementWafInitMetric, updateWafRequestsMetricTags, + updateRaspRequestsMetricTags, incrementWafUpdatesMetric, - incrementWafRequestsMetric + incrementWafRequestsMetric, + getRequestMetrics } = require('./telemetry') const zlib = require('zlib') +const { MANUAL_KEEP } = require('../../../../ext/tags') +const standalone = require('./standalone') // default limiter, configurable with setRateLimit() let limiter = new Limiter(100) const metricsQueue = new Map() +// following header lists are ordered in the same way the spec orders them, it doesn't matter but it's easier to compare const contentHeaderList = [ - 'content-encoding', - 'content-language', 'content-length', - 'content-type' + 'content-type', + 'content-encoding', + 'content-language' ] -const REQUEST_HEADERS_MAP = mapHeaderAndTags([ - 'accept', - 'accept-encoding', - 'accept-language', - 'host', - 'user-agent', +const EVENT_HEADERS_MAP = mapHeaderAndTags([ + ...ipHeaderList, 'forwarded', 'via', + ...contentHeaderList, + 'host', + 'accept-encoding', + 'accept-language' +], 'http.request.headers.') - ...ipHeaderList, - ...contentHeaderList +const identificationHeaders = [ + 'x-amzn-trace-id', + 'cloudfront-viewer-ja3-fingerprint', + 'cf-ray', + 'x-cloud-trace-context', + 'x-appgw-trace-id', + 'x-sigsci-requestid', + 'x-sigsci-tags', + 'akamai-user-risk' +] + +// these request headers are always collected - it breaks the expected spec orders +const REQUEST_HEADERS_MAP = mapHeaderAndTags([ + 'content-type', + 'user-agent', + 'accept', + ...identificationHeaders ], 'http.request.headers.') const RESPONSE_HEADERS_MAP = mapHeaderAndTags(contentHeaderList, 'http.response.headers.') @@ -75,30 +96,24 @@ function reportWafInit (wafVersion, rulesVersion, diagnosticsRules = {}) { metricsQueue.set('_dd.appsec.event_rules.errors', JSON.stringify(diagnosticsRules.errors)) } - metricsQueue.set('manual.keep', 'true') + metricsQueue.set(MANUAL_KEEP, 'true') incrementWafInitMetric(wafVersion, rulesVersion) } -function reportMetrics (metrics) { - // TODO: metrics should be incremental, there already is an RFC to report metrics +function reportMetrics (metrics, raspRuleType) { const store = storage.getStore() const rootSpan = store?.req && web.root(store.req) if (!rootSpan) return - if (metrics.duration) { - rootSpan.setTag('_dd.appsec.waf.duration', metrics.duration) - } - - if (metrics.durationExt) { - rootSpan.setTag('_dd.appsec.waf.duration_ext', metrics.durationExt) - } - if (metrics.rulesVersion) { rootSpan.setTag('_dd.appsec.event_rules.version', metrics.rulesVersion) } - - updateWafRequestsMetricTags(metrics, store.req) + if (raspRuleType) { + updateRaspRequestsMetricTags(metrics, store.req, raspRuleType) + } else { + updateWafRequestsMetricTags(metrics, store.req) + } } function reportAttack (attackData) { @@ -109,12 +124,14 @@ function reportAttack (attackData) { const currentTags = rootSpan.context()._tags - const newTags = filterHeaders(req.headers, REQUEST_HEADERS_MAP) - - newTags['appsec.event'] = 'true' + const newTags = { + 'appsec.event': 'true' + } if (limiter.isAllowed()) { - newTags['manual.keep'] = 'true' // TODO: figure out how to keep appsec traces with sampling revamp + newTags[MANUAL_KEEP] = 'true' + + standalone.sample(rootSpan) } // TODO: maybe add this to format.js later (to take decision as late as possible) @@ -131,17 +148,16 @@ function reportAttack (attackData) { newTags['_dd.appsec.json'] = '{"triggers":' + attackData + '}' } - const ua = newTags['http.request.headers.user-agent'] - if (ua) { - newTags['http.useragent'] = ua - } - newTags['network.client.ip'] = req.socket.remoteAddress rootSpan.addTags(newTags) } -function reportSchemas (derivatives) { +function isFingerprintDerivative (derivative) { + return derivative.startsWith('_dd.appsec.fp') +} + +function reportDerivatives (derivatives) { if (!derivatives) return const req = storage.getStore()?.req @@ -150,11 +166,12 @@ function reportSchemas (derivatives) { if (!rootSpan) return const tags = {} - for (const [address, value] of Object.entries(derivatives)) { - if (address.startsWith('_dd.appsec.s.req')) { + for (let [tag, value] of Object.entries(derivatives)) { + if (!isFingerprintDerivative(tag)) { const gzippedValue = zlib.gzipSync(JSON.stringify(value)) - tags[address] = gzippedValue.toString('base64') + value = gzippedValue.toString('base64') } + tags[tag] = value } rootSpan.addTags(tags) @@ -167,22 +184,65 @@ function finishRequest (req, res) { if (metricsQueue.size) { rootSpan.addTags(Object.fromEntries(metricsQueue)) + standalone.sample(rootSpan) + metricsQueue.clear() } + const metrics = getRequestMetrics(req) + if (metrics?.duration) { + rootSpan.setTag('_dd.appsec.waf.duration', metrics.duration) + } + + if (metrics?.durationExt) { + rootSpan.setTag('_dd.appsec.waf.duration_ext', metrics.durationExt) + } + + if (metrics?.raspDuration) { + rootSpan.setTag('_dd.appsec.rasp.duration', metrics.raspDuration) + } + + if (metrics?.raspDurationExt) { + rootSpan.setTag('_dd.appsec.rasp.duration_ext', metrics.raspDurationExt) + } + + if (metrics?.raspEvalCount) { + rootSpan.setTag('_dd.appsec.rasp.rule.eval', metrics.raspEvalCount) + } + incrementWafRequestsMetric(req) - if (!rootSpan.context()._tags['appsec.event']) return + // collect some headers even when no attack is detected + const mandatoryTags = filterHeaders(req.headers, REQUEST_HEADERS_MAP) + rootSpan.addTags(mandatoryTags) + + const tags = rootSpan.context()._tags + if (!shouldCollectEventHeaders(tags)) return const newTags = filterHeaders(res.getHeaders(), RESPONSE_HEADERS_MAP) + Object.assign(newTags, filterHeaders(req.headers, EVENT_HEADERS_MAP)) - if (req.route && typeof req.route.path === 'string') { + if (tags['appsec.event'] === 'true' && typeof req.route?.path === 'string') { newTags['http.endpoint'] = req.route.path } rootSpan.addTags(newTags) } +function shouldCollectEventHeaders (tags = {}) { + if (tags['appsec.event'] === 'true') { + return true + } + + for (const tagName of Object.keys(tags)) { + if (tagName.startsWith('appsec.events.')) { + return true + } + } + + return false +} + function setRateLimit (rateLimit) { limiter = new Limiter(rateLimit) } @@ -195,7 +255,7 @@ module.exports = { reportMetrics, reportAttack, reportWafUpdate: incrementWafUpdatesMetric, - reportSchemas, + reportDerivatives, finishRequest, setRateLimit, mapHeaderAndTags diff --git a/packages/dd-trace/src/appsec/rule_manager.js b/packages/dd-trace/src/appsec/rule_manager.js index 7f13d14bb34..5bb6fcf98bb 100644 --- a/packages/dd-trace/src/appsec/rule_manager.js +++ b/packages/dd-trace/src/appsec/rule_manager.js @@ -3,6 +3,7 @@ const fs = require('fs') const waf = require('./waf') const { ACKNOWLEDGED, ERROR } = require('./remote_config/apply_states') + const blocking = require('./blocking') let defaultRules @@ -21,9 +22,7 @@ function loadRules (config) { waf.init(defaultRules, config) - if (defaultRules.actions) { - blocking.updateBlockingConfiguration(defaultRules.actions.find(action => action.id === 'block')) - } + blocking.setDefaultBlockingActionParameters(defaultRules?.actions) } function updateWafFromRC ({ toUnapply, toApply, toModify }) { @@ -68,40 +67,33 @@ function updateWafFromRC ({ toUnapply, toApply, toModify }) { item.apply_state = ERROR item.apply_error = 'Multiple ruleset received in ASM_DD' } else { - if (file && file.rules && file.rules.length) { - const { version, metadata, rules } = file + if (file?.rules?.length) { + const { version, metadata, rules, processors, scanners } = file - newRuleset = { version, metadata, rules } + newRuleset = { version, metadata, rules, processors, scanners } newRulesetId = id } batch.add(item) } } else if (product === 'ASM') { - let batchConfiguration = false - if (file && file.rules_override && file.rules_override.length) { - batchConfiguration = true + if (file?.rules_override?.length) { newRulesOverride.set(id, file.rules_override) } - if (file && file.exclusions && file.exclusions.length) { - batchConfiguration = true + if (file?.exclusions?.length) { newExclusions.set(id, file.exclusions) } - if (file && file.custom_rules && file.custom_rules.length) { - batchConfiguration = true + if (file?.custom_rules?.length) { newCustomRules.set(id, file.custom_rules) } - if (file && file.actions && file.actions.length) { + if (file?.actions?.length) { newActions.set(id, file.actions) } - // "actions" data is managed by tracer and not by waf - if (batchConfiguration) { - batch.add(item) - } + batch.add(item) } } @@ -112,7 +104,9 @@ function updateWafFromRC ({ toUnapply, toApply, toModify }) { newRuleset || newRulesOverride.modified || newExclusions.modified || - newCustomRules.modified) { + newCustomRules.modified || + newActions.modified + ) { const payload = newRuleset || {} if (newRulesData.modified) { @@ -127,6 +121,9 @@ function updateWafFromRC ({ toUnapply, toApply, toModify }) { if (newCustomRules.modified) { payload.custom_rules = concatArrays(newCustomRules) } + if (newActions.modified) { + payload.actions = concatArrays(newActions) + } try { waf.update(payload) @@ -146,6 +143,11 @@ function updateWafFromRC ({ toUnapply, toApply, toModify }) { if (newCustomRules.modified) { appliedCustomRules = newCustomRules } + if (newActions.modified) { + appliedActions = newActions + + blocking.setDefaultBlockingActionParameters(concatArrays(newActions)) + } } catch (err) { newApplyState = ERROR newApplyError = err.toString() @@ -156,11 +158,6 @@ function updateWafFromRC ({ toUnapply, toApply, toModify }) { config.apply_state = newApplyState if (newApplyError) config.apply_error = newApplyError } - - if (newActions.modified) { - blocking.updateBlockingConfiguration(concatArrays(newActions).find(action => action.id === 'block')) - appliedActions = newActions - } } // A Map with a new prop `modified`, a bool that indicates if the Map was modified @@ -242,7 +239,6 @@ function copyRulesData (rulesData) { function clearAllRules () { waf.destroy() - blocking.updateBlockingConfiguration(undefined) defaultRules = undefined @@ -252,6 +248,8 @@ function clearAllRules () { appliedExclusions.clear() appliedCustomRules.clear() appliedActions.clear() + + blocking.setDefaultBlockingActionParameters(undefined) } module.exports = { diff --git a/packages/dd-trace/src/appsec/sdk/track_event.js b/packages/dd-trace/src/appsec/sdk/track_event.js index 8debb932090..36c40093b19 100644 --- a/packages/dd-trace/src/appsec/sdk/track_event.js +++ b/packages/dd-trace/src/appsec/sdk/track_event.js @@ -4,6 +4,8 @@ const log = require('../../log') const { getRootSpan } = require('./utils') const { MANUAL_KEEP } = require('../../../../../ext/tags') const { setUserTags } = require('./set_user') +const standalone = require('../standalone') +const waf = require('../waf') function trackUserLoginSuccessEvent (tracer, user, metadata) { // TODO: better user check here and in _setUser() ? @@ -73,6 +75,12 @@ function trackEvent (eventName, fields, sdkMethodName, rootSpan, mode) { } rootSpan.addTags(tags) + + standalone.sample(rootSpan) + + if (['users.login.success', 'users.login.failure'].includes(eventName)) { + waf.run({ persistent: { [`server.business_logic.${eventName}`]: null } }) + } } module.exports = { diff --git a/packages/dd-trace/src/appsec/sdk/user_blocking.js b/packages/dd-trace/src/appsec/sdk/user_blocking.js index e36686884ed..19997d3ff9c 100644 --- a/packages/dd-trace/src/appsec/sdk/user_blocking.js +++ b/packages/dd-trace/src/appsec/sdk/user_blocking.js @@ -3,17 +3,14 @@ const { USER_ID } = require('../addresses') const waf = require('../waf') const { getRootSpan } = require('./utils') -const { block } = require('../blocking') +const { block, getBlockingAction } = require('../blocking') const { storage } = require('../../../../datadog-core') const { setUserTags } = require('./set_user') const log = require('../../log') function isUserBlocked (user) { - const actions = waf.run({ [USER_ID]: user.id }) - - if (!actions) return false - - return actions.includes('block') + const actions = waf.run({ persistent: { [USER_ID]: user.id } }) + return !!getBlockingAction(actions) } function checkUserAndSetUser (tracer, user) { diff --git a/packages/dd-trace/src/appsec/stack_trace.js b/packages/dd-trace/src/appsec/stack_trace.js new file mode 100644 index 00000000000..ea49ed1e877 --- /dev/null +++ b/packages/dd-trace/src/appsec/stack_trace.js @@ -0,0 +1,90 @@ +'use strict' + +const { calculateDDBasePath } = require('../util') + +const ddBasePath = calculateDDBasePath(__dirname) + +const LIBRARY_FRAMES_BUFFER = 20 + +function getCallSiteList (maxDepth = 100) { + const previousPrepareStackTrace = Error.prepareStackTrace + const previousStackTraceLimit = Error.stackTraceLimit + let callsiteList + Error.stackTraceLimit = maxDepth + + try { + Error.prepareStackTrace = function (_, callsites) { + callsiteList = callsites + } + const e = new Error() + e.stack + } finally { + Error.prepareStackTrace = previousPrepareStackTrace + Error.stackTraceLimit = previousStackTraceLimit + } + + return callsiteList +} + +function filterOutFramesFromLibrary (callSiteList) { + return callSiteList.filter(callSite => !callSite.getFileName()?.startsWith(ddBasePath)) +} + +function getFramesForMetaStruct (callSiteList, maxDepth = 32) { + const filteredFrames = filterOutFramesFromLibrary(callSiteList) + + const half = filteredFrames.length > maxDepth ? Math.round(maxDepth / 2) : Infinity + + const indexedFrames = [] + for (let i = 0; i < Math.min(filteredFrames.length, maxDepth); i++) { + const index = i < half ? i : i + filteredFrames.length - maxDepth + const callSite = filteredFrames[index] + indexedFrames.push({ + id: index, + file: callSite.getFileName(), + line: callSite.getLineNumber(), + column: callSite.getColumnNumber(), + function: callSite.getFunctionName(), + class_name: callSite.getTypeName() + }) + } + + return indexedFrames +} + +function reportStackTrace (rootSpan, stackId, maxDepth, maxStackTraces, callSiteListGetter = getCallSiteList) { + if (!rootSpan) return + + if (maxStackTraces < 1 || (rootSpan.meta_struct?.['_dd.stack']?.exploit?.length ?? 0) < maxStackTraces) { + // Since some frames will be discarded because they come from tracer codebase, a buffer is added + // to the limit in order to get as close as `maxDepth` number of frames. + if (maxDepth < 1) maxDepth = Infinity + const callSiteList = callSiteListGetter(maxDepth + LIBRARY_FRAMES_BUFFER) + if (!Array.isArray(callSiteList)) return + + if (!rootSpan.meta_struct) { + rootSpan.meta_struct = {} + } + + if (!rootSpan.meta_struct['_dd.stack']) { + rootSpan.meta_struct['_dd.stack'] = {} + } + + if (!rootSpan.meta_struct['_dd.stack'].exploit) { + rootSpan.meta_struct['_dd.stack'].exploit = [] + } + + const frames = getFramesForMetaStruct(callSiteList, maxDepth) + + rootSpan.meta_struct['_dd.stack'].exploit.push({ + id: stackId, + language: 'nodejs', + frames + }) + } +} + +module.exports = { + getCallSiteList, + reportStackTrace +} diff --git a/packages/dd-trace/src/appsec/standalone.js b/packages/dd-trace/src/appsec/standalone.js new file mode 100644 index 00000000000..9d75dd36260 --- /dev/null +++ b/packages/dd-trace/src/appsec/standalone.js @@ -0,0 +1,130 @@ +'use strict' + +const { channel } = require('dc-polyfill') +const { USER_KEEP, AUTO_KEEP, AUTO_REJECT } = require('../../../../ext/priority') +const { MANUAL_KEEP } = require('../../../../ext/tags') +const PrioritySampler = require('../priority_sampler') +const RateLimiter = require('../rate_limiter') +const TraceState = require('../opentracing/propagation/tracestate') +const { hasOwn } = require('../util') +const { APM_TRACING_ENABLED_KEY, APPSEC_PROPAGATION_KEY, SAMPLING_MECHANISM_DEFAULT } = require('../constants') + +const startCh = channel('dd-trace:span:start') +const injectCh = channel('dd-trace:span:inject') +const extractCh = channel('dd-trace:span:extract') + +let enabled + +class StandAloneAsmPrioritySampler extends PrioritySampler { + constructor (env) { + super(env, { sampleRate: 0, rateLimit: 0, rules: [] }) + + // let some regular APM traces go through, 1 per minute to keep alive the service + this._limiter = new RateLimiter(1, 'minute') + } + + configure (env, config) { + // rules not supported + this._env = env + } + + _getPriorityFromTags (tags, context) { + if (hasOwn(tags, MANUAL_KEEP) && + tags[MANUAL_KEEP] !== false && + hasOwn(context._trace.tags, APPSEC_PROPAGATION_KEY) + ) { + return USER_KEEP + } + } + + _getPriorityFromAuto (span) { + const context = this._getContext(span) + + context._sampling.mechanism = SAMPLING_MECHANISM_DEFAULT + + if (hasOwn(context._trace.tags, APPSEC_PROPAGATION_KEY)) { + return USER_KEEP + } + + return this._isSampledByRateLimit(context) ? AUTO_KEEP : AUTO_REJECT + } +} + +function onSpanStart ({ span, fields }) { + const tags = span.context?.()?._tags + if (!tags) return + + const { parent } = fields + if (!parent || parent._isRemote) { + tags[APM_TRACING_ENABLED_KEY] = 0 + } +} + +function onSpanInject ({ spanContext, carrier }) { + if (!spanContext?._trace?.tags || !carrier) return + + // do not inject trace and sampling if there is no appsec event + if (!hasOwn(spanContext._trace.tags, APPSEC_PROPAGATION_KEY)) { + for (const key in carrier) { + const lKey = key.toLowerCase() + if (lKey.startsWith('x-datadog')) { + delete carrier[key] + } else if (lKey === 'tracestate') { + const tracestate = TraceState.fromString(carrier[key]) + tracestate.forVendor('dd', state => state.clear()) + carrier[key] = tracestate.toString() + } + } + } +} + +function onSpanExtract ({ spanContext = {} }) { + if (!spanContext._trace?.tags || !spanContext._sampling) return + + // reset upstream priority if _dd.p.appsec is not found + if (!hasOwn(spanContext._trace.tags, APPSEC_PROPAGATION_KEY)) { + spanContext._sampling.priority = undefined + } else if (spanContext._sampling.priority !== USER_KEEP) { + spanContext._sampling.priority = USER_KEEP + } +} + +function sample (span) { + const spanContext = span.context?.() + if (enabled && spanContext?._trace?.tags) { + spanContext._trace.tags[APPSEC_PROPAGATION_KEY] = '1' + + // TODO: ask. can we reset here sampling like this? + if (spanContext._sampling?.priority < AUTO_KEEP) { + spanContext._sampling.priority = undefined + } + } +} + +function configure (config) { + const configChanged = enabled !== config.appsec?.standalone?.enabled + if (!configChanged) return + + enabled = config.appsec?.standalone?.enabled + + let prioritySampler + if (enabled) { + startCh.subscribe(onSpanStart) + injectCh.subscribe(onSpanInject) + extractCh.subscribe(onSpanExtract) + + prioritySampler = new StandAloneAsmPrioritySampler(config.env) + } else { + if (startCh.hasSubscribers) startCh.unsubscribe(onSpanStart) + if (injectCh.hasSubscribers) injectCh.unsubscribe(onSpanInject) + if (extractCh.hasSubscribers) extractCh.unsubscribe(onSpanExtract) + } + + return prioritySampler +} + +module.exports = { + configure, + sample, + StandAloneAsmPrioritySampler +} diff --git a/packages/dd-trace/src/appsec/telemetry.js b/packages/dd-trace/src/appsec/telemetry.js index 145e8e3a436..d96ca77601f 100644 --- a/packages/dd-trace/src/appsec/telemetry.js +++ b/packages/dd-trace/src/appsec/telemetry.js @@ -5,6 +5,7 @@ const telemetryMetrics = require('../telemetry/metrics') const appsecMetrics = telemetryMetrics.manager.namespace('appsec') const DD_TELEMETRY_WAF_RESULT_TAGS = Symbol('_dd.appsec.telemetry.waf.result.tags') +const DD_TELEMETRY_REQUEST_METRICS = Symbol('_dd.appsec.telemetry.request.metrics') const tags = { REQUEST_BLOCKED: 'request_blocked', @@ -26,10 +27,22 @@ function disable () { enabled = false } +function newStore () { + return { + [DD_TELEMETRY_REQUEST_METRICS]: { + duration: 0, + durationExt: 0, + raspDuration: 0, + raspDurationExt: 0, + raspEvalCount: 0 + } + } +} + function getStore (req) { let store = metricsStoreMap.get(req) if (!store) { - store = {} + store = newStore() metricsStoreMap.set(req, store) } return store @@ -51,9 +64,7 @@ function trackWafDurations (metrics, versionsTags) { } } -function getOrCreateMetricTags (req, versionsTags) { - const store = getStore(req) - +function getOrCreateMetricTags (store, versionsTags) { let metricTags = store[DD_TELEMETRY_WAF_RESULT_TAGS] if (!metricTags) { metricTags = { @@ -68,14 +79,43 @@ function getOrCreateMetricTags (req, versionsTags) { return metricTags } +function updateRaspRequestsMetricTags (metrics, req, raspRuleType) { + if (!req) return + + const store = getStore(req) + + // it does not depend on whether telemetry is enabled or not + addRaspRequestMetrics(store, metrics) + + if (!enabled) return + + const tags = { rule_type: raspRuleType, waf_version: metrics.wafVersion } + appsecMetrics.count('rasp.rule.eval', tags).inc(1) + + if (metrics.wafTimeout) { + appsecMetrics.count('rasp.timeout', tags).inc(1) + } + + if (metrics.ruleTriggered) { + appsecMetrics.count('rasp.rule.match', tags).inc(1) + } +} + function updateWafRequestsMetricTags (metrics, req) { - if (!req || !enabled) return + if (!req) return + + const store = getStore(req) + + // it does not depend on whether telemetry is enabled or not + addRequestMetrics(store, metrics) + + if (!enabled) return const versionsTags = getVersionsTags(metrics.wafVersion, metrics.rulesVersion) trackWafDurations(metrics, versionsTags) - const metricTags = getOrCreateMetricTags(req, versionsTags) + const metricTags = getOrCreateMetricTags(store, versionsTags) const { blockTriggered, ruleTriggered, wafTimeout } = metrics @@ -121,12 +161,33 @@ function incrementWafRequestsMetric (req) { metricsStoreMap.delete(req) } +function addRequestMetrics (store, { duration, durationExt }) { + store[DD_TELEMETRY_REQUEST_METRICS].duration += duration || 0 + store[DD_TELEMETRY_REQUEST_METRICS].durationExt += durationExt || 0 +} + +function addRaspRequestMetrics (store, { duration, durationExt }) { + store[DD_TELEMETRY_REQUEST_METRICS].raspDuration += duration || 0 + store[DD_TELEMETRY_REQUEST_METRICS].raspDurationExt += durationExt || 0 + store[DD_TELEMETRY_REQUEST_METRICS].raspEvalCount++ +} + +function getRequestMetrics (req) { + if (req) { + const store = getStore(req) + return store?.[DD_TELEMETRY_REQUEST_METRICS] + } +} + module.exports = { enable, disable, updateWafRequestsMetricTags, + updateRaspRequestsMetricTags, incrementWafInitMetric, incrementWafUpdatesMetric, - incrementWafRequestsMetric + incrementWafRequestsMetric, + + getRequestMetrics } diff --git a/packages/dd-trace/src/appsec/waf/index.js b/packages/dd-trace/src/appsec/waf/index.js index 13190e9c7be..8aa30fabbb4 100644 --- a/packages/dd-trace/src/appsec/waf/index.js +++ b/packages/dd-trace/src/appsec/waf/index.js @@ -46,7 +46,7 @@ function update (newRules) { } } -function run (data, req) { +function run (data, req, raspRuleType) { if (!req) { const store = storage.getStore() if (!store || !store.req) { @@ -59,7 +59,7 @@ function run (data, req) { const wafContext = waf.wafManager.getWAFContext(req) - return wafContext.run(data) + return wafContext.run(data, raspRuleType) } function disposeContext (req) { diff --git a/packages/dd-trace/src/appsec/waf/waf_context_wrapper.js b/packages/dd-trace/src/appsec/waf/waf_context_wrapper.js index 83ab3dcc1cd..a2dae737a86 100644 --- a/packages/dd-trace/src/appsec/waf/waf_context_wrapper.js +++ b/packages/dd-trace/src/appsec/waf/waf_context_wrapper.js @@ -3,6 +3,8 @@ const log = require('../../log') const Reporter = require('../reporter') const addresses = require('../addresses') +const { getBlockingAction } = require('../blocking') +const { wafRunFinished } = require('../channels') // TODO: remove once ephemeral addresses are implemented const preventDuplicateAddresses = new Set([ @@ -10,45 +12,72 @@ const preventDuplicateAddresses = new Set([ ]) class WAFContextWrapper { - constructor (ddwafContext, wafTimeout, wafVersion, rulesVersion) { + constructor (ddwafContext, wafTimeout, wafVersion, rulesVersion, knownAddresses) { this.ddwafContext = ddwafContext this.wafTimeout = wafTimeout this.wafVersion = wafVersion this.rulesVersion = rulesVersion this.addressesToSkip = new Set() + this.knownAddresses = knownAddresses } - run (params) { - const inputs = {} - let someInputAdded = false + run ({ persistent, ephemeral }, raspRuleType) { + if (this.ddwafContext.disposed) { + log.warn('Calling run on a disposed context') + return + } + + const payload = {} + let payloadHasData = false const newAddressesToSkip = new Set(this.addressesToSkip) - // TODO: possible optimizaion: only send params that haven't already been sent with same value to this wafContext - for (const key of Object.keys(params)) { - // TODO: requiredAddresses is no longer used due to processor addresses are not included in the list. Check on - // future versions when the actual addresses are included in the 'loaded' section inside diagnostics. - if (!this.addressesToSkip.has(key)) { - inputs[key] = params[key] - if (preventDuplicateAddresses.has(key)) { - newAddressesToSkip.add(key) + if (persistent !== null && typeof persistent === 'object') { + const persistentInputs = {} + + for (const key of Object.keys(persistent)) { + if (!this.addressesToSkip.has(key) && this.knownAddresses.has(key)) { + persistentInputs[key] = persistent[key] + if (preventDuplicateAddresses.has(key)) { + newAddressesToSkip.add(key) + } } - someInputAdded = true + } + + if (Object.keys(persistentInputs).length) { + payload.persistent = persistentInputs + payloadHasData = true } } - if (!someInputAdded) return + if (ephemeral !== null && typeof ephemeral === 'object') { + const ephemeralInputs = {} + + for (const key of Object.keys(ephemeral)) { + if (this.knownAddresses.has(key)) { + ephemeralInputs[key] = ephemeral[key] + } + } + + if (Object.keys(ephemeralInputs).length) { + payload.ephemeral = ephemeralInputs + payloadHasData = true + } + } + + if (!payloadHasData) return try { const start = process.hrtime.bigint() - const result = this.ddwafContext.run(inputs, this.wafTimeout) + const result = this.ddwafContext.run(payload, this.wafTimeout) const end = process.hrtime.bigint() this.addressesToSkip = newAddressesToSkip const ruleTriggered = !!result.events?.length - const blockTriggered = result.actions?.includes('block') + + const blockTriggered = !!getBlockingAction(result.actions) Reporter.reportMetrics({ duration: result.totalRuntime / 1e3, @@ -58,13 +87,17 @@ class WAFContextWrapper { blockTriggered, wafVersion: this.wafVersion, wafTimeout: result.timeout - }) + }, raspRuleType) if (ruleTriggered) { Reporter.reportAttack(JSON.stringify(result.events)) } - Reporter.reportSchemas(result.derivatives) + Reporter.reportDerivatives(result.derivatives) + + if (wafRunFinished.hasSubscribers) { + wafRunFinished.publish({ payload }) + } return result.actions } catch (err) { diff --git a/packages/dd-trace/src/appsec/waf/waf_manager.js b/packages/dd-trace/src/appsec/waf/waf_manager.js index deac04f80ed..b3cc91e6104 100644 --- a/packages/dd-trace/src/appsec/waf/waf_manager.js +++ b/packages/dd-trace/src/appsec/waf/waf_manager.js @@ -39,7 +39,8 @@ class WAFManager { this.ddwaf.createContext(), this.wafTimeout, this.ddwafVersion, - this.rulesVersion + this.rulesVersion, + this.ddwaf.knownAddresses ) contexts.set(req, wafContext) } @@ -50,6 +51,10 @@ class WAFManager { update (newRules) { this.ddwaf.update(newRules) + if (this.ddwaf.diagnostics.ruleset_version) { + this.rulesVersion = this.ddwaf.diagnostics.ruleset_version + } + Reporter.reportWafUpdate(this.ddwafVersion, this.rulesVersion) } diff --git a/packages/dd-trace/src/azure_metadata.js b/packages/dd-trace/src/azure_metadata.js new file mode 100644 index 00000000000..94c29c9dd16 --- /dev/null +++ b/packages/dd-trace/src/azure_metadata.js @@ -0,0 +1,120 @@ +'use strict' + +// eslint-disable-next-line max-len +// Modeled after https://github.com/DataDog/libdatadog/blob/f3994857a59bb5679a65967138c5a3aec418a65f/ddcommon/src/azure_app_services.rs + +const os = require('os') +const { getIsAzureFunction } = require('./serverless') + +function extractSubscriptionID (ownerName) { + if (ownerName !== undefined) { + const subId = ownerName.split('+')[0].trim() + if (subId.length > 0) { + return subId + } + } + return undefined +} + +function extractResourceGroup (ownerName) { + return /.+\+(.+)-.+webspace(-Linux)?/.exec(ownerName)?.[1] +} + +function buildResourceID (subscriptionID, siteName, resourceGroup) { + if (subscriptionID === undefined || siteName === undefined || resourceGroup === undefined) { + return undefined + } + return `/subscriptions/${subscriptionID}/resourcegroups/${resourceGroup}/providers/microsoft.web/sites/${siteName}` + .toLowerCase() +} + +function trimObject (obj) { + Object.entries(obj) + .filter(([_, value]) => value === undefined) + .forEach(([key, _]) => { delete obj[key] }) + return obj +} + +function buildMetadata () { + const { + COMPUTERNAME, + DD_AAS_DOTNET_EXTENSION_VERSION, + FUNCTIONS_EXTENSION_VERSION, + FUNCTIONS_WORKER_RUNTIME, + FUNCTIONS_WORKER_RUNTIME_VERSION, + WEBSITE_INSTANCE_ID, + WEBSITE_OWNER_NAME, + WEBSITE_OS, + WEBSITE_RESOURCE_GROUP, + WEBSITE_SITE_NAME + } = process.env + + const subscriptionID = extractSubscriptionID(WEBSITE_OWNER_NAME) + + const siteName = WEBSITE_SITE_NAME + + const [siteKind, siteType] = getIsAzureFunction() + ? ['functionapp', 'function'] + : ['app', 'app'] + + const resourceGroup = WEBSITE_RESOURCE_GROUP ?? extractResourceGroup(WEBSITE_OWNER_NAME) + + return trimObject({ + extensionVersion: DD_AAS_DOTNET_EXTENSION_VERSION, + functionRuntimeVersion: FUNCTIONS_EXTENSION_VERSION, + instanceID: WEBSITE_INSTANCE_ID, + instanceName: COMPUTERNAME, + operatingSystem: WEBSITE_OS ?? os.platform(), + resourceGroup, + resourceID: buildResourceID(subscriptionID, siteName, resourceGroup), + runtime: FUNCTIONS_WORKER_RUNTIME, + runtimeVersion: FUNCTIONS_WORKER_RUNTIME_VERSION, + siteKind, + siteName, + siteType, + subscriptionID + }) +} + +function getAzureAppMetadata () { + // DD_AZURE_APP_SERVICES is an environment variable introduced by the .NET APM team and is set automatically for + // anyone using the Datadog APM Extensions (.NET, Java, or Node) for Windows Azure App Services + // eslint-disable-next-line max-len + // See: https://github.com/DataDog/datadog-aas-extension/blob/01f94b5c28b7fa7a9ab264ca28bd4e03be603900/node/src/applicationHost.xdt#L20-L21 + return process.env.DD_AZURE_APP_SERVICES !== undefined ? buildMetadata() : undefined +} + +function getAzureFunctionMetadata () { + return getIsAzureFunction() ? buildMetadata() : undefined +} + +// eslint-disable-next-line max-len +// Modeled after https://github.com/DataDog/libdatadog/blob/92272e90a7919f07178f3246ef8f82295513cfed/profiling/src/exporter/mod.rs#L187 +// eslint-disable-next-line max-len +// and https://github.com/DataDog/libdatadog/blob/f3994857a59bb5679a65967138c5a3aec418a65f/trace-utils/src/trace_utils.rs#L533 +function getAzureTagsFromMetadata (metadata) { + if (metadata === undefined) { + return {} + } + return trimObject({ + 'aas.environment.extension_version': metadata.extensionVersion, + 'aas.environment.function_runtime': metadata.functionRuntimeVersion, + 'aas.environment.instance_id': metadata.instanceID, + 'aas.environment.instance_name': metadata.instanceName, + 'aas.environment.os': metadata.operatingSystem, + 'aas.environment.runtime': metadata.runtime, + 'aas.environment.runtime_version': metadata.runtimeVersion, + 'aas.resource.group': metadata.resourceGroup, + 'aas.resource.id': metadata.resourceID, + 'aas.site.kind': metadata.siteKind, + 'aas.site.name': metadata.siteName, + 'aas.site.type': metadata.siteType, + 'aas.subscription.id': metadata.subscriptionID + }) +} + +module.exports = { + getAzureAppMetadata, + getAzureFunctionMetadata, + getAzureTagsFromMetadata +} diff --git a/packages/dd-trace/src/ci-visibility/early-flake-detection/get-known-tests.js b/packages/dd-trace/src/ci-visibility/early-flake-detection/get-known-tests.js new file mode 100644 index 00000000000..3027baff50a --- /dev/null +++ b/packages/dd-trace/src/ci-visibility/early-flake-detection/get-known-tests.js @@ -0,0 +1,108 @@ +const request = require('../../exporters/common/request') +const id = require('../../id') +const log = require('../../log') + +const { + incrementCountMetric, + distributionMetric, + TELEMETRY_KNOWN_TESTS, + TELEMETRY_KNOWN_TESTS_MS, + TELEMETRY_KNOWN_TESTS_ERRORS, + TELEMETRY_KNOWN_TESTS_RESPONSE_TESTS, + TELEMETRY_KNOWN_TESTS_RESPONSE_BYTES +} = require('../../ci-visibility/telemetry') + +const { getNumFromKnownTests } = require('../../plugins/util/test') + +function getKnownTests ({ + url, + isEvpProxy, + evpProxyPrefix, + isGzipCompatible, + env, + service, + repositoryUrl, + sha, + osVersion, + osPlatform, + osArchitecture, + runtimeName, + runtimeVersion, + custom +}, done) { + const options = { + path: '/api/v2/ci/libraries/tests', + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + timeout: 20000, + url + } + + if (isGzipCompatible) { + options.headers['accept-encoding'] = 'gzip' + } + + if (isEvpProxy) { + options.path = `${evpProxyPrefix}/api/v2/ci/libraries/tests` + options.headers['X-Datadog-EVP-Subdomain'] = 'api' + } else { + const apiKey = process.env.DATADOG_API_KEY || process.env.DD_API_KEY + if (!apiKey) { + return done(new Error('Known tests were not fetched because Datadog API key is not defined.')) + } + + options.headers['dd-api-key'] = apiKey + } + + const data = JSON.stringify({ + data: { + id: id().toString(10), + type: 'ci_app_libraries_tests_request', + attributes: { + configurations: { + 'os.platform': osPlatform, + 'os.version': osVersion, + 'os.architecture': osArchitecture, + 'runtime.name': runtimeName, + 'runtime.version': runtimeVersion, + custom + }, + service, + env, + repository_url: repositoryUrl, + sha + } + } + }) + + incrementCountMetric(TELEMETRY_KNOWN_TESTS) + + const startTime = Date.now() + + request(data, options, (err, res, statusCode) => { + distributionMetric(TELEMETRY_KNOWN_TESTS_MS, {}, Date.now() - startTime) + if (err) { + incrementCountMetric(TELEMETRY_KNOWN_TESTS_ERRORS, { statusCode }) + done(err) + } else { + try { + const { data: { attributes: { tests: knownTests } } } = JSON.parse(res) + + const numTests = getNumFromKnownTests(knownTests) + + incrementCountMetric(TELEMETRY_KNOWN_TESTS_RESPONSE_TESTS, {}, numTests) + distributionMetric(TELEMETRY_KNOWN_TESTS_RESPONSE_BYTES, {}, res.length) + + log.debug(() => `Number of received known tests: ${numTests}`) + + done(null, knownTests) + } catch (err) { + done(err) + } + } + }) +} + +module.exports = { getKnownTests } diff --git a/packages/dd-trace/src/ci-visibility/exporters/agent-proxy/index.js b/packages/dd-trace/src/ci-visibility/exporters/agent-proxy/index.js index a9e5a444674..bb1367057f4 100644 --- a/packages/dd-trace/src/ci-visibility/exporters/agent-proxy/index.js +++ b/packages/dd-trace/src/ci-visibility/exporters/agent-proxy/index.js @@ -5,10 +5,23 @@ const AgentlessWriter = require('../agentless/writer') const CoverageWriter = require('../agentless/coverage-writer') const CiVisibilityExporter = require('../ci-visibility-exporter') -const AGENT_EVP_PROXY_PATH = '/evp_proxy/v2' +const AGENT_EVP_PROXY_PATH_PREFIX = '/evp_proxy/v' +const AGENT_EVP_PROXY_PATH_REGEX = /\/evp_proxy\/v(\d+)\/?/ -function getIsEvpCompatible (err, agentInfo) { - return !err && agentInfo.endpoints.some(url => url.includes(AGENT_EVP_PROXY_PATH)) +function getLatestEvpProxyVersion (err, agentInfo) { + if (err) { + return 0 + } + return agentInfo.endpoints.reduce((acc, endpoint) => { + if (endpoint.includes(AGENT_EVP_PROXY_PATH_PREFIX)) { + const version = Number(endpoint.replace(AGENT_EVP_PROXY_PATH_REGEX, '$1')) + if (isNaN(version)) { + return acc + } + return version > acc ? version : acc + } + return acc + }, 0) } class AgentProxyCiVisibilityExporter extends CiVisibilityExporter { @@ -25,17 +38,27 @@ class AgentProxyCiVisibilityExporter extends CiVisibilityExporter { this.getAgentInfo((err, agentInfo) => { this._isInitialized = true - const isEvpCompatible = getIsEvpCompatible(err, agentInfo) + let latestEvpProxyVersion = getLatestEvpProxyVersion(err, agentInfo) + const isEvpCompatible = latestEvpProxyVersion >= 2 + const isGzipCompatible = latestEvpProxyVersion >= 4 + + // v3 does not work well citestcycle, so we downgrade to v2 + if (latestEvpProxyVersion === 3) { + latestEvpProxyVersion = 2 + } + + const evpProxyPrefix = `${AGENT_EVP_PROXY_PATH_PREFIX}${latestEvpProxyVersion}` if (isEvpCompatible) { this._isUsingEvpProxy = true + this.evpProxyPrefix = evpProxyPrefix this._writer = new AgentlessWriter({ url: this._url, tags, - evpProxyPrefix: AGENT_EVP_PROXY_PATH + evpProxyPrefix }) this._coverageWriter = new CoverageWriter({ url: this._url, - evpProxyPrefix: AGENT_EVP_PROXY_PATH + evpProxyPrefix }) } else { this._writer = new AgentWriter({ @@ -51,6 +74,7 @@ class AgentProxyCiVisibilityExporter extends CiVisibilityExporter { this._resolveCanUseCiVisProtocol(isEvpCompatible) this.exportUncodedTraces() this.exportUncodedCoverages() + this._isGzipCompatible = isGzipCompatible }) } diff --git a/packages/dd-trace/src/ci-visibility/exporters/agentless/coverage-writer.js b/packages/dd-trace/src/ci-visibility/exporters/agentless/coverage-writer.js index 8728e4a2e04..98eff61a6fd 100644 --- a/packages/dd-trace/src/ci-visibility/exporters/agentless/coverage-writer.js +++ b/packages/dd-trace/src/ci-visibility/exporters/agentless/coverage-writer.js @@ -5,6 +5,15 @@ const { safeJSONStringify } = require('../../../exporters/common/util') const { CoverageCIVisibilityEncoder } = require('../../../encode/coverage-ci-visibility') const BaseWriter = require('../../../exporters/common/writer') +const { + incrementCountMetric, + distributionMetric, + TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS, + TELEMETRY_ENDPOINT_PAYLOAD_BYTES, + TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_MS, + TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_ERRORS, + TELEMETRY_ENDPOINT_PAYLOAD_DROPPED +} = require('../../../ci-visibility/telemetry') class Writer extends BaseWriter { constructor ({ url, evpProxyPrefix = '' }) { @@ -34,8 +43,26 @@ class Writer extends BaseWriter { log.debug(() => `Request to the intake: ${safeJSONStringify(options)}`) - request(form, options, (err, res) => { + const startRequestTime = Date.now() + + incrementCountMetric(TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS, { endpoint: 'code_coverage' }) + distributionMetric(TELEMETRY_ENDPOINT_PAYLOAD_BYTES, { endpoint: 'code_coverage' }, form.size()) + + request(form, options, (err, res, statusCode) => { + distributionMetric( + TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_MS, + { endpoint: 'code_coverage' }, + Date.now() - startRequestTime + ) if (err) { + incrementCountMetric( + TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_ERRORS, + { endpoint: 'code_coverage', statusCode } + ) + incrementCountMetric( + TELEMETRY_ENDPOINT_PAYLOAD_DROPPED, + { endpoint: 'code_coverage' } + ) log.error(err) done() return diff --git a/packages/dd-trace/src/ci-visibility/exporters/agentless/index.js b/packages/dd-trace/src/ci-visibility/exporters/agentless/index.js index 70276a3521c..dcbded6a54e 100644 --- a/packages/dd-trace/src/ci-visibility/exporters/agentless/index.js +++ b/packages/dd-trace/src/ci-visibility/exporters/agentless/index.js @@ -21,6 +21,8 @@ class AgentlessCiVisibilityExporter extends CiVisibilityExporter { this._coverageWriter = new CoverageWriter({ url: this._coverageUrl }) this._apiUrl = url || new URL(`https://api.${site}`) + // Agentless is always gzip compatible + this._isGzipCompatible = true } setUrl (url, coverageUrl = url, apiUrl = url) { diff --git a/packages/dd-trace/src/ci-visibility/exporters/agentless/writer.js b/packages/dd-trace/src/ci-visibility/exporters/agentless/writer.js index d04406f33b9..466c5230b22 100644 --- a/packages/dd-trace/src/ci-visibility/exporters/agentless/writer.js +++ b/packages/dd-trace/src/ci-visibility/exporters/agentless/writer.js @@ -5,6 +5,15 @@ const log = require('../../../log') const { AgentlessCiVisibilityEncoder } = require('../../../encode/agentless-ci-visibility') const BaseWriter = require('../../../exporters/common/writer') +const { + incrementCountMetric, + distributionMetric, + TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS, + TELEMETRY_ENDPOINT_PAYLOAD_BYTES, + TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_MS, + TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_ERRORS, + TELEMETRY_ENDPOINT_PAYLOAD_DROPPED +} = require('../../../ci-visibility/telemetry') class Writer extends BaseWriter { constructor ({ url, tags, evpProxyPrefix = '' }) { @@ -35,8 +44,26 @@ class Writer extends BaseWriter { log.debug(() => `Request to the intake: ${safeJSONStringify(options)}`) - request(data, options, (err, res) => { + const startRequestTime = Date.now() + + incrementCountMetric(TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS, { endpoint: 'test_cycle' }) + distributionMetric(TELEMETRY_ENDPOINT_PAYLOAD_BYTES, { endpoint: 'test_cycle' }, data.length) + + request(data, options, (err, res, statusCode) => { + distributionMetric( + TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_MS, + { endpoint: 'test_cycle' }, + Date.now() - startRequestTime + ) if (err) { + incrementCountMetric( + TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_ERRORS, + { endpoint: 'test_cycle', statusCode } + ) + incrementCountMetric( + TELEMETRY_ENDPOINT_PAYLOAD_DROPPED, + { endpoint: 'test_cycle' } + ) log.error(err) done() return @@ -45,6 +72,10 @@ class Writer extends BaseWriter { done() }) } + + setMetadataTags (tags) { + this._encoder.setMetadataTags(tags) + } } module.exports = Writer diff --git a/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js b/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js index 446479e1af7..9dabd34f7f3 100644 --- a/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js +++ b/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js @@ -3,8 +3,9 @@ const URL = require('url').URL const { sendGitMetadata: sendGitMetadataRequest } = require('./git/git_metadata') -const { getItrConfiguration: getItrConfigurationRequest } = require('../intelligent-test-runner/get-itr-configuration') +const { getLibraryConfiguration: getLibraryConfigurationRequest } = require('../requests/get-library-configuration') const { getSkippableSuites: getSkippableSuitesRequest } = require('../intelligent-test-runner/get-skippable-suites') +const { getKnownTests: getKnownTestsRequest } = require('../early-flake-detection/get-known-tests') const log = require('../../log') const AgentInfoExporter = require('../../exporters/common/agent-info-exporter') @@ -76,11 +77,18 @@ class CiVisibilityExporter extends AgentInfoExporter { shouldRequestSkippableSuites () { return !!(this._config.isIntelligentTestRunnerEnabled && this._canUseCiVisProtocol && - this._itrConfig && - this._itrConfig.isSuitesSkippingEnabled) + this._libraryConfig?.isSuitesSkippingEnabled) } - shouldRequestItrConfiguration () { + shouldRequestKnownTests () { + return !!( + this._config.isEarlyFlakeDetectionEnabled && + this._canUseCiVisProtocol && + this._libraryConfig?.isEarlyFlakeDetectionEnabled + ) + } + + shouldRequestLibraryConfiguration () { return this._config.isIntelligentTestRunnerEnabled } @@ -92,6 +100,19 @@ class CiVisibilityExporter extends AgentInfoExporter { return this._canUseCiVisProtocol } + getRequestConfiguration (testConfiguration) { + return { + url: this._getApiUrl(), + env: this._config.env, + service: this._config.service, + isEvpProxy: !!this._isUsingEvpProxy, + isGzipCompatible: this._isGzipCompatible, + evpProxyPrefix: this.evpProxyPrefix, + custom: getTestConfigurationTags(this._config.tags), + ...testConfiguration + } + } + // We can't call the skippable endpoint until git upload has finished, // hence the this._gitUploadPromise.then getSkippableSuites (testConfiguration, callback) { @@ -102,68 +123,89 @@ class CiVisibilityExporter extends AgentInfoExporter { if (gitUploadError) { return callback(gitUploadError, []) } - const configuration = { - url: this._getApiUrl(), - site: this._config.site, - env: this._config.env, - service: this._config.service, - isEvpProxy: !!this._isUsingEvpProxy, - custom: getTestConfigurationTags(this._config.tags), - ...testConfiguration - } - getSkippableSuitesRequest(configuration, callback) + getSkippableSuitesRequest(this.getRequestConfiguration(testConfiguration), callback) }) } + getKnownTests (testConfiguration, callback) { + if (!this.shouldRequestKnownTests()) { + return callback(null) + } + getKnownTestsRequest(this.getRequestConfiguration(testConfiguration), callback) + } + /** - * We can't request ITR configuration until we know whether we can use the + * We can't request library configuration until we know whether we can use the * CI Visibility Protocol, hence the this._canUseCiVisProtocol promise. */ - getItrConfiguration (testConfiguration, callback) { + getLibraryConfiguration (testConfiguration, callback) { const { repositoryUrl } = testConfiguration this.sendGitMetadata(repositoryUrl) - if (!this.shouldRequestItrConfiguration()) { + if (!this.shouldRequestLibraryConfiguration()) { return callback(null, {}) } this._canUseCiVisProtocolPromise.then((canUseCiVisProtocol) => { if (!canUseCiVisProtocol) { return callback(null, {}) } - const configuration = { - url: this._getApiUrl(), - env: this._config.env, - service: this._config.service, - isEvpProxy: !!this._isUsingEvpProxy, - custom: getTestConfigurationTags(this._config.tags), - ...testConfiguration - } - getItrConfigurationRequest(configuration, (err, itrConfig) => { + const configuration = this.getRequestConfiguration(testConfiguration) + + getLibraryConfigurationRequest(configuration, (err, libraryConfig) => { /** - * **Important**: this._itrConfig remains empty in testing frameworks - * where the tests run in a subprocess, because `getItrConfiguration` is called only once. + * **Important**: this._libraryConfig remains empty in testing frameworks + * where the tests run in a subprocess, like Jest, + * because `getLibraryConfiguration` is called only once in the main process. */ - this._itrConfig = itrConfig + this._libraryConfig = this.filterConfiguration(libraryConfig) if (err) { callback(err, {}) - } else if (itrConfig?.requireGit) { + } else if (libraryConfig?.requireGit) { // If the backend requires git, we'll wait for the upload to finish and request settings again this._gitUploadPromise.then(gitUploadError => { if (gitUploadError) { return callback(gitUploadError, {}) } - getItrConfigurationRequest(configuration, (err, finalItrConfig) => { - this._itrConfig = finalItrConfig - callback(err, finalItrConfig) + getLibraryConfigurationRequest(configuration, (err, finalLibraryConfig) => { + this._libraryConfig = this.filterConfiguration(finalLibraryConfig) + callback(err, this._libraryConfig) }) }) } else { - callback(null, itrConfig) + callback(null, this._libraryConfig) } }) }) } + // Takes into account potential kill switches + filterConfiguration (remoteConfiguration) { + if (!remoteConfiguration) { + return {} + } + const { + isCodeCoverageEnabled, + isSuitesSkippingEnabled, + isItrEnabled, + requireGit, + isEarlyFlakeDetectionEnabled, + earlyFlakeDetectionNumRetries, + earlyFlakeDetectionFaultyThreshold, + isFlakyTestRetriesEnabled + } = remoteConfiguration + return { + isCodeCoverageEnabled, + isSuitesSkippingEnabled, + isItrEnabled, + requireGit, + isEarlyFlakeDetectionEnabled: isEarlyFlakeDetectionEnabled && this._config.isEarlyFlakeDetectionEnabled, + earlyFlakeDetectionNumRetries, + earlyFlakeDetectionFaultyThreshold, + isFlakyTestRetriesEnabled: isFlakyTestRetriesEnabled && this._config.isFlakyTestRetriesEnabled, + flakyTestRetriesCount: this._config.flakyTestRetriesCount + } + } + sendGitMetadata (repositoryUrl) { if (!this._config.isGitUploadEnabled) { return @@ -172,14 +214,19 @@ class CiVisibilityExporter extends AgentInfoExporter { if (!canUseCiVisProtocol) { return } - sendGitMetadataRequest(this._getApiUrl(), !!this._isUsingEvpProxy, repositoryUrl, (err) => { - if (err) { - log.error(`Error uploading git metadata: ${err.message}`) - } else { - log.debug('Successfully uploaded git metadata') + sendGitMetadataRequest( + this._getApiUrl(), + { isEvpProxy: !!this._isUsingEvpProxy, evpProxyPrefix: this.evpProxyPrefix }, + repositoryUrl, + (err) => { + if (err) { + log.error(`Error uploading git metadata: ${err.message}`) + } else { + log.debug('Successfully uploaded git metadata') + } + this._resolveGit(err) } - this._resolveGit(err) - }) + ) }) } @@ -244,6 +291,19 @@ class CiVisibilityExporter extends AgentInfoExporter { _getApiUrl () { return this._url } + + // By the time setMetadataTags is called, the agent info request might not have finished + setMetadataTags (tags) { + if (this._writer?.setMetadataTags) { + this._writer.setMetadataTags(tags) + } else { + this._canUseCiVisProtocolPromise.then(() => { + if (this._writer?.setMetadataTags) { + this._writer.setMetadataTags(tags) + } + }) + } + } } module.exports = CiVisibilityExporter diff --git a/packages/dd-trace/src/ci-visibility/exporters/git/git_metadata.js b/packages/dd-trace/src/ci-visibility/exporters/git/git_metadata.js index 7c5201566ff..1585a94166f 100644 --- a/packages/dd-trace/src/ci-visibility/exporters/git/git_metadata.js +++ b/packages/dd-trace/src/ci-visibility/exporters/git/git_metadata.js @@ -1,4 +1,3 @@ - const fs = require('fs') const path = require('path') @@ -10,11 +9,25 @@ const { getLatestCommits, getRepositoryUrl, generatePackFilesForCommits, - getCommitsToUpload, + getCommitsRevList, isShallowRepository, - unshallowRepository + unshallowRepository, + isGitAvailable } = require('../../../plugins/util/git') +const { + incrementCountMetric, + distributionMetric, + TELEMETRY_GIT_REQUESTS_SEARCH_COMMITS, + TELEMETRY_GIT_REQUESTS_SEARCH_COMMITS_MS, + TELEMETRY_GIT_REQUESTS_SEARCH_COMMITS_ERRORS, + TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_NUM, + TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES, + TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_MS, + TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_ERRORS, + TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_BYTES +} = require('../../../ci-visibility/telemetry') + const isValidSha1 = (sha) => /^[0-9a-f]{40}$/.test(sha) const isValidSha256 = (sha) => /^[0-9a-f]{64}$/.test(sha) @@ -46,11 +59,7 @@ function getCommonRequestOptions (url) { * The response are the commits for which the backend already has information * This response is used to know which commits can be ignored from there on */ -function getCommitsToExclude ({ url, isEvpProxy, repositoryUrl }, callback) { - const latestCommits = getLatestCommits() - - log.debug(`There were ${latestCommits.length} commits since last month.`) - +function getCommitsToUpload ({ url, repositoryUrl, latestCommits, isEvpProxy, evpProxyPrefix }, callback) { const commonOptions = getCommonRequestOptions(url) const options = { @@ -63,7 +72,7 @@ function getCommitsToExclude ({ url, isEvpProxy, repositoryUrl }, callback) { } if (isEvpProxy) { - options.path = '/evp_proxy/v2/api/v2/git/repository/search_commits' + options.path = `${evpProxyPrefix}/api/v2/git/repository/search_commits` options.headers['X-Datadog-EVP-Subdomain'] = 'api' delete options.headers['dd-api-key'] } @@ -78,25 +87,44 @@ function getCommitsToExclude ({ url, isEvpProxy, repositoryUrl }, callback) { })) }) - request(localCommitData, options, (err, response) => { + incrementCountMetric(TELEMETRY_GIT_REQUESTS_SEARCH_COMMITS) + const startTime = Date.now() + request(localCommitData, options, (err, response, statusCode) => { + distributionMetric(TELEMETRY_GIT_REQUESTS_SEARCH_COMMITS_MS, {}, Date.now() - startTime) if (err) { + incrementCountMetric(TELEMETRY_GIT_REQUESTS_SEARCH_COMMITS_ERRORS, { statusCode }) const error = new Error(`Error fetching commits to exclude: ${err.message}`) return callback(error) } - let commitsToExclude + let alreadySeenCommits try { - commitsToExclude = validateCommits(JSON.parse(response).data) + alreadySeenCommits = validateCommits(JSON.parse(response).data) } catch (e) { + incrementCountMetric(TELEMETRY_GIT_REQUESTS_SEARCH_COMMITS_ERRORS, { errorType: 'network' }) return callback(new Error(`Can't parse commits to exclude response: ${e.message}`)) } - callback(null, commitsToExclude, latestCommits) + log.debug(`There are ${alreadySeenCommits.length} commits to exclude.`) + const commitsToInclude = latestCommits.filter((commit) => !alreadySeenCommits.includes(commit)) + log.debug(`There are ${commitsToInclude.length} commits to include.`) + + if (!commitsToInclude.length) { + return callback(null, []) + } + + const commitsToUpload = getCommitsRevList(alreadySeenCommits, commitsToInclude) + + if (commitsToUpload === null) { + return callback(new Error('git rev-list failed')) + } + + callback(null, commitsToUpload) }) } /** * This function uploads a git packfile */ -function uploadPackFile ({ url, isEvpProxy, packFileToUpload, repositoryUrl, headCommit }, callback) { +function uploadPackFile ({ url, isEvpProxy, evpProxyPrefix, packFileToUpload, repositoryUrl, headCommit }, callback) { const form = new FormData() const pushedSha = JSON.stringify({ @@ -136,24 +164,88 @@ function uploadPackFile ({ url, isEvpProxy, packFileToUpload, repositoryUrl, hea } if (isEvpProxy) { - options.path = '/evp_proxy/v2/api/v2/git/repository/packfile' + options.path = `${evpProxyPrefix}/api/v2/git/repository/packfile` options.headers['X-Datadog-EVP-Subdomain'] = 'api' delete options.headers['dd-api-key'] } + incrementCountMetric(TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES) + + const uploadSize = form.size() + + const startTime = Date.now() request(form, options, (err, _, statusCode) => { + distributionMetric(TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_MS, {}, Date.now() - startTime) if (err) { + incrementCountMetric(TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_ERRORS, { statusCode }) const error = new Error(`Could not upload packfiles: status code ${statusCode}: ${err.message}`) - return callback(error) + return callback(error, uploadSize) } - callback(null) + callback(null, uploadSize) }) } +function generateAndUploadPackFiles ({ + url, + isEvpProxy, + evpProxyPrefix, + commitsToUpload, + repositoryUrl, + headCommit +}, callback) { + log.debug(`There are ${commitsToUpload.length} commits to upload`) + + const packFilesToUpload = generatePackFilesForCommits(commitsToUpload) + + log.debug(`Uploading ${packFilesToUpload.length} packfiles.`) + + if (!packFilesToUpload.length) { + return callback(new Error('Failed to generate packfiles')) + } + + distributionMetric(TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_NUM, {}, packFilesToUpload.length) + let packFileIndex = 0 + let totalUploadedBytes = 0 + // This uploads packfiles sequentially + const uploadPackFileCallback = (err, byteLength) => { + totalUploadedBytes += byteLength + if (err || packFileIndex === packFilesToUpload.length) { + distributionMetric(TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_BYTES, {}, totalUploadedBytes) + return callback(err) + } + return uploadPackFile( + { + packFileToUpload: packFilesToUpload[packFileIndex++], + url, + isEvpProxy, + evpProxyPrefix, + repositoryUrl, + headCommit + }, + uploadPackFileCallback + ) + } + + uploadPackFile( + { + packFileToUpload: packFilesToUpload[packFileIndex++], + url, + isEvpProxy, + evpProxyPrefix, + repositoryUrl, + headCommit + }, + uploadPackFileCallback + ) +} + /** * This function uploads git metadata to CI Visibility's backend. */ -function sendGitMetadata (url, isEvpProxy, configRepositoryUrl, callback) { +function sendGitMetadata (url, { isEvpProxy, evpProxyPrefix }, configRepositoryUrl, callback) { + if (!isGitAvailable()) { + return callback(new Error('Git is not available')) + } let repositoryUrl = configRepositoryUrl if (!repositoryUrl) { repositoryUrl = getRepositoryUrl() @@ -165,65 +257,53 @@ function sendGitMetadata (url, isEvpProxy, configRepositoryUrl, callback) { return callback(new Error('Repository URL is empty')) } - if (isShallowRepository()) { - log.debug('It is shallow clone, unshallowing...') - unshallowRepository() - } + let latestCommits = getLatestCommits() + log.debug(`There were ${latestCommits.length} commits since last month.`) - getCommitsToExclude({ url, repositoryUrl, isEvpProxy }, (err, commitsToExclude, latestCommits) => { + const getOnFinishGetCommitsToUpload = (hasCheckedShallow) => (err, commitsToUpload) => { if (err) { return callback(err) } - log.debug(`There are ${commitsToExclude.length} commits to exclude.`) - const [headCommit] = latestCommits - const commitsToInclude = latestCommits.filter((commit) => !commitsToExclude.includes(commit)) - log.debug(`There are ${commitsToInclude.length} commits to include.`) - - const commitsToUpload = getCommitsToUpload(commitsToExclude, commitsToInclude) if (!commitsToUpload.length) { log.debug('No commits to upload') return callback(null) } - log.debug(`There are ${commitsToUpload.length} commits to upload`) - - const packFilesToUpload = generatePackFilesForCommits(commitsToUpload) - log.debug(`Uploading ${packFilesToUpload.length} packfiles.`) - - if (!packFilesToUpload.length) { - return callback(new Error('Failed to generate packfiles')) - } - - let packFileIndex = 0 - // This uploads packfiles sequentially - const uploadPackFileCallback = (err) => { - if (err || packFileIndex === packFilesToUpload.length) { - return callback(err) - } - return uploadPackFile( - { - packFileToUpload: packFilesToUpload[packFileIndex++], - url, - isEvpProxy, - repositoryUrl, - headCommit - }, - uploadPackFileCallback - ) - } - - uploadPackFile( - { - packFileToUpload: packFilesToUpload[packFileIndex++], + // If it has already unshallowed or the clone is not shallow, we move on + if (hasCheckedShallow || !isShallowRepository()) { + const [headCommit] = latestCommits + return generateAndUploadPackFiles({ url, isEvpProxy, + evpProxyPrefix, + commitsToUpload, repositoryUrl, headCommit - }, - uploadPackFileCallback - ) - }) + }, callback) + } + // Otherwise we unshallow and get commits to upload again + log.debug('It is shallow clone, unshallowing...') + unshallowRepository() + + // The latest commits change after unshallowing + latestCommits = getLatestCommits() + getCommitsToUpload({ + url, + repositoryUrl, + latestCommits, + isEvpProxy, + evpProxyPrefix + }, getOnFinishGetCommitsToUpload(true)) + } + + getCommitsToUpload({ + url, + repositoryUrl, + latestCommits, + isEvpProxy, + evpProxyPrefix + }, getOnFinishGetCommitsToUpload(false)) } module.exports = { diff --git a/packages/dd-trace/src/ci-visibility/exporters/jest-worker/index.js b/packages/dd-trace/src/ci-visibility/exporters/jest-worker/index.js deleted file mode 100644 index 8f96937e22f..00000000000 --- a/packages/dd-trace/src/ci-visibility/exporters/jest-worker/index.js +++ /dev/null @@ -1,33 +0,0 @@ -'use strict' - -const Writer = require('./writer') -const { - JEST_WORKER_COVERAGE_PAYLOAD_CODE, - JEST_WORKER_TRACE_PAYLOAD_CODE -} = require('../../../plugins/util/test') - -/** - * Lightweight exporter whose writers only do simple JSON serialization - * of trace and coverage payloads, which they send to the jest main process. - */ -class JestWorkerCiVisibilityExporter { - constructor () { - this._writer = new Writer(JEST_WORKER_TRACE_PAYLOAD_CODE) - this._coverageWriter = new Writer(JEST_WORKER_COVERAGE_PAYLOAD_CODE) - } - - export (payload) { - this._writer.append(payload) - } - - exportCoverage (formattedCoverage) { - this._coverageWriter.append(formattedCoverage) - } - - flush () { - this._writer.flush() - this._coverageWriter.flush() - } -} - -module.exports = JestWorkerCiVisibilityExporter diff --git a/packages/dd-trace/src/ci-visibility/exporters/test-worker/index.js b/packages/dd-trace/src/ci-visibility/exporters/test-worker/index.js new file mode 100644 index 00000000000..e74869dbe82 --- /dev/null +++ b/packages/dd-trace/src/ci-visibility/exporters/test-worker/index.js @@ -0,0 +1,60 @@ +'use strict' + +const Writer = require('./writer') +const { + JEST_WORKER_COVERAGE_PAYLOAD_CODE, + JEST_WORKER_TRACE_PAYLOAD_CODE, + CUCUMBER_WORKER_TRACE_PAYLOAD_CODE, + MOCHA_WORKER_TRACE_PAYLOAD_CODE +} = require('../../../plugins/util/test') + +function getInterprocessTraceCode () { + if (process.env.JEST_WORKER_ID) { + return JEST_WORKER_TRACE_PAYLOAD_CODE + } + if (process.env.CUCUMBER_WORKER_ID) { + return CUCUMBER_WORKER_TRACE_PAYLOAD_CODE + } + if (process.env.MOCHA_WORKER_ID) { + return MOCHA_WORKER_TRACE_PAYLOAD_CODE + } + return null +} + +// TODO: make it available with cucumber +function getInterprocessCoverageCode () { + if (process.env.JEST_WORKER_ID) { + return JEST_WORKER_COVERAGE_PAYLOAD_CODE + } + return null +} + +/** + * Lightweight exporter whose writers only do simple JSON serialization + * of trace and coverage payloads, which they send to the test framework's main process. + * Currently used by Jest and Cucumber workers. + */ +class TestWorkerCiVisibilityExporter { + constructor () { + const interprocessTraceCode = getInterprocessTraceCode() + const interprocessCoverageCode = getInterprocessCoverageCode() + + this._writer = new Writer(interprocessTraceCode) + this._coverageWriter = new Writer(interprocessCoverageCode) + } + + export (payload) { + this._writer.append(payload) + } + + exportCoverage (formattedCoverage) { + this._coverageWriter.append(formattedCoverage) + } + + flush () { + this._writer.flush() + this._coverageWriter.flush() + } +} + +module.exports = TestWorkerCiVisibilityExporter diff --git a/packages/dd-trace/src/ci-visibility/exporters/jest-worker/writer.js b/packages/dd-trace/src/ci-visibility/exporters/test-worker/writer.js similarity index 76% rename from packages/dd-trace/src/ci-visibility/exporters/jest-worker/writer.js rename to packages/dd-trace/src/ci-visibility/exporters/test-worker/writer.js index 8b4ada0c500..d5004b28273 100644 --- a/packages/dd-trace/src/ci-visibility/exporters/jest-worker/writer.js +++ b/packages/dd-trace/src/ci-visibility/exporters/test-worker/writer.js @@ -23,11 +23,18 @@ class Writer { } _sendPayload (data) { + // ## Jest // Only available when `child_process` is used for the jest worker. // eslint-disable-next-line // https://github.com/facebook/jest/blob/bb39cb2c617a3334bf18daeca66bd87b7ccab28b/packages/jest-worker/README.md#experimental-worker // If worker_threads is used, this will not work // TODO: make it compatible with worker_threads + + // ## Cucumber + // This reports to the test's main process the same way test data is reported by Cucumber + // See cucumber code: + // eslint-disable-next-line + // https://github.com/cucumber/cucumber-js/blob/5ce371870b677fe3d1a14915dc535688946f734c/src/runtime/parallel/run_worker.ts#L13 if (process.send) { // it only works if process.send is available process.send([this._interprocessCode, data]) } diff --git a/packages/dd-trace/src/ci-visibility/intelligent-test-runner/get-skippable-suites.js b/packages/dd-trace/src/ci-visibility/intelligent-test-runner/get-skippable-suites.js index 04448e9a651..3ff8f3afde3 100644 --- a/packages/dd-trace/src/ci-visibility/intelligent-test-runner/get-skippable-suites.js +++ b/packages/dd-trace/src/ci-visibility/intelligent-test-runner/get-skippable-suites.js @@ -1,9 +1,21 @@ const request = require('../../exporters/common/request') const log = require('../../log') +const { + incrementCountMetric, + distributionMetric, + TELEMETRY_ITR_SKIPPABLE_TESTS, + TELEMETRY_ITR_SKIPPABLE_TESTS_MS, + TELEMETRY_ITR_SKIPPABLE_TESTS_ERRORS, + TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_SUITES, + TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_TESTS, + TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_BYTES +} = require('../../ci-visibility/telemetry') function getSkippableSuites ({ url, isEvpProxy, + evpProxyPrefix, + isGzipCompatible, env, service, repositoryUrl, @@ -26,8 +38,12 @@ function getSkippableSuites ({ url } + if (isGzipCompatible) { + options.headers['accept-encoding'] = 'gzip' + } + if (isEvpProxy) { - options.path = '/evp_proxy/v2/api/v2/ci/tests/skippable' + options.path = `${evpProxyPrefix}/api/v2/ci/tests/skippable` options.headers['X-Datadog-EVP-Subdomain'] = 'api' } else { const apiKey = process.env.DATADOG_API_KEY || process.env.DD_API_KEY @@ -59,13 +75,20 @@ function getSkippableSuites ({ } }) - request(data, options, (err, res) => { + incrementCountMetric(TELEMETRY_ITR_SKIPPABLE_TESTS) + + const startTime = Date.now() + + request(data, options, (err, res, statusCode) => { + distributionMetric(TELEMETRY_ITR_SKIPPABLE_TESTS_MS, {}, Date.now() - startTime) if (err) { + incrementCountMetric(TELEMETRY_ITR_SKIPPABLE_TESTS_ERRORS, { statusCode }) done(err) } else { let skippableSuites = [] try { - skippableSuites = JSON.parse(res) + const parsedResponse = JSON.parse(res) + skippableSuites = parsedResponse .data .filter(({ type }) => type === testLevel) .map(({ attributes: { suite, name } }) => { @@ -74,8 +97,17 @@ function getSkippableSuites ({ } return { suite, name } }) + const { meta: { correlation_id: correlationId } } = parsedResponse + incrementCountMetric( + testLevel === 'test' + ? TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_TESTS + : TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_SUITES, + {}, + skippableSuites.length + ) + distributionMetric(TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_BYTES, {}, res.length) log.debug(() => `Number of received skippable ${testLevel}s: ${skippableSuites.length}`) - done(null, skippableSuites) + done(null, skippableSuites, correlationId) } catch (err) { done(err) } diff --git a/packages/dd-trace/src/ci-visibility/log-submission/log-submission-plugin.js b/packages/dd-trace/src/ci-visibility/log-submission/log-submission-plugin.js new file mode 100644 index 00000000000..aa437f4cd87 --- /dev/null +++ b/packages/dd-trace/src/ci-visibility/log-submission/log-submission-plugin.js @@ -0,0 +1,53 @@ +const Plugin = require('../../plugins/plugin') +const log = require('../../log') + +function getWinstonLogSubmissionParameters (config) { + const { site, service } = config + + const defaultParameters = { + host: `http-intake.logs.${site}`, + path: `/api/v2/logs?ddsource=winston&service=${service}`, + ssl: true, + headers: { + 'DD-API-KEY': process.env.DD_API_KEY + } + } + + if (!process.env.DD_AGENTLESS_LOG_SUBMISSION_URL) { + return defaultParameters + } + + try { + const url = new URL(process.env.DD_AGENTLESS_LOG_SUBMISSION_URL) + return { + host: url.hostname, + port: url.port, + ssl: url.protocol === 'https:', + path: defaultParameters.path, + headers: defaultParameters.headers + } + } catch (e) { + log.error('Could not parse DD_AGENTLESS_LOG_SUBMISSION_URL') + return defaultParameters + } +} + +class LogSubmissionPlugin extends Plugin { + static get id () { + return 'log-submission' + } + + constructor (...args) { + super(...args) + + this.addSub('ci:log-submission:winston:configure', (httpClass) => { + this.HttpClass = httpClass + }) + + this.addSub('ci:log-submission:winston:add-transport', (logger) => { + logger.add(new this.HttpClass(getWinstonLogSubmissionParameters(this.config))) + }) + } +} + +module.exports = LogSubmissionPlugin diff --git a/packages/dd-trace/src/ci-visibility/intelligent-test-runner/get-itr-configuration.js b/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js similarity index 57% rename from packages/dd-trace/src/ci-visibility/intelligent-test-runner/get-itr-configuration.js rename to packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js index 2aee819004d..9a32efad05e 100644 --- a/packages/dd-trace/src/ci-visibility/intelligent-test-runner/get-itr-configuration.js +++ b/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js @@ -1,10 +1,22 @@ const request = require('../../exporters/common/request') const id = require('../../id') const log = require('../../log') +const { + incrementCountMetric, + distributionMetric, + TELEMETRY_GIT_REQUESTS_SETTINGS, + TELEMETRY_GIT_REQUESTS_SETTINGS_MS, + TELEMETRY_GIT_REQUESTS_SETTINGS_ERRORS, + TELEMETRY_GIT_REQUESTS_SETTINGS_RESPONSE +} = require('../telemetry') -function getItrConfiguration ({ +const DEFAULT_EARLY_FLAKE_DETECTION_NUM_RETRIES = 2 +const DEFAULT_EARLY_FLAKE_DETECTION_ERROR_THRESHOLD = 30 + +function getLibraryConfiguration ({ url, isEvpProxy, + evpProxyPrefix, env, service, repositoryUrl, @@ -29,7 +41,7 @@ function getItrConfiguration ({ } if (isEvpProxy) { - options.path = '/evp_proxy/v2/api/v2/libraries/tests/services/setting' + options.path = `${evpProxyPrefix}/api/v2/libraries/tests/services/setting` options.headers['X-Datadog-EVP-Subdomain'] = 'api' } else { const apiKey = process.env.DATADOG_API_KEY || process.env.DD_API_KEY @@ -62,8 +74,13 @@ function getItrConfiguration ({ } }) - request(data, options, (err, res) => { + incrementCountMetric(TELEMETRY_GIT_REQUESTS_SETTINGS) + + const startTime = Date.now() + request(data, options, (err, res, statusCode) => { + distributionMetric(TELEMETRY_GIT_REQUESTS_SETTINGS_MS, {}, Date.now() - startTime) if (err) { + incrementCountMetric(TELEMETRY_GIT_REQUESTS_SETTINGS_ERRORS, { statusCode }) done(err) } else { try { @@ -73,12 +90,25 @@ function getItrConfiguration ({ code_coverage: isCodeCoverageEnabled, tests_skipping: isSuitesSkippingEnabled, itr_enabled: isItrEnabled, - require_git: requireGit + require_git: requireGit, + early_flake_detection: earlyFlakeDetectionConfig, + flaky_test_retries_enabled: isFlakyTestRetriesEnabled } } } = JSON.parse(res) - const settings = { isCodeCoverageEnabled, isSuitesSkippingEnabled, isItrEnabled, requireGit } + const settings = { + isCodeCoverageEnabled, + isSuitesSkippingEnabled, + isItrEnabled, + requireGit, + isEarlyFlakeDetectionEnabled: earlyFlakeDetectionConfig?.enabled ?? false, + earlyFlakeDetectionNumRetries: + earlyFlakeDetectionConfig?.slow_test_retries?.['5s'] || DEFAULT_EARLY_FLAKE_DETECTION_NUM_RETRIES, + earlyFlakeDetectionFaultyThreshold: + earlyFlakeDetectionConfig?.faulty_session_threshold ?? DEFAULT_EARLY_FLAKE_DETECTION_ERROR_THRESHOLD, + isFlakyTestRetriesEnabled + } log.debug(() => `Remote settings: ${JSON.stringify(settings)}`) @@ -91,6 +121,8 @@ function getItrConfiguration ({ log.debug(() => 'Dangerously set test skipping to true') } + incrementCountMetric(TELEMETRY_GIT_REQUESTS_SETTINGS_RESPONSE, settings) + done(null, settings) } catch (err) { done(err) @@ -99,4 +131,4 @@ function getItrConfiguration ({ }) } -module.exports = { getItrConfiguration } +module.exports = { getLibraryConfiguration } diff --git a/packages/dd-trace/src/ci-visibility/telemetry.js b/packages/dd-trace/src/ci-visibility/telemetry.js new file mode 100644 index 00000000000..7b24bc02096 --- /dev/null +++ b/packages/dd-trace/src/ci-visibility/telemetry.js @@ -0,0 +1,157 @@ +const telemetryMetrics = require('../telemetry/metrics') + +const ciVisibilityMetrics = telemetryMetrics.manager.namespace('civisibility') + +const formattedTags = { + testLevel: 'event_type', + testFramework: 'test_framework', + errorType: 'error_type', + exitCode: 'exit_code', + isCodeCoverageEnabled: 'coverage_enabled', + isSuitesSkippingEnabled: 'itrskip_enabled', + hasCodeOwners: 'has_code_owners', + isUnsupportedCIProvider: 'is_unsupported_ci', + isNew: 'is_new', + isRum: 'is_rum', + browserDriver: 'browser_driver' +} + +// Transform tags dictionary to array of strings. +// If tag value is true, then only tag key is added to the array. +function formatMetricTags (tagsDictionary) { + return Object.keys(tagsDictionary).reduce((acc, tagKey) => { + if (tagKey === 'statusCode') { + const statusCode = tagsDictionary[tagKey] + if (isStatusCode400(statusCode)) { + acc.push(`status_code:${statusCode}`) + } + acc.push(`error_type:${getErrorTypeFromStatusCode(statusCode)}`) + return acc + } + const formattedTagKey = formattedTags[tagKey] || tagKey + if (tagsDictionary[tagKey] === true) { + acc.push(formattedTagKey) + } else if (tagsDictionary[tagKey] !== undefined && tagsDictionary[tagKey] !== null) { + acc.push(`${formattedTagKey}:${tagsDictionary[tagKey]}`) + } + return acc + }, []) +} + +function incrementCountMetric (name, tags = {}, value = 1) { + ciVisibilityMetrics.count(name, formatMetricTags(tags)).inc(value) +} + +function distributionMetric (name, tags, measure) { + ciVisibilityMetrics.distribution(name, formatMetricTags(tags)).track(measure) +} + +// CI Visibility telemetry events +const TELEMETRY_TEST_SESSION = 'test_session' +const TELEMETRY_EVENT_CREATED = 'event_created' +const TELEMETRY_EVENT_FINISHED = 'event_finished' +const TELEMETRY_CODE_COVERAGE_STARTED = 'code_coverage_started' +const TELEMETRY_CODE_COVERAGE_FINISHED = 'code_coverage_finished' +const TELEMETRY_ITR_SKIPPED = 'itr_skipped' +const TELEMETRY_ITR_UNSKIPPABLE = 'itr_unskippable' +const TELEMETRY_ITR_FORCED_TO_RUN = 'itr_forced_run' +const TELEMETRY_CODE_COVERAGE_EMPTY = 'code_coverage.is_empty' +const TELEMETRY_CODE_COVERAGE_NUM_FILES = 'code_coverage.files' +const TELEMETRY_EVENTS_ENQUEUED_FOR_SERIALIZATION = 'events_enqueued_for_serialization' +const TELEMETRY_ENDPOINT_PAYLOAD_SERIALIZATION_MS = 'endpoint_payload.events_serialization_ms' +const TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS = 'endpoint_payload.requests' +const TELEMETRY_ENDPOINT_PAYLOAD_BYTES = 'endpoint_payload.bytes' +const TELEMETRY_ENDPOINT_PAYLOAD_EVENTS_COUNT = 'endpoint_payload.events_count' +const TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_MS = 'endpoint_payload.requests_ms' +const TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_ERRORS = 'endpoint_payload.requests_errors' +const TELEMETRY_ENDPOINT_PAYLOAD_DROPPED = 'endpoint_payload.dropped' +const TELEMETRY_GIT_COMMAND = 'git.command' +const TELEMETRY_GIT_COMMAND_MS = 'git.command_ms' +const TELEMETRY_GIT_COMMAND_ERRORS = 'git.command_errors' +const TELEMETRY_GIT_REQUESTS_SEARCH_COMMITS = 'git_requests.search_commits' +const TELEMETRY_GIT_REQUESTS_SEARCH_COMMITS_MS = 'git_requests.search_commits_ms' +const TELEMETRY_GIT_REQUESTS_SEARCH_COMMITS_ERRORS = 'git_requests.search_commits_errors' +const TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES = 'git_requests.objects_pack' +const TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_MS = 'git_requests.objects_pack_ms' +const TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_ERRORS = 'git_requests.objects_pack_errors' +const TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_NUM = 'git_requests.objects_pack_files' +const TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_BYTES = 'git_requests.objects_pack_bytes' +const TELEMETRY_GIT_REQUESTS_SETTINGS = 'git_requests.settings' +const TELEMETRY_GIT_REQUESTS_SETTINGS_MS = 'git_requests.settings_ms' +const TELEMETRY_GIT_REQUESTS_SETTINGS_ERRORS = 'git_requests.settings_errors' +const TELEMETRY_GIT_REQUESTS_SETTINGS_RESPONSE = 'git_requests.settings_response' +const TELEMETRY_ITR_SKIPPABLE_TESTS = 'itr_skippable_tests.request' +const TELEMETRY_ITR_SKIPPABLE_TESTS_MS = 'itr_skippable_tests.request_ms' +const TELEMETRY_ITR_SKIPPABLE_TESTS_ERRORS = 'itr_skippable_tests.request_errors' +const TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_SUITES = 'itr_skippable_tests.response_suites' +const TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_TESTS = 'itr_skippable_tests.response_tests' +const TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_BYTES = 'itr_skippable_tests.response_bytes' +// early flake detection +const TELEMETRY_KNOWN_TESTS = 'early_flake_detection.request' +const TELEMETRY_KNOWN_TESTS_MS = 'early_flake_detection.request_ms' +const TELEMETRY_KNOWN_TESTS_ERRORS = 'early_flake_detection.request_errors' +const TELEMETRY_KNOWN_TESTS_RESPONSE_TESTS = 'early_flake_detection.response_tests' +const TELEMETRY_KNOWN_TESTS_RESPONSE_BYTES = 'early_flake_detection.response_bytes' + +function isStatusCode400 (statusCode) { + return statusCode >= 400 && statusCode < 500 +} + +function getErrorTypeFromStatusCode (statusCode) { + if (statusCode >= 400 && statusCode < 500) { + return 'status_code_4xx_response' + } + if (statusCode >= 500) { + return 'status_code_5xx_response' + } + return 'network' +} + +module.exports = { + incrementCountMetric, + distributionMetric, + TELEMETRY_TEST_SESSION, + TELEMETRY_EVENT_CREATED, + TELEMETRY_EVENT_FINISHED, + TELEMETRY_CODE_COVERAGE_STARTED, + TELEMETRY_CODE_COVERAGE_FINISHED, + TELEMETRY_ITR_SKIPPED, + TELEMETRY_ITR_UNSKIPPABLE, + TELEMETRY_ITR_FORCED_TO_RUN, + TELEMETRY_CODE_COVERAGE_EMPTY, + TELEMETRY_CODE_COVERAGE_NUM_FILES, + TELEMETRY_EVENTS_ENQUEUED_FOR_SERIALIZATION, + TELEMETRY_ENDPOINT_PAYLOAD_SERIALIZATION_MS, + TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS, + TELEMETRY_ENDPOINT_PAYLOAD_BYTES, + TELEMETRY_ENDPOINT_PAYLOAD_EVENTS_COUNT, + TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_MS, + TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_ERRORS, + TELEMETRY_ENDPOINT_PAYLOAD_DROPPED, + TELEMETRY_GIT_COMMAND, + TELEMETRY_GIT_COMMAND_MS, + TELEMETRY_GIT_COMMAND_ERRORS, + TELEMETRY_GIT_REQUESTS_SEARCH_COMMITS, + TELEMETRY_GIT_REQUESTS_SEARCH_COMMITS_MS, + TELEMETRY_GIT_REQUESTS_SEARCH_COMMITS_ERRORS, + TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_NUM, + TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_BYTES, + TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES, + TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_MS, + TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_ERRORS, + TELEMETRY_GIT_REQUESTS_SETTINGS, + TELEMETRY_GIT_REQUESTS_SETTINGS_MS, + TELEMETRY_GIT_REQUESTS_SETTINGS_ERRORS, + TELEMETRY_GIT_REQUESTS_SETTINGS_RESPONSE, + TELEMETRY_ITR_SKIPPABLE_TESTS, + TELEMETRY_ITR_SKIPPABLE_TESTS_MS, + TELEMETRY_ITR_SKIPPABLE_TESTS_ERRORS, + TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_SUITES, + TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_TESTS, + TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_BYTES, + TELEMETRY_KNOWN_TESTS, + TELEMETRY_KNOWN_TESTS_MS, + TELEMETRY_KNOWN_TESTS_ERRORS, + TELEMETRY_KNOWN_TESTS_RESPONSE_TESTS, + TELEMETRY_KNOWN_TESTS_RESPONSE_BYTES +} diff --git a/packages/dd-trace/src/ci-visibility/test-api-manual/test-api-manual-plugin.js b/packages/dd-trace/src/ci-visibility/test-api-manual/test-api-manual-plugin.js index f6a1612b373..8e0b9351b06 100644 --- a/packages/dd-trace/src/ci-visibility/test-api-manual/test-api-manual-plugin.js +++ b/packages/dd-trace/src/ci-visibility/test-api-manual/test-api-manual-plugin.js @@ -10,6 +10,7 @@ class TestApiManualPlugin extends CiPlugin { static get id () { return 'test-api-manual' } + constructor (...args) { super(...args) this.sourceRoot = process.cwd() diff --git a/packages/dd-trace/src/config.js b/packages/dd-trace/src/config.js index 3cc35ecb6aa..fa502ccb5a2 100644 --- a/packages/dd-trace/src/config.js +++ b/packages/dd-trace/src/config.js @@ -2,23 +2,143 @@ const fs = require('fs') const os = require('os') -const uuid = require('crypto-randomuuid') -const URL = require('url').URL +const uuid = require('crypto-randomuuid') // we need to keep the old uuid dep because of cypress +const { URL } = require('url') const log = require('./log') const pkg = require('./pkg') const coalesce = require('koalas') const tagger = require('./tagger') +const get = require('../../datadog-core/src/utils/src/get') +const has = require('../../datadog-core/src/utils/src/has') +const set = require('../../datadog-core/src/utils/src/set') const { isTrue, isFalse } = require('./util') const { GIT_REPOSITORY_URL, GIT_COMMIT_SHA } = require('./plugins/util/tags') const { getGitMetadataFromGitProperties, removeUserSensitiveInfo } = require('./git_properties') const { updateConfig } = require('./telemetry') -const { getIsGCPFunction, getIsAzureFunctionConsumptionPlan } = require('./serverless') +const telemetryMetrics = require('./telemetry/metrics') +const { getIsGCPFunction, getIsAzureFunction } = require('./serverless') +const { ORIGIN_KEY, GRPC_CLIENT_ERROR_STATUSES, GRPC_SERVER_ERROR_STATUSES } = require('./constants') +const { appendRules } = require('./payload-tagging/config') + +const tracerMetrics = telemetryMetrics.manager.namespace('tracers') + +const telemetryCounters = { + 'otel.env.hiding': {}, + 'otel.env.invalid': {} +} + +function getCounter (event, ddVar, otelVar) { + const counters = telemetryCounters[event] + const tags = [] + const ddVarPrefix = 'config_datadog:' + const otelVarPrefix = 'config_opentelemetry:' + if (ddVar) { + ddVar = ddVarPrefix + ddVar.toLowerCase() + tags.push(ddVar) + } + if (otelVar) { + otelVar = otelVarPrefix + otelVar.toLowerCase() + tags.push(otelVar) + } + + if (!(otelVar in counters)) counters[otelVar] = {} + + const counter = tracerMetrics.count(event, tags) + counters[otelVar][ddVar] = counter + return counter +} + +const otelDdEnvMapping = { + OTEL_LOG_LEVEL: 'DD_TRACE_LOG_LEVEL', + OTEL_PROPAGATORS: 'DD_TRACE_PROPAGATION_STYLE', + OTEL_SERVICE_NAME: 'DD_SERVICE', + OTEL_TRACES_SAMPLER: 'DD_TRACE_SAMPLE_RATE', + OTEL_TRACES_SAMPLER_ARG: 'DD_TRACE_SAMPLE_RATE', + OTEL_TRACES_EXPORTER: 'DD_TRACE_ENABLED', + OTEL_METRICS_EXPORTER: 'DD_RUNTIME_METRICS_ENABLED', + OTEL_RESOURCE_ATTRIBUTES: 'DD_TAGS', + OTEL_SDK_DISABLED: 'DD_TRACE_OTEL_ENABLED', + OTEL_LOGS_EXPORTER: undefined +} + +const VALID_PROPAGATION_STYLES = new Set(['datadog', 'tracecontext', 'b3', 'b3 single header', 'none']) + +const VALID_LOG_LEVELS = new Set(['debug', 'info', 'warn', 'error']) + +function getFromOtelSamplerMap (otelTracesSampler, otelTracesSamplerArg) { + const OTEL_TRACES_SAMPLER_MAPPING = { + always_on: '1.0', + always_off: '0.0', + traceidratio: otelTracesSamplerArg, + parentbased_always_on: '1.0', + parentbased_always_off: '0.0', + parentbased_traceidratio: otelTracesSamplerArg + } + return OTEL_TRACES_SAMPLER_MAPPING[otelTracesSampler] +} + +function validateOtelPropagators (propagators) { + if (!process.env.PROPAGATION_STYLE_EXTRACT && + !process.env.PROPAGATION_STYLE_INJECT && + !process.env.DD_TRACE_PROPAGATION_STYLE && + process.env.OTEL_PROPAGATORS) { + for (const style in propagators) { + if (!VALID_PROPAGATION_STYLES.has(style)) { + log.warn('unexpected value for OTEL_PROPAGATORS environment variable') + getCounter('otel.env.invalid', 'DD_TRACE_PROPAGATION_STYLE', 'OTEL_PROPAGATORS').inc() + } + } + } +} + +function validateEnvVarType (envVar) { + const value = process.env[envVar] + switch (envVar) { + case 'OTEL_LOG_LEVEL': + return VALID_LOG_LEVELS.has(value) + case 'OTEL_PROPAGATORS': + case 'OTEL_RESOURCE_ATTRIBUTES': + case 'OTEL_SERVICE_NAME': + return typeof value === 'string' + case 'OTEL_TRACES_SAMPLER': + return getFromOtelSamplerMap(value, process.env.OTEL_TRACES_SAMPLER_ARG) !== undefined + case 'OTEL_TRACES_SAMPLER_ARG': + return !isNaN(parseFloat(value)) + case 'OTEL_SDK_DISABLED': + return value.toLowerCase() === 'true' || value.toLowerCase() === 'false' + case 'OTEL_TRACES_EXPORTER': + case 'OTEL_METRICS_EXPORTER': + case 'OTEL_LOGS_EXPORTER': + return value.toLowerCase() === 'none' + default: + return false + } +} + +function checkIfBothOtelAndDdEnvVarSet () { + for (const [otelEnvVar, ddEnvVar] of Object.entries(otelDdEnvMapping)) { + if (ddEnvVar && process.env[ddEnvVar] && process.env[otelEnvVar]) { + log.warn(`both ${ddEnvVar} and ${otelEnvVar} environment variables are set`) + getCounter('otel.env.hiding', ddEnvVar, otelEnvVar).inc() + } + + if (process.env[otelEnvVar] && !validateEnvVarType(otelEnvVar)) { + log.warn(`unexpected value for ${otelEnvVar} environment variable`) + getCounter('otel.env.invalid', ddEnvVar, otelEnvVar).inc() + } + } +} const fromEntries = Object.fromEntries || (entries => entries.reduce((obj, [k, v]) => Object.assign(obj, { [k]: v }), {})) // eslint-disable-next-line max-len const qsRegex = '(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?|access_?|secret_?)key(?:_?id)?|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)(?:(?:\\s|%20)*(?:=|%3D)[^&]+|(?:"|%22)(?:\\s|%20)*(?::|%3A)(?:\\s|%20)*(?:"|%22)(?:%2[^2]|%[^2]|[^"%])+(?:"|%22))|bearer(?:\\s|%20)+[a-z0-9\\._\\-]+|token(?::|%3A)[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L](?:[\\w=-]|%3D)+\\.ey[I-L](?:[\\w=-]|%3D)+(?:\\.(?:[\\w.+\\/=-]|%3D|%2F|%2B)+)?|[\\-]{5}BEGIN(?:[a-z\\s]|%20)+PRIVATE(?:\\s|%20)KEY[\\-]{5}[^\\-]+[\\-]{5}END(?:[a-z\\s]|%20)+PRIVATE(?:\\s|%20)KEY|ssh-rsa(?:\\s|%20)*(?:[a-z0-9\\/\\.+]|%2F|%5C|%2B){100,}' +// eslint-disable-next-line max-len +const defaultWafObfuscatorKeyRegex = '(?i)pass|pw(?:or)?d|secret|(?:api|private|public|access)[_-]?key|token|consumer[_-]?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization|jsessionid|phpsessid|asp\\.net[_-]sessionid|sid|jwt' +// eslint-disable-next-line max-len +const defaultWafObfuscatorValueRegex = '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:[_-]?phrase)?|secret(?:[_-]?key)?|(?:(?:api|private|public|access)[_-]?)key(?:[_-]?id)?|(?:(?:auth|access|id|refresh)[_-]?)?token|consumer[_-]?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?|jsessionid|phpsessid|asp\\.net(?:[_-]|-)sessionid|sid|jwt)(?:\\s*=[^;]|"\\s*:\\s*"[^"]+")|bearer\\s+[a-z0-9\\._\\-]+|token:[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L][\\w=-]+\\.ey[I-L][\\w=-]+(?:\\.[\\w.+\\/=-]+)?|[\\-]{5}BEGIN[a-z\\s]+PRIVATE\\sKEY[\\-]{5}[^\\-]+[\\-]{5}END[a-z\\s]+PRIVATE\\sKEY|ssh-rsa\\s*[a-z0-9\\/\\.+]{100,}' +const runtimeId = uuid() function maybeFile (filepath) { if (!filepath) return @@ -54,6 +174,21 @@ function validateNamingVersion (versionString) { return versionString } +/** + * Given a string of comma-separated paths, return the array of paths. + * If a blank path is provided a null is returned to signal that the feature is disabled. + * An empty array means the feature is enabled but that no rules need to be applied. + * + * @param {string} input + * @returns {[string]|null} + */ +function splitJSONPathRules (input) { + if (!input) return null + if (Array.isArray(input)) return input + if (input === 'all') return [] + return input.split(',') +} + // Shallow clone with property name remapping function remapify (input, mappings) { if (!input) return @@ -64,9 +199,9 @@ function remapify (input, mappings) { return output } -function propagationStyle (key, option, defaultValue) { +function propagationStyle (key, option) { // Extract by key if in object-form value - if (typeof option === 'object' && !Array.isArray(option)) { + if (option !== null && typeof option === 'object' && !Array.isArray(option)) { option = option[key] } @@ -74,221 +209,55 @@ function propagationStyle (key, option, defaultValue) { if (Array.isArray(option)) return option.map(v => v.toLowerCase()) // If it's not an array but not undefined there's something wrong with the input - if (typeof option !== 'undefined') { + if (option !== undefined) { log.warn('Unexpected input for config.tracePropagationStyle') } // Otherwise, fallback to env var parsing const envKey = `DD_TRACE_PROPAGATION_STYLE_${key.toUpperCase()}` - const envVar = coalesce(process.env[envKey], process.env.DD_TRACE_PROPAGATION_STYLE) - if (typeof envVar !== 'undefined') { + + const envVar = coalesce(process.env[envKey], process.env.DD_TRACE_PROPAGATION_STYLE, process.env.OTEL_PROPAGATORS) + if (envVar !== undefined) { return envVar.split(',') .filter(v => v !== '') .map(v => v.trim().toLowerCase()) } +} - return defaultValue +function reformatSpanSamplingRules (rules) { + if (!rules) return rules + return rules.map(rule => { + return remapify(rule, { + sample_rate: 'sampleRate', + max_per_second: 'maxPerSecond' + }) + }) } class Config { - constructor (options) { - options = options || {} + constructor (options = {}) { + options = { + ...options, + appsec: options.appsec != null ? options.appsec : options.experimental?.appsec, + iast: options.iast != null ? options.iast : options.experimental?.iast + } // Configure the logger first so it can be used to warn about other configs - this.debug = isTrue(coalesce( - process.env.DD_TRACE_DEBUG, - false - )) - this.logger = options.logger - this.logLevel = coalesce( - options.logLevel, - process.env.DD_TRACE_LOG_LEVEL, - 'debug' - ) + const logConfig = log.getConfig() + this.debug = logConfig.enabled + this.logger = coalesce(options.logger, logConfig.logger) + this.logLevel = coalesce(options.logLevel, logConfig.logLevel) log.use(this.logger) - log.toggle(this.debug, this.logLevel, this) + log.toggle(this.debug, this.logLevel) - this.tags = {} - - tagger.add(this.tags, process.env.DD_TAGS) - tagger.add(this.tags, process.env.DD_TRACE_TAGS) - tagger.add(this.tags, process.env.DD_TRACE_GLOBAL_TAGS) - tagger.add(this.tags, options.tags) - - const DD_TRACING_ENABLED = coalesce( - process.env.DD_TRACING_ENABLED, - true - ) - const DD_PROFILING_ENABLED = coalesce( - options.profiling, // TODO: remove when enabled by default - process.env.DD_EXPERIMENTAL_PROFILING_ENABLED, - process.env.DD_PROFILING_ENABLED, - false - ) - const DD_PROFILING_EXPORTERS = coalesce( - process.env.DD_PROFILING_EXPORTERS, - 'agent' - ) - const DD_PROFILING_SOURCE_MAP = process.env.DD_PROFILING_SOURCE_MAP - const DD_RUNTIME_METRICS_ENABLED = coalesce( - options.runtimeMetrics, // TODO: remove when enabled by default - process.env.DD_RUNTIME_METRICS_ENABLED, - false - ) - const DD_DBM_PROPAGATION_MODE = coalesce( - options.dbmPropagationMode, - process.env.DD_DBM_PROPAGATION_MODE, - 'disabled' - ) - const DD_DATA_STREAMS_ENABLED = coalesce( - options.dsmEnabled, - process.env.DD_DATA_STREAMS_ENABLED, - false - ) - const DD_AGENT_HOST = coalesce( - options.hostname, - process.env.DD_AGENT_HOST, - process.env.DD_TRACE_AGENT_HOSTNAME, - '127.0.0.1' - ) - const DD_TRACE_AGENT_PORT = coalesce( - options.port, - process.env.DD_TRACE_AGENT_PORT, - '8126' - ) - const DD_TRACE_AGENT_URL = coalesce( - options.url, - process.env.DD_TRACE_AGENT_URL, - process.env.DD_TRACE_URL, - null - ) - const DD_IS_CIVISIBILITY = coalesce( - options.isCiVisibility, - false - ) - const DD_CIVISIBILITY_AGENTLESS_URL = process.env.DD_CIVISIBILITY_AGENTLESS_URL - - const DD_CIVISIBILITY_ITR_ENABLED = coalesce( - process.env.DD_CIVISIBILITY_ITR_ENABLED, - true - ) - - const DD_CIVISIBILITY_MANUAL_API_ENABLED = coalesce( - process.env.DD_CIVISIBILITY_MANUAL_API_ENABLED, - false - ) - - const DD_TRACE_MEMCACHED_COMMAND_ENABLED = coalesce( - process.env.DD_TRACE_MEMCACHED_COMMAND_ENABLED, - false - ) - - const DD_SERVICE = options.service || - process.env.DD_SERVICE || - process.env.DD_SERVICE_NAME || - this.tags.service || - process.env.AWS_LAMBDA_FUNCTION_NAME || - process.env.FUNCTION_NAME || // Google Cloud Function Name set by deprecated runtimes - process.env.K_SERVICE || // Google Cloud Function Name set by newer runtimes - process.env.WEBSITE_SITE_NAME || // set by Azure Functions - pkg.name || - 'node' - const DD_SERVICE_MAPPING = coalesce( - options.serviceMapping, - process.env.DD_SERVICE_MAPPING ? fromEntries( - process.env.DD_SERVICE_MAPPING.split(',').map(x => x.trim().split(':')) - ) : {} - ) - const DD_ENV = coalesce( - options.env, - process.env.DD_ENV, - this.tags.env - ) - const DD_VERSION = coalesce( - options.version, - process.env.DD_VERSION, - this.tags.version, - pkg.version - ) - const DD_TRACE_STARTUP_LOGS = coalesce( - options.startupLogs, - process.env.DD_TRACE_STARTUP_LOGS, - false - ) - - const DD_OPENAI_LOGS_ENABLED = coalesce( - options.openAiLogsEnabled, - process.env.DD_OPENAI_LOGS_ENABLED, - false - ) + checkIfBothOtelAndDdEnvVarSet() const DD_API_KEY = coalesce( process.env.DATADOG_API_KEY, process.env.DD_API_KEY ) - const inAWSLambda = process.env.AWS_LAMBDA_FUNCTION_NAME !== undefined - - const isGCPFunction = getIsGCPFunction() - const isAzureFunctionConsumptionPlan = getIsAzureFunctionConsumptionPlan() - - const inServerlessEnvironment = inAWSLambda || isGCPFunction || isAzureFunctionConsumptionPlan - - const DD_INSTRUMENTATION_TELEMETRY_ENABLED = coalesce( - process.env.DD_TRACE_TELEMETRY_ENABLED, // for backward compatibility - process.env.DD_INSTRUMENTATION_TELEMETRY_ENABLED, // to comply with instrumentation telemetry specs - !inServerlessEnvironment - ) - const DD_TELEMETRY_HEARTBEAT_INTERVAL = process.env.DD_TELEMETRY_HEARTBEAT_INTERVAL - ? Math.floor(parseFloat(process.env.DD_TELEMETRY_HEARTBEAT_INTERVAL) * 1000) - : 60000 - const DD_OPENAI_SPAN_CHAR_LIMIT = process.env.DD_OPENAI_SPAN_CHAR_LIMIT - ? parseInt(process.env.DD_OPENAI_SPAN_CHAR_LIMIT) - : 128 - const DD_TELEMETRY_DEBUG = coalesce( - process.env.DD_TELEMETRY_DEBUG, - false - ) - const DD_TELEMETRY_METRICS_ENABLED = coalesce( - process.env.DD_TELEMETRY_METRICS_ENABLED, - true - ) - const DD_TRACE_AGENT_PROTOCOL_VERSION = coalesce( - options.protocolVersion, - process.env.DD_TRACE_AGENT_PROTOCOL_VERSION, - '0.4' - ) - const DD_TRACE_PARTIAL_FLUSH_MIN_SPANS = coalesce( - parseInt(options.flushMinSpans), - parseInt(process.env.DD_TRACE_PARTIAL_FLUSH_MIN_SPANS), - 1000 - ) - const DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP = coalesce( - process.env.DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP, - qsRegex - ) - const DD_TRACE_CLIENT_IP_ENABLED = coalesce( - options.clientIpEnabled, - process.env.DD_TRACE_CLIENT_IP_ENABLED && isTrue(process.env.DD_TRACE_CLIENT_IP_ENABLED), - false - ) - const DD_TRACE_CLIENT_IP_HEADER = coalesce( - options.clientIpHeader, - process.env.DD_TRACE_CLIENT_IP_HEADER, - null - ) - // TODO: Remove the experimental env vars as a major? - const DD_TRACE_B3_ENABLED = coalesce( - options.experimental && options.experimental.b3, - process.env.DD_TRACE_EXPERIMENTAL_B3_ENABLED, - false - ) - const defaultPropagationStyle = ['datadog', 'tracecontext'] - if (isTrue(DD_TRACE_B3_ENABLED)) { - defaultPropagationStyle.push('b3') - defaultPropagationStyle.push('b3 single header') - } if (process.env.DD_TRACE_PROPAGATION_STYLE && ( process.env.DD_TRACE_PROPAGATION_STYLE_INJECT || process.env.DD_TRACE_PROPAGATION_STYLE_EXTRACT @@ -299,379 +268,93 @@ class Config { 'environment variables' ) } - const DD_TRACE_PROPAGATION_STYLE_INJECT = propagationStyle( + const PROPAGATION_STYLE_INJECT = propagationStyle( 'inject', options.tracePropagationStyle, - defaultPropagationStyle - ) - const DD_TRACE_PROPAGATION_STYLE_EXTRACT = propagationStyle( - 'extract', - options.tracePropagationStyle, - defaultPropagationStyle - ) - const DD_TRACE_PROPAGATION_EXTRACT_FIRST = coalesce( - process.env.DD_TRACE_PROPAGATION_EXTRACT_FIRST, - false - ) - const DD_TRACE_RUNTIME_ID_ENABLED = coalesce( - options.experimental && options.experimental.runtimeId, - process.env.DD_TRACE_EXPERIMENTAL_RUNTIME_ID_ENABLED, - false - ) - const DD_TRACE_EXPORTER = coalesce( - options.experimental && options.experimental.exporter, - process.env.DD_TRACE_EXPERIMENTAL_EXPORTER - ) - const DD_TRACE_GET_RUM_DATA_ENABLED = coalesce( - options.experimental && options.experimental.enableGetRumData, - process.env.DD_TRACE_EXPERIMENTAL_GET_RUM_DATA_ENABLED, - false - ) - const DD_TRACE_SPAN_ATTRIBUTE_SCHEMA = validateNamingVersion( - coalesce( - options.spanAttributeSchema, - process.env.DD_TRACE_SPAN_ATTRIBUTE_SCHEMA - ) - ) - const DD_TRACE_PEER_SERVICE_MAPPING = coalesce( - options.peerServiceMapping, - process.env.DD_TRACE_PEER_SERVICE_MAPPING ? fromEntries( - process.env.DD_TRACE_PEER_SERVICE_MAPPING.split(',').map(x => x.trim().split(':')) - ) : {} - ) - - const peerServiceSet = ( - options.hasOwnProperty('spanComputePeerService') || - process.env.hasOwnProperty('DD_TRACE_PEER_SERVICE_DEFAULTS_ENABLED') - ) - const peerServiceValue = coalesce( - options.spanComputePeerService, - process.env.DD_TRACE_PEER_SERVICE_DEFAULTS_ENABLED - ) - - const DD_TRACE_PEER_SERVICE_DEFAULTS_ENABLED = ( - DD_TRACE_SPAN_ATTRIBUTE_SCHEMA === 'v0' - // In v0, peer service is computed only if it is explicitly set to true - ? peerServiceSet && isTrue(peerServiceValue) - // In >v0, peer service is false only if it is explicitly set to false - : (peerServiceSet ? !isFalse(peerServiceValue) : true) + this._getDefaultPropagationStyle(options) ) - const DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED = coalesce( - options.spanRemoveIntegrationFromService, - isTrue(process.env.DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED) - ) - const DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH = coalesce( - process.env.DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH, - '512' - ) - - const DD_TRACE_STATS_COMPUTATION_ENABLED = coalesce( - options.stats, - process.env.DD_TRACE_STATS_COMPUTATION_ENABLED, - isGCPFunction || isAzureFunctionConsumptionPlan - ) - - // the tracer generates 128 bit IDs by default as of v5 - const DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED = coalesce( - options.traceId128BitGenerationEnabled, - process.env.DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED, - true - ) + validateOtelPropagators(PROPAGATION_STYLE_INJECT) - const DD_TRACE_128_BIT_TRACEID_LOGGING_ENABLED = coalesce( - options.traceId128BitLoggingEnabled, - process.env.DD_TRACE_128_BIT_TRACEID_LOGGING_ENABLED, - false - ) - - let appsec = options.appsec != null ? options.appsec : options.experimental && options.experimental.appsec - - if (typeof appsec === 'boolean') { - appsec = { - enabled: appsec + if (typeof options.appsec === 'boolean') { + options.appsec = { + enabled: options.appsec } - } else if (appsec == null) { - appsec = {} + } else if (options.appsec == null) { + options.appsec = {} } - const DD_APPSEC_ENABLED = coalesce( - appsec.enabled, - process.env.DD_APPSEC_ENABLED && isTrue(process.env.DD_APPSEC_ENABLED) - ) - const DD_APPSEC_RULES = coalesce( - appsec.rules, - process.env.DD_APPSEC_RULES - ) - const DD_APPSEC_TRACE_RATE_LIMIT = coalesce( - parseInt(appsec.rateLimit), - parseInt(process.env.DD_APPSEC_TRACE_RATE_LIMIT), - 100 - ) - const DD_APPSEC_WAF_TIMEOUT = coalesce( - parseInt(appsec.wafTimeout), - parseInt(process.env.DD_APPSEC_WAF_TIMEOUT), - 5e3 // µs - ) - const DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP = coalesce( - appsec.obfuscatorKeyRegex, - process.env.DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP, - `(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?)key)|token|consumer_?(?:id|key|se\ -cret)|sign(?:ed|ature)|bearer|authorization` - ) - const DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP = coalesce( - appsec.obfuscatorValueRegex, - process.env.DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP, - `(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?|access_?|secret_?)key(?:_?id)?|to\ -ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)(?:\\s*=[^;]|"\\s*:\\s*"[^"]+")|bearer\ -\\s+[a-z0-9\\._\\-]+|token:[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L][\\w=-]+\\.ey[I-L][\\w=-]+(?:\\.[\\w.+\\/=-]+)?\ -|[\\-]{5}BEGIN[a-z\\s]+PRIVATE\\sKEY[\\-]{5}[^\\-]+[\\-]{5}END[a-z\\s]+PRIVATE\\sKEY|ssh-rsa\\s*[a-z0-9\\/\\.+]{100,}` - ) - const DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML = coalesce( - maybeFile(appsec.blockedTemplateHtml), - maybeFile(process.env.DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML) - ) - const DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON = coalesce( - maybeFile(appsec.blockedTemplateJson), - maybeFile(process.env.DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON) - ) - const DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING = coalesce( - appsec.eventTracking && appsec.eventTracking.mode, - process.env.DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING, - 'safe' - ).toLowerCase() - const DD_EXPERIMENTAL_API_SECURITY_ENABLED = coalesce( - appsec?.apiSecurity?.enabled, - isTrue(process.env.DD_EXPERIMENTAL_API_SECURITY_ENABLED), - false - ) - const DD_API_SECURITY_REQUEST_SAMPLE_RATE = coalesce( - appsec?.apiSecurity?.requestSampling, - parseFloat(process.env.DD_API_SECURITY_REQUEST_SAMPLE_RATE), - 0.1 - ) - - const remoteConfigOptions = options.remoteConfig || {} - const DD_REMOTE_CONFIGURATION_ENABLED = coalesce( - process.env.DD_REMOTE_CONFIGURATION_ENABLED && isTrue(process.env.DD_REMOTE_CONFIGURATION_ENABLED), - !inServerlessEnvironment - ) - const DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS = coalesce( - parseFloat(remoteConfigOptions.pollInterval), - parseFloat(process.env.DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS), - 5 // seconds - ) - - const iastOptions = options?.experimental?.iast - const DD_IAST_ENABLED = coalesce( - iastOptions && - (iastOptions === true || iastOptions.enabled === true), - process.env.DD_IAST_ENABLED, - false - ) - const DD_TELEMETRY_LOG_COLLECTION_ENABLED = coalesce( - process.env.DD_TELEMETRY_LOG_COLLECTION_ENABLED, - DD_IAST_ENABLED - ) - - const defaultIastRequestSampling = 30 - const iastRequestSampling = coalesce( - parseInt(iastOptions?.requestSampling), - parseInt(process.env.DD_IAST_REQUEST_SAMPLING), - defaultIastRequestSampling - ) - const DD_IAST_REQUEST_SAMPLING = iastRequestSampling < 0 || - iastRequestSampling > 100 ? defaultIastRequestSampling : iastRequestSampling - - const DD_IAST_MAX_CONCURRENT_REQUESTS = coalesce( - parseInt(iastOptions?.maxConcurrentRequests), - parseInt(process.env.DD_IAST_MAX_CONCURRENT_REQUESTS), - 2 - ) - - const DD_IAST_MAX_CONTEXT_OPERATIONS = coalesce( - parseInt(iastOptions?.maxContextOperations), - parseInt(process.env.DD_IAST_MAX_CONTEXT_OPERATIONS), - 2 - ) - - const DD_IAST_DEDUPLICATION_ENABLED = coalesce( - iastOptions?.deduplicationEnabled, - process.env.DD_IAST_DEDUPLICATION_ENABLED && isTrue(process.env.DD_IAST_DEDUPLICATION_ENABLED), - true - ) - - const DD_IAST_REDACTION_ENABLED = coalesce( - iastOptions?.redactionEnabled, - !isFalse(process.env.DD_IAST_REDACTION_ENABLED), - true + const DD_INSTRUMENTATION_INSTALL_ID = coalesce( + process.env.DD_INSTRUMENTATION_INSTALL_ID, + null ) - - const DD_IAST_REDACTION_NAME_PATTERN = coalesce( - iastOptions?.redactionNamePattern, - process.env.DD_IAST_REDACTION_NAME_PATTERN, + const DD_INSTRUMENTATION_INSTALL_TIME = coalesce( + process.env.DD_INSTRUMENTATION_INSTALL_TIME, null ) - - const DD_IAST_REDACTION_VALUE_PATTERN = coalesce( - iastOptions?.redactionValuePattern, - process.env.DD_IAST_REDACTION_VALUE_PATTERN, + const DD_INSTRUMENTATION_INSTALL_TYPE = coalesce( + process.env.DD_INSTRUMENTATION_INSTALL_TYPE, null ) - const DD_IAST_TELEMETRY_VERBOSITY = coalesce( - iastOptions?.telemetryVerbosity, - process.env.DD_IAST_TELEMETRY_VERBOSITY, - 'INFORMATION' - ) + const DD_TRACE_CLOUD_REQUEST_PAYLOAD_TAGGING = splitJSONPathRules( + coalesce( + process.env.DD_TRACE_CLOUD_REQUEST_PAYLOAD_TAGGING, + options.cloudPayloadTagging?.request, + '' + )) - const DD_CIVISIBILITY_GIT_UPLOAD_ENABLED = coalesce( - process.env.DD_CIVISIBILITY_GIT_UPLOAD_ENABLED, - true - ) + const DD_TRACE_CLOUD_RESPONSE_PAYLOAD_TAGGING = splitJSONPathRules( + coalesce( + process.env.DD_TRACE_CLOUD_RESPONSE_PAYLOAD_TAGGING, + options.cloudPayloadTagging?.response, + '' + )) - const DD_TRACE_GIT_METADATA_ENABLED = coalesce( - process.env.DD_TRACE_GIT_METADATA_ENABLED, - true + const DD_TRACE_CLOUD_PAYLOAD_TAGGING_MAX_DEPTH = coalesce( + process.env.DD_TRACE_CLOUD_PAYLOAD_TAGGING_MAX_DEPTH, + options.cloudPayloadTagging?.maxDepth, + 10 ) - const ingestion = options.ingestion || {} - const dogstatsd = coalesce(options.dogstatsd, {}) - const sampler = { - rateLimit: coalesce(options.rateLimit, process.env.DD_TRACE_RATE_LIMIT, ingestion.rateLimit), - rules: coalesce( - options.samplingRules, - safeJsonParse(process.env.DD_TRACE_SAMPLING_RULES), - [] - ).map(rule => { - return remapify(rule, { - sample_rate: 'sampleRate' - }) - }), - spanSamplingRules: coalesce( - options.spanSamplingRules, - safeJsonParse(maybeFile(process.env.DD_SPAN_SAMPLING_RULES_FILE)), - safeJsonParse(process.env.DD_SPAN_SAMPLING_RULES), - [] - ).map(rule => { - return remapify(rule, { - sample_rate: 'sampleRate', - max_per_second: 'maxPerSecond' - }) - }) - } - - const defaultFlushInterval = inAWSLambda ? 0 : 2000 - - this.tracing = !isFalse(DD_TRACING_ENABLED) - this.dbmPropagationMode = DD_DBM_PROPAGATION_MODE - this.dsmEnabled = isTrue(DD_DATA_STREAMS_ENABLED) - this.openAiLogsEnabled = DD_OPENAI_LOGS_ENABLED + // TODO: refactor this.apiKey = DD_API_KEY - this.env = DD_ENV - this.url = DD_CIVISIBILITY_AGENTLESS_URL ? new URL(DD_CIVISIBILITY_AGENTLESS_URL) - : getAgentUrl(DD_TRACE_AGENT_URL, options) - this.site = coalesce(options.site, process.env.DD_SITE, 'datadoghq.com') - this.hostname = DD_AGENT_HOST || (this.url && this.url.hostname) - this.port = String(DD_TRACE_AGENT_PORT || (this.url && this.url.port)) - this.flushInterval = coalesce(parseInt(options.flushInterval, 10), defaultFlushInterval) - this.flushMinSpans = DD_TRACE_PARTIAL_FLUSH_MIN_SPANS - this.queryStringObfuscation = DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP - this.clientIpEnabled = DD_TRACE_CLIENT_IP_ENABLED - this.clientIpHeader = DD_TRACE_CLIENT_IP_HEADER - this.plugins = !!coalesce(options.plugins, true) - this.service = DD_SERVICE - this.serviceMapping = DD_SERVICE_MAPPING - this.version = DD_VERSION - this.dogstatsd = { - hostname: coalesce(dogstatsd.hostname, process.env.DD_DOGSTATSD_HOSTNAME, this.hostname), - port: String(coalesce(dogstatsd.port, process.env.DD_DOGSTATSD_PORT, 8125)) - } - this.runtimeMetrics = isTrue(DD_RUNTIME_METRICS_ENABLED) - this.tracePropagationStyle = { - inject: DD_TRACE_PROPAGATION_STYLE_INJECT, - extract: DD_TRACE_PROPAGATION_STYLE_EXTRACT - } - this.tracePropagationExtractFirst = isTrue(DD_TRACE_PROPAGATION_EXTRACT_FIRST) - this.experimental = { - runtimeId: isTrue(DD_TRACE_RUNTIME_ID_ENABLED), - exporter: DD_TRACE_EXPORTER, - enableGetRumData: isTrue(DD_TRACE_GET_RUM_DATA_ENABLED) - } - this.sampler = sampler - this.reportHostname = isTrue(coalesce(options.reportHostname, process.env.DD_TRACE_REPORT_HOSTNAME, false)) - this.scope = process.env.DD_TRACE_SCOPE - this.profiling = { - enabled: isTrue(DD_PROFILING_ENABLED), - sourceMap: !isFalse(DD_PROFILING_SOURCE_MAP), - exporters: DD_PROFILING_EXPORTERS - } - this.spanAttributeSchema = DD_TRACE_SPAN_ATTRIBUTE_SCHEMA - this.spanComputePeerService = DD_TRACE_PEER_SERVICE_DEFAULTS_ENABLED - this.spanRemoveIntegrationFromService = DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED - this.peerServiceMapping = DD_TRACE_PEER_SERVICE_MAPPING - this.lookup = options.lookup - this.startupLogs = isTrue(DD_TRACE_STARTUP_LOGS) - // Disabled for CI Visibility's agentless - this.telemetry = { - enabled: DD_TRACE_EXPORTER !== 'datadog' && isTrue(DD_INSTRUMENTATION_TELEMETRY_ENABLED), - heartbeatInterval: DD_TELEMETRY_HEARTBEAT_INTERVAL, - debug: isTrue(DD_TELEMETRY_DEBUG), - logCollection: isTrue(DD_TELEMETRY_LOG_COLLECTION_ENABLED), - metrics: isTrue(DD_TELEMETRY_METRICS_ENABLED) - } - this.protocolVersion = DD_TRACE_AGENT_PROTOCOL_VERSION - this.tagsHeaderMaxLength = parseInt(DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH) - this.appsec = { - enabled: DD_APPSEC_ENABLED, - rules: DD_APPSEC_RULES, - customRulesProvided: !!DD_APPSEC_RULES, - rateLimit: DD_APPSEC_TRACE_RATE_LIMIT, - wafTimeout: DD_APPSEC_WAF_TIMEOUT, - obfuscatorKeyRegex: DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP, - obfuscatorValueRegex: DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP, - blockedTemplateHtml: DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML, - blockedTemplateJson: DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON, - eventTracking: { - enabled: ['extended', 'safe'].includes(DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING), - mode: DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING - }, - apiSecurity: { - enabled: DD_EXPERIMENTAL_API_SECURITY_ENABLED, - // Coerce value between 0 and 1 - requestSampling: Math.min(1, Math.max(0, DD_API_SECURITY_REQUEST_SAMPLE_RATE)) - } - } - this.remoteConfig = { - enabled: DD_REMOTE_CONFIGURATION_ENABLED, - pollInterval: DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS + // sent in telemetry event app-started + this.installSignature = { + id: DD_INSTRUMENTATION_INSTALL_ID, + time: DD_INSTRUMENTATION_INSTALL_TIME, + type: DD_INSTRUMENTATION_INSTALL_TYPE } - this.iast = { - enabled: isTrue(DD_IAST_ENABLED), - requestSampling: DD_IAST_REQUEST_SAMPLING, - maxConcurrentRequests: DD_IAST_MAX_CONCURRENT_REQUESTS, - maxContextOperations: DD_IAST_MAX_CONTEXT_OPERATIONS, - deduplicationEnabled: DD_IAST_DEDUPLICATION_ENABLED, - redactionEnabled: DD_IAST_REDACTION_ENABLED, - redactionNamePattern: DD_IAST_REDACTION_NAME_PATTERN, - redactionValuePattern: DD_IAST_REDACTION_VALUE_PATTERN, - telemetryVerbosity: DD_IAST_TELEMETRY_VERBOSITY - } - - this.isCiVisibility = isTrue(DD_IS_CIVISIBILITY) - this.isIntelligentTestRunnerEnabled = this.isCiVisibility && isTrue(DD_CIVISIBILITY_ITR_ENABLED) - this.isGitUploadEnabled = this.isCiVisibility && - (this.isIntelligentTestRunnerEnabled && !isFalse(DD_CIVISIBILITY_GIT_UPLOAD_ENABLED)) + this.cloudPayloadTagging = { + requestsEnabled: !!DD_TRACE_CLOUD_REQUEST_PAYLOAD_TAGGING, + responsesEnabled: !!DD_TRACE_CLOUD_RESPONSE_PAYLOAD_TAGGING, + maxDepth: DD_TRACE_CLOUD_PAYLOAD_TAGGING_MAX_DEPTH, + rules: appendRules( + DD_TRACE_CLOUD_REQUEST_PAYLOAD_TAGGING, DD_TRACE_CLOUD_RESPONSE_PAYLOAD_TAGGING + ) + } - this.gitMetadataEnabled = isTrue(DD_TRACE_GIT_METADATA_ENABLED) - this.isManualApiEnabled = this.isCiVisibility && isTrue(DD_CIVISIBILITY_MANUAL_API_ENABLED) + this._applyDefaults() + this._applyEnvironment() + this._applyOptions(options) + this._applyCalculated() + this._applyRemote({}) + this._merge() - this.openaiSpanCharLimit = DD_OPENAI_SPAN_CHAR_LIMIT + tagger.add(this.tags, { + service: this.service, + env: this.env, + version: this.version, + 'runtime-id': runtimeId + }) - // Requires an accompanying DD_APM_OBFUSCATION_MEMCACHED_KEEP_COMMAND=true in the agent - this.memcachedCommandEnabled = isTrue(DD_TRACE_MEMCACHED_COMMAND_ENABLED) + if (this.isCiVisibility) { + tagger.add(this.tags, { + [ORIGIN_KEY]: 'ciapp-test' + }) + } if (this.gitMetadataEnabled) { this.repositoryUrl = removeUserSensitiveInfo( @@ -705,29 +388,6 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?) } } } - - this.stats = { - enabled: isTrue(DD_TRACE_STATS_COMPUTATION_ENABLED) - } - - this.traceId128BitGenerationEnabled = isTrue(DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED) - this.traceId128BitLoggingEnabled = isTrue(DD_TRACE_128_BIT_TRACEID_LOGGING_ENABLED) - - this.isGCPFunction = isGCPFunction - this.isAzureFunctionConsumptionPlan = isAzureFunctionConsumptionPlan - - tagger.add(this.tags, { - service: this.service, - env: this.env, - version: this.version, - 'runtime-id': uuid() - }) - - this._applyDefaults() - this._applyEnvironment() - this._applyOptions(options) - this._applyRemote({}) - this._merge() } // Supports only a subset of options for now. @@ -738,52 +398,741 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?) this._applyOptions(options) } + // TODO: test + this._applyCalculated() this._merge() } + _getDefaultPropagationStyle (options) { + // TODO: Remove the experimental env vars as a major? + const DD_TRACE_B3_ENABLED = coalesce( + options.experimental && options.experimental.b3, + process.env.DD_TRACE_EXPERIMENTAL_B3_ENABLED, + false + ) + const defaultPropagationStyle = ['datadog', 'tracecontext'] + if (isTrue(DD_TRACE_B3_ENABLED)) { + defaultPropagationStyle.push('b3') + defaultPropagationStyle.push('b3 single header') + } + return defaultPropagationStyle + } + + _isInServerlessEnvironment () { + const inAWSLambda = process.env.AWS_LAMBDA_FUNCTION_NAME !== undefined + const isGCPFunction = getIsGCPFunction() + const isAzureFunction = getIsAzureFunction() + return inAWSLambda || isGCPFunction || isAzureFunction + } + + // for _merge to work, every config value must have a default value _applyDefaults () { - const defaults = this._defaults = {} + const { + AWS_LAMBDA_FUNCTION_NAME, + FUNCTION_NAME, + K_SERVICE, + WEBSITE_SITE_NAME + } = process.env + + const service = AWS_LAMBDA_FUNCTION_NAME || + FUNCTION_NAME || // Google Cloud Function Name set by deprecated runtimes + K_SERVICE || // Google Cloud Function Name set by newer runtimes + WEBSITE_SITE_NAME || // set by Azure Functions + pkg.name || + 'node' - this._setUnit(defaults, 'sampleRate', undefined) - this._setBoolean(defaults, 'logInjection', false) - this._setArray(defaults, 'headerTags', []) + const defaults = setHiddenProperty(this, '_defaults', {}) + + this._setValue(defaults, 'appsec.apiSecurity.enabled', true) + this._setValue(defaults, 'appsec.apiSecurity.requestSampling', 0.1) + this._setValue(defaults, 'appsec.blockedTemplateGraphql', undefined) + this._setValue(defaults, 'appsec.blockedTemplateHtml', undefined) + this._setValue(defaults, 'appsec.blockedTemplateJson', undefined) + this._setValue(defaults, 'appsec.enabled', undefined) + this._setValue(defaults, 'appsec.eventTracking.enabled', true) + this._setValue(defaults, 'appsec.eventTracking.mode', 'safe') + this._setValue(defaults, 'appsec.obfuscatorKeyRegex', defaultWafObfuscatorKeyRegex) + this._setValue(defaults, 'appsec.obfuscatorValueRegex', defaultWafObfuscatorValueRegex) + this._setValue(defaults, 'appsec.rasp.enabled', true) + this._setValue(defaults, 'appsec.rateLimit', 100) + this._setValue(defaults, 'appsec.rules', undefined) + this._setValue(defaults, 'appsec.sca.enabled', null) + this._setValue(defaults, 'appsec.standalone.enabled', undefined) + this._setValue(defaults, 'appsec.stackTrace.enabled', true) + this._setValue(defaults, 'appsec.stackTrace.maxDepth', 32) + this._setValue(defaults, 'appsec.stackTrace.maxStackTraces', 2) + this._setValue(defaults, 'appsec.wafTimeout', 5e3) // µs + this._setValue(defaults, 'clientIpEnabled', false) + this._setValue(defaults, 'clientIpHeader', null) + this._setValue(defaults, 'codeOriginForSpans.enabled', false) + this._setValue(defaults, 'dbmPropagationMode', 'disabled') + this._setValue(defaults, 'dogstatsd.hostname', '127.0.0.1') + this._setValue(defaults, 'dogstatsd.port', '8125') + this._setValue(defaults, 'dsmEnabled', false) + this._setValue(defaults, 'dynamicInstrumentationEnabled', false) + this._setValue(defaults, 'env', undefined) + this._setValue(defaults, 'experimental.enableGetRumData', false) + this._setValue(defaults, 'experimental.exporter', undefined) + this._setValue(defaults, 'experimental.runtimeId', false) + this._setValue(defaults, 'flushInterval', 2000) + this._setValue(defaults, 'flushMinSpans', 1000) + this._setValue(defaults, 'gitMetadataEnabled', true) + this._setValue(defaults, 'grpc.client.error.statuses', GRPC_CLIENT_ERROR_STATUSES) + this._setValue(defaults, 'grpc.server.error.statuses', GRPC_SERVER_ERROR_STATUSES) + this._setValue(defaults, 'headerTags', []) + this._setValue(defaults, 'hostname', '127.0.0.1') + this._setValue(defaults, 'iast.cookieFilterPattern', '.{32,}') + this._setValue(defaults, 'iast.deduplicationEnabled', true) + this._setValue(defaults, 'iast.enabled', false) + this._setValue(defaults, 'iast.maxConcurrentRequests', 2) + this._setValue(defaults, 'iast.maxContextOperations', 2) + this._setValue(defaults, 'iast.redactionEnabled', true) + this._setValue(defaults, 'iast.redactionNamePattern', null) + this._setValue(defaults, 'iast.redactionValuePattern', null) + this._setValue(defaults, 'iast.requestSampling', 30) + this._setValue(defaults, 'iast.telemetryVerbosity', 'INFORMATION') + this._setValue(defaults, 'injectionEnabled', []) + this._setValue(defaults, 'isAzureFunction', false) + this._setValue(defaults, 'isCiVisibility', false) + this._setValue(defaults, 'isEarlyFlakeDetectionEnabled', false) + this._setValue(defaults, 'isFlakyTestRetriesEnabled', false) + this._setValue(defaults, 'flakyTestRetriesCount', 5) + this._setValue(defaults, 'isGCPFunction', false) + this._setValue(defaults, 'isGitUploadEnabled', false) + this._setValue(defaults, 'isIntelligentTestRunnerEnabled', false) + this._setValue(defaults, 'isManualApiEnabled', false) + this._setValue(defaults, 'ciVisibilityTestSessionName', '') + this._setValue(defaults, 'ciVisAgentlessLogSubmissionEnabled', false) + this._setValue(defaults, 'logInjection', false) + this._setValue(defaults, 'lookup', undefined) + this._setValue(defaults, 'memcachedCommandEnabled', false) + this._setValue(defaults, 'openAiLogsEnabled', false) + this._setValue(defaults, 'openaiSpanCharLimit', 128) + this._setValue(defaults, 'peerServiceMapping', {}) + this._setValue(defaults, 'plugins', true) + this._setValue(defaults, 'port', '8126') + this._setValue(defaults, 'profiling.enabled', undefined) + this._setValue(defaults, 'profiling.exporters', 'agent') + this._setValue(defaults, 'profiling.sourceMap', true) + this._setValue(defaults, 'profiling.longLivedThreshold', undefined) + this._setValue(defaults, 'protocolVersion', '0.4') + this._setValue(defaults, 'queryStringObfuscation', qsRegex) + this._setValue(defaults, 'remoteConfig.enabled', true) + this._setValue(defaults, 'remoteConfig.pollInterval', 5) // seconds + this._setValue(defaults, 'reportHostname', false) + this._setValue(defaults, 'runtimeMetrics', false) + this._setValue(defaults, 'sampleRate', undefined) + this._setValue(defaults, 'sampler.rateLimit', 100) + this._setValue(defaults, 'sampler.rules', []) + this._setValue(defaults, 'sampler.spanSamplingRules', []) + this._setValue(defaults, 'scope', undefined) + this._setValue(defaults, 'service', service) + this._setValue(defaults, 'serviceMapping', {}) + this._setValue(defaults, 'site', 'datadoghq.com') + this._setValue(defaults, 'spanAttributeSchema', 'v0') + this._setValue(defaults, 'spanComputePeerService', false) + this._setValue(defaults, 'spanLeakDebug', 0) + this._setValue(defaults, 'spanRemoveIntegrationFromService', false) + this._setValue(defaults, 'startupLogs', false) + this._setValue(defaults, 'stats.enabled', false) + this._setValue(defaults, 'tags', {}) + this._setValue(defaults, 'tagsHeaderMaxLength', 512) + this._setValue(defaults, 'telemetry.debug', false) + this._setValue(defaults, 'telemetry.dependencyCollection', true) + this._setValue(defaults, 'telemetry.enabled', true) + this._setValue(defaults, 'telemetry.heartbeatInterval', 60000) + this._setValue(defaults, 'telemetry.logCollection', false) + this._setValue(defaults, 'telemetry.metrics', true) + this._setValue(defaults, 'traceEnabled', true) + this._setValue(defaults, 'traceId128BitGenerationEnabled', true) + this._setValue(defaults, 'traceId128BitLoggingEnabled', false) + this._setValue(defaults, 'tracePropagationExtractFirst', false) + this._setValue(defaults, 'tracePropagationStyle.inject', ['datadog', 'tracecontext']) + this._setValue(defaults, 'tracePropagationStyle.extract', ['datadog', 'tracecontext']) + this._setValue(defaults, 'tracePropagationStyle.otelPropagators', false) + this._setValue(defaults, 'tracing', true) + this._setValue(defaults, 'url', undefined) + this._setValue(defaults, 'version', pkg.version) + this._setValue(defaults, 'instrumentation_config_id', undefined) } _applyEnvironment () { const { - DD_TRACE_SAMPLE_RATE, + AWS_LAMBDA_FUNCTION_NAME, + DD_AGENT_HOST, + DD_API_SECURITY_ENABLED, + DD_API_SECURITY_REQUEST_SAMPLE_RATE, + DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING, + DD_APPSEC_ENABLED, + DD_APPSEC_GRAPHQL_BLOCKED_TEMPLATE_JSON, + DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML, + DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON, + DD_APPSEC_MAX_STACK_TRACES, + DD_APPSEC_MAX_STACK_TRACE_DEPTH, + DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP, + DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP, + DD_APPSEC_RULES, + DD_APPSEC_SCA_ENABLED, + DD_APPSEC_STACK_TRACE_ENABLED, + DD_APPSEC_RASP_ENABLED, + DD_APPSEC_TRACE_RATE_LIMIT, + DD_APPSEC_WAF_TIMEOUT, + DD_CODE_ORIGIN_FOR_SPANS_ENABLED, + DD_DATA_STREAMS_ENABLED, + DD_DBM_PROPAGATION_MODE, + DD_DOGSTATSD_HOSTNAME, + DD_DOGSTATSD_PORT, + DD_DYNAMIC_INSTRUMENTATION_ENABLED, + DD_ENV, + DD_EXPERIMENTAL_API_SECURITY_ENABLED, + DD_EXPERIMENTAL_APPSEC_STANDALONE_ENABLED, + DD_EXPERIMENTAL_PROFILING_ENABLED, + DD_GRPC_CLIENT_ERROR_STATUSES, + DD_GRPC_SERVER_ERROR_STATUSES, + JEST_WORKER_ID, + DD_IAST_COOKIE_FILTER_PATTERN, + DD_IAST_DEDUPLICATION_ENABLED, + DD_IAST_ENABLED, + DD_IAST_MAX_CONCURRENT_REQUESTS, + DD_IAST_MAX_CONTEXT_OPERATIONS, + DD_IAST_REDACTION_ENABLED, + DD_IAST_REDACTION_NAME_PATTERN, + DD_IAST_REDACTION_VALUE_PATTERN, + DD_IAST_REQUEST_SAMPLING, + DD_IAST_TELEMETRY_VERBOSITY, + DD_INJECTION_ENABLED, + DD_INSTRUMENTATION_TELEMETRY_ENABLED, + DD_INSTRUMENTATION_CONFIG_ID, DD_LOGS_INJECTION, - DD_TRACE_HEADER_TAGS + DD_OPENAI_LOGS_ENABLED, + DD_OPENAI_SPAN_CHAR_LIMIT, + DD_PROFILING_ENABLED, + DD_PROFILING_EXPORTERS, + DD_PROFILING_SOURCE_MAP, + DD_INTERNAL_PROFILING_LONG_LIVED_THRESHOLD, + DD_REMOTE_CONFIGURATION_ENABLED, + DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS, + DD_RUNTIME_METRICS_ENABLED, + DD_SERVICE, + DD_SERVICE_MAPPING, + DD_SERVICE_NAME, + DD_SITE, + DD_SPAN_SAMPLING_RULES, + DD_SPAN_SAMPLING_RULES_FILE, + DD_TAGS, + DD_TELEMETRY_DEBUG, + DD_TELEMETRY_DEPENDENCY_COLLECTION_ENABLED, + DD_TELEMETRY_HEARTBEAT_INTERVAL, + DD_TELEMETRY_LOG_COLLECTION_ENABLED, + DD_TELEMETRY_METRICS_ENABLED, + DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED, + DD_TRACE_128_BIT_TRACEID_LOGGING_ENABLED, + DD_TRACE_AGENT_HOSTNAME, + DD_TRACE_AGENT_PORT, + DD_TRACE_AGENT_PROTOCOL_VERSION, + DD_TRACE_CLIENT_IP_ENABLED, + DD_TRACE_CLIENT_IP_HEADER, + DD_TRACE_ENABLED, + DD_TRACE_EXPERIMENTAL_EXPORTER, + DD_TRACE_EXPERIMENTAL_GET_RUM_DATA_ENABLED, + DD_TRACE_EXPERIMENTAL_RUNTIME_ID_ENABLED, + DD_TRACE_GIT_METADATA_ENABLED, + DD_TRACE_GLOBAL_TAGS, + DD_TRACE_HEADER_TAGS, + DD_TRACE_MEMCACHED_COMMAND_ENABLED, + DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP, + DD_TRACE_PARTIAL_FLUSH_MIN_SPANS, + DD_TRACE_PEER_SERVICE_MAPPING, + DD_TRACE_PROPAGATION_EXTRACT_FIRST, + DD_TRACE_PROPAGATION_STYLE, + DD_TRACE_PROPAGATION_STYLE_INJECT, + DD_TRACE_PROPAGATION_STYLE_EXTRACT, + DD_TRACE_RATE_LIMIT, + DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED, + DD_TRACE_REPORT_HOSTNAME, + DD_TRACE_SAMPLE_RATE, + DD_TRACE_SAMPLING_RULES, + DD_TRACE_SCOPE, + DD_TRACE_SPAN_ATTRIBUTE_SCHEMA, + DD_TRACE_SPAN_LEAK_DEBUG, + DD_TRACE_STARTUP_LOGS, + DD_TRACE_TAGS, + DD_TRACE_TELEMETRY_ENABLED, + DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH, + DD_TRACING_ENABLED, + DD_VERSION, + OTEL_METRICS_EXPORTER, + OTEL_PROPAGATORS, + OTEL_RESOURCE_ATTRIBUTES, + OTEL_SERVICE_NAME, + OTEL_TRACES_SAMPLER, + OTEL_TRACES_SAMPLER_ARG } = process.env - const env = this._env = {} + const tags = {} + const env = setHiddenProperty(this, '_env', {}) + setHiddenProperty(this, '_envUnprocessed', {}) - this._setUnit(env, 'sampleRate', DD_TRACE_SAMPLE_RATE) - this._setBoolean(env, 'logInjection', DD_LOGS_INJECTION) + tagger.add(tags, OTEL_RESOURCE_ATTRIBUTES, true) + tagger.add(tags, DD_TAGS) + tagger.add(tags, DD_TRACE_TAGS) + tagger.add(tags, DD_TRACE_GLOBAL_TAGS) + + this._setBoolean(env, 'appsec.apiSecurity.enabled', coalesce( + DD_API_SECURITY_ENABLED && isTrue(DD_API_SECURITY_ENABLED), + DD_EXPERIMENTAL_API_SECURITY_ENABLED && isTrue(DD_EXPERIMENTAL_API_SECURITY_ENABLED) + )) + this._setUnit(env, 'appsec.apiSecurity.requestSampling', DD_API_SECURITY_REQUEST_SAMPLE_RATE) + this._setValue(env, 'appsec.blockedTemplateGraphql', maybeFile(DD_APPSEC_GRAPHQL_BLOCKED_TEMPLATE_JSON)) + this._setValue(env, 'appsec.blockedTemplateHtml', maybeFile(DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML)) + this._envUnprocessed['appsec.blockedTemplateHtml'] = DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML + this._setValue(env, 'appsec.blockedTemplateJson', maybeFile(DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON)) + this._envUnprocessed['appsec.blockedTemplateJson'] = DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON + this._setBoolean(env, 'appsec.enabled', DD_APPSEC_ENABLED) + if (DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING) { + this._setValue(env, 'appsec.eventTracking.enabled', + ['extended', 'safe'].includes(DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING.toLowerCase())) + this._setValue(env, 'appsec.eventTracking.mode', DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING.toLowerCase()) + } + this._setString(env, 'appsec.obfuscatorKeyRegex', DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP) + this._setString(env, 'appsec.obfuscatorValueRegex', DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP) + this._setBoolean(env, 'appsec.rasp.enabled', DD_APPSEC_RASP_ENABLED) + this._setValue(env, 'appsec.rateLimit', maybeInt(DD_APPSEC_TRACE_RATE_LIMIT)) + this._envUnprocessed['appsec.rateLimit'] = DD_APPSEC_TRACE_RATE_LIMIT + this._setString(env, 'appsec.rules', DD_APPSEC_RULES) + // DD_APPSEC_SCA_ENABLED is never used locally, but only sent to the backend + this._setBoolean(env, 'appsec.sca.enabled', DD_APPSEC_SCA_ENABLED) + this._setBoolean(env, 'appsec.standalone.enabled', DD_EXPERIMENTAL_APPSEC_STANDALONE_ENABLED) + this._setBoolean(env, 'appsec.stackTrace.enabled', DD_APPSEC_STACK_TRACE_ENABLED) + this._setValue(env, 'appsec.stackTrace.maxDepth', maybeInt(DD_APPSEC_MAX_STACK_TRACE_DEPTH)) + this._envUnprocessed['appsec.stackTrace.maxDepth'] = DD_APPSEC_MAX_STACK_TRACE_DEPTH + this._setValue(env, 'appsec.stackTrace.maxStackTraces', maybeInt(DD_APPSEC_MAX_STACK_TRACES)) + this._envUnprocessed['appsec.stackTrace.maxStackTraces'] = DD_APPSEC_MAX_STACK_TRACES + this._setValue(env, 'appsec.wafTimeout', maybeInt(DD_APPSEC_WAF_TIMEOUT)) + this._envUnprocessed['appsec.wafTimeout'] = DD_APPSEC_WAF_TIMEOUT + this._setBoolean(env, 'clientIpEnabled', DD_TRACE_CLIENT_IP_ENABLED) + this._setString(env, 'clientIpHeader', DD_TRACE_CLIENT_IP_HEADER) + this._setBoolean(env, 'codeOriginForSpans.enabled', DD_CODE_ORIGIN_FOR_SPANS_ENABLED) + this._setString(env, 'dbmPropagationMode', DD_DBM_PROPAGATION_MODE) + this._setString(env, 'dogstatsd.hostname', DD_DOGSTATSD_HOSTNAME) + this._setString(env, 'dogstatsd.port', DD_DOGSTATSD_PORT) + this._setBoolean(env, 'dsmEnabled', DD_DATA_STREAMS_ENABLED) + this._setBoolean(env, 'dynamicInstrumentationEnabled', DD_DYNAMIC_INSTRUMENTATION_ENABLED) + this._setString(env, 'env', DD_ENV || tags.env) + this._setBoolean(env, 'traceEnabled', DD_TRACE_ENABLED) + this._setBoolean(env, 'experimental.enableGetRumData', DD_TRACE_EXPERIMENTAL_GET_RUM_DATA_ENABLED) + this._setString(env, 'experimental.exporter', DD_TRACE_EXPERIMENTAL_EXPORTER) + this._setBoolean(env, 'experimental.runtimeId', DD_TRACE_EXPERIMENTAL_RUNTIME_ID_ENABLED) + if (AWS_LAMBDA_FUNCTION_NAME) this._setValue(env, 'flushInterval', 0) + this._setValue(env, 'flushMinSpans', maybeInt(DD_TRACE_PARTIAL_FLUSH_MIN_SPANS)) + this._envUnprocessed.flushMinSpans = DD_TRACE_PARTIAL_FLUSH_MIN_SPANS + this._setBoolean(env, 'gitMetadataEnabled', DD_TRACE_GIT_METADATA_ENABLED) + this._setIntegerRangeSet(env, 'grpc.client.error.statuses', DD_GRPC_CLIENT_ERROR_STATUSES) + this._setIntegerRangeSet(env, 'grpc.server.error.statuses', DD_GRPC_SERVER_ERROR_STATUSES) this._setArray(env, 'headerTags', DD_TRACE_HEADER_TAGS) + this._setString(env, 'hostname', coalesce(DD_AGENT_HOST, DD_TRACE_AGENT_HOSTNAME)) + this._setString(env, 'iast.cookieFilterPattern', DD_IAST_COOKIE_FILTER_PATTERN) + this._setBoolean(env, 'iast.deduplicationEnabled', DD_IAST_DEDUPLICATION_ENABLED) + this._setBoolean(env, 'iast.enabled', DD_IAST_ENABLED) + this._setValue(env, 'iast.maxConcurrentRequests', maybeInt(DD_IAST_MAX_CONCURRENT_REQUESTS)) + this._envUnprocessed['iast.maxConcurrentRequests'] = DD_IAST_MAX_CONCURRENT_REQUESTS + this._setValue(env, 'iast.maxContextOperations', maybeInt(DD_IAST_MAX_CONTEXT_OPERATIONS)) + this._envUnprocessed['iast.maxContextOperations'] = DD_IAST_MAX_CONTEXT_OPERATIONS + this._setBoolean(env, 'iast.redactionEnabled', DD_IAST_REDACTION_ENABLED && !isFalse(DD_IAST_REDACTION_ENABLED)) + this._setString(env, 'iast.redactionNamePattern', DD_IAST_REDACTION_NAME_PATTERN) + this._setString(env, 'iast.redactionValuePattern', DD_IAST_REDACTION_VALUE_PATTERN) + const iastRequestSampling = maybeInt(DD_IAST_REQUEST_SAMPLING) + if (iastRequestSampling > -1 && iastRequestSampling < 101) { + this._setValue(env, 'iast.requestSampling', iastRequestSampling) + } + this._envUnprocessed['iast.requestSampling'] = DD_IAST_REQUEST_SAMPLING + this._setString(env, 'iast.telemetryVerbosity', DD_IAST_TELEMETRY_VERBOSITY) + this._setArray(env, 'injectionEnabled', DD_INJECTION_ENABLED) + this._setBoolean(env, 'isAzureFunction', getIsAzureFunction()) + this._setBoolean(env, 'isGCPFunction', getIsGCPFunction()) + this._setBoolean(env, 'logInjection', DD_LOGS_INJECTION) + // Requires an accompanying DD_APM_OBFUSCATION_MEMCACHED_KEEP_COMMAND=true in the agent + this._setBoolean(env, 'memcachedCommandEnabled', DD_TRACE_MEMCACHED_COMMAND_ENABLED) + this._setBoolean(env, 'openAiLogsEnabled', DD_OPENAI_LOGS_ENABLED) + this._setValue(env, 'openaiSpanCharLimit', maybeInt(DD_OPENAI_SPAN_CHAR_LIMIT)) + this._envUnprocessed.openaiSpanCharLimit = DD_OPENAI_SPAN_CHAR_LIMIT + if (DD_TRACE_PEER_SERVICE_MAPPING) { + this._setValue(env, 'peerServiceMapping', fromEntries( + DD_TRACE_PEER_SERVICE_MAPPING.split(',').map(x => x.trim().split(':')) + )) + this._envUnprocessed.peerServiceMapping = DD_TRACE_PEER_SERVICE_MAPPING + } + this._setString(env, 'port', DD_TRACE_AGENT_PORT) + const profilingEnabledEnv = coalesce(DD_EXPERIMENTAL_PROFILING_ENABLED, DD_PROFILING_ENABLED) + const profilingEnabled = isTrue(profilingEnabledEnv) + ? 'true' + : isFalse(profilingEnabledEnv) + ? 'false' + : profilingEnabledEnv === 'auto' ? 'auto' : undefined + this._setString(env, 'profiling.enabled', profilingEnabled) + this._setString(env, 'profiling.exporters', DD_PROFILING_EXPORTERS) + this._setBoolean(env, 'profiling.sourceMap', DD_PROFILING_SOURCE_MAP && !isFalse(DD_PROFILING_SOURCE_MAP)) + if (DD_INTERNAL_PROFILING_LONG_LIVED_THRESHOLD) { + // This is only used in testing to not have to wait 30s + this._setValue(env, 'profiling.longLivedThreshold', Number(DD_INTERNAL_PROFILING_LONG_LIVED_THRESHOLD)) + } + + this._setString(env, 'protocolVersion', DD_TRACE_AGENT_PROTOCOL_VERSION) + this._setString(env, 'queryStringObfuscation', DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP) + this._setBoolean(env, 'remoteConfig.enabled', coalesce( + DD_REMOTE_CONFIGURATION_ENABLED, + !this._isInServerlessEnvironment() + )) + this._setValue(env, 'remoteConfig.pollInterval', maybeFloat(DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS)) + this._envUnprocessed['remoteConfig.pollInterval'] = DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS + this._setBoolean(env, 'reportHostname', DD_TRACE_REPORT_HOSTNAME) + // only used to explicitly set runtimeMetrics to false + const otelSetRuntimeMetrics = String(OTEL_METRICS_EXPORTER).toLowerCase() === 'none' + ? false + : undefined + this._setBoolean(env, 'runtimeMetrics', DD_RUNTIME_METRICS_ENABLED || + otelSetRuntimeMetrics) + this._setArray(env, 'sampler.spanSamplingRules', reformatSpanSamplingRules(coalesce( + safeJsonParse(maybeFile(DD_SPAN_SAMPLING_RULES_FILE)), + safeJsonParse(DD_SPAN_SAMPLING_RULES) + ))) + this._setUnit(env, 'sampleRate', DD_TRACE_SAMPLE_RATE || + getFromOtelSamplerMap(OTEL_TRACES_SAMPLER, OTEL_TRACES_SAMPLER_ARG)) + this._setValue(env, 'sampler.rateLimit', DD_TRACE_RATE_LIMIT) + this._setSamplingRule(env, 'sampler.rules', safeJsonParse(DD_TRACE_SAMPLING_RULES)) + this._envUnprocessed['sampler.rules'] = DD_TRACE_SAMPLING_RULES + this._setString(env, 'scope', DD_TRACE_SCOPE) + this._setString(env, 'service', DD_SERVICE || DD_SERVICE_NAME || tags.service || OTEL_SERVICE_NAME) + if (DD_SERVICE_MAPPING) { + this._setValue(env, 'serviceMapping', fromEntries( + process.env.DD_SERVICE_MAPPING.split(',').map(x => x.trim().split(':')) + )) + } + this._setString(env, 'site', DD_SITE) + if (DD_TRACE_SPAN_ATTRIBUTE_SCHEMA) { + this._setString(env, 'spanAttributeSchema', validateNamingVersion(DD_TRACE_SPAN_ATTRIBUTE_SCHEMA)) + this._envUnprocessed.spanAttributeSchema = DD_TRACE_SPAN_ATTRIBUTE_SCHEMA + } + // 0: disabled, 1: logging, 2: garbage collection + logging + this._setValue(env, 'spanLeakDebug', maybeInt(DD_TRACE_SPAN_LEAK_DEBUG)) + this._setBoolean(env, 'spanRemoveIntegrationFromService', DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED) + this._setBoolean(env, 'startupLogs', DD_TRACE_STARTUP_LOGS) + this._setTags(env, 'tags', tags) + this._setValue(env, 'tagsHeaderMaxLength', DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH) + this._setBoolean(env, 'telemetry.enabled', coalesce( + DD_TRACE_TELEMETRY_ENABLED, // for backward compatibility + DD_INSTRUMENTATION_TELEMETRY_ENABLED, // to comply with instrumentation telemetry specs + !(this._isInServerlessEnvironment() || JEST_WORKER_ID) + )) + this._setString(env, 'instrumentation_config_id', DD_INSTRUMENTATION_CONFIG_ID) + this._setBoolean(env, 'telemetry.debug', DD_TELEMETRY_DEBUG) + this._setBoolean(env, 'telemetry.dependencyCollection', DD_TELEMETRY_DEPENDENCY_COLLECTION_ENABLED) + this._setValue(env, 'telemetry.heartbeatInterval', maybeInt(Math.floor(DD_TELEMETRY_HEARTBEAT_INTERVAL * 1000))) + this._envUnprocessed['telemetry.heartbeatInterval'] = DD_TELEMETRY_HEARTBEAT_INTERVAL * 1000 + this._setBoolean(env, 'telemetry.logCollection', DD_TELEMETRY_LOG_COLLECTION_ENABLED) + this._setBoolean(env, 'telemetry.metrics', DD_TELEMETRY_METRICS_ENABLED) + this._setBoolean(env, 'traceId128BitGenerationEnabled', DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED) + this._setBoolean(env, 'traceId128BitLoggingEnabled', DD_TRACE_128_BIT_TRACEID_LOGGING_ENABLED) + this._setBoolean(env, 'tracePropagationExtractFirst', DD_TRACE_PROPAGATION_EXTRACT_FIRST) + this._setBoolean(env, 'tracePropagationStyle.otelPropagators', + DD_TRACE_PROPAGATION_STYLE || + DD_TRACE_PROPAGATION_STYLE_INJECT || + DD_TRACE_PROPAGATION_STYLE_EXTRACT + ? false + : !!OTEL_PROPAGATORS) + this._setBoolean(env, 'tracing', DD_TRACING_ENABLED) + this._setString(env, 'version', DD_VERSION || tags.version) } _applyOptions (options) { - const opts = this._options = this._options || {} + const opts = setHiddenProperty(this, '_options', this._options || {}) + const tags = {} + setHiddenProperty(this, '_optsUnprocessed', {}) + + options = setHiddenProperty(this, '_optionsArg', Object.assign({ ingestion: {} }, options, opts)) + + tagger.add(tags, options.tags) + + this._setBoolean(opts, 'appsec.apiSecurity.enabled', options.appsec.apiSecurity?.enabled) + this._setUnit(opts, 'appsec.apiSecurity.requestSampling', options.appsec.apiSecurity?.requestSampling) + this._setValue(opts, 'appsec.blockedTemplateGraphql', maybeFile(options.appsec.blockedTemplateGraphql)) + this._setValue(opts, 'appsec.blockedTemplateHtml', maybeFile(options.appsec.blockedTemplateHtml)) + this._optsUnprocessed['appsec.blockedTemplateHtml'] = options.appsec.blockedTemplateHtml + this._setValue(opts, 'appsec.blockedTemplateJson', maybeFile(options.appsec.blockedTemplateJson)) + this._optsUnprocessed['appsec.blockedTemplateJson'] = options.appsec.blockedTemplateJson + this._setBoolean(opts, 'appsec.enabled', options.appsec.enabled) + let eventTracking = options.appsec.eventTracking?.mode + if (eventTracking) { + eventTracking = eventTracking.toLowerCase() + this._setValue(opts, 'appsec.eventTracking.enabled', ['extended', 'safe'].includes(eventTracking)) + this._setValue(opts, 'appsec.eventTracking.mode', eventTracking) + } + this._setString(opts, 'appsec.obfuscatorKeyRegex', options.appsec.obfuscatorKeyRegex) + this._setString(opts, 'appsec.obfuscatorValueRegex', options.appsec.obfuscatorValueRegex) + this._setBoolean(opts, 'appsec.rasp.enabled', options.appsec.rasp?.enabled) + this._setValue(opts, 'appsec.rateLimit', maybeInt(options.appsec.rateLimit)) + this._optsUnprocessed['appsec.rateLimit'] = options.appsec.rateLimit + this._setString(opts, 'appsec.rules', options.appsec.rules) + this._setBoolean(opts, 'appsec.standalone.enabled', options.experimental?.appsec?.standalone?.enabled) + this._setBoolean(opts, 'appsec.stackTrace.enabled', options.appsec.stackTrace?.enabled) + this._setValue(opts, 'appsec.stackTrace.maxDepth', maybeInt(options.appsec.stackTrace?.maxDepth)) + this._optsUnprocessed['appsec.stackTrace.maxDepth'] = options.appsec.stackTrace?.maxDepth + this._setValue(opts, 'appsec.stackTrace.maxStackTraces', maybeInt(options.appsec.stackTrace?.maxStackTraces)) + this._optsUnprocessed['appsec.stackTrace.maxStackTraces'] = options.appsec.stackTrace?.maxStackTraces + this._setValue(opts, 'appsec.wafTimeout', maybeInt(options.appsec.wafTimeout)) + this._optsUnprocessed['appsec.wafTimeout'] = options.appsec.wafTimeout + this._setBoolean(opts, 'clientIpEnabled', options.clientIpEnabled) + this._setString(opts, 'clientIpHeader', options.clientIpHeader) + this._setBoolean(opts, 'codeOriginForSpans.enabled', options.codeOriginForSpans?.enabled) + this._setString(opts, 'dbmPropagationMode', options.dbmPropagationMode) + if (options.dogstatsd) { + this._setString(opts, 'dogstatsd.hostname', options.dogstatsd.hostname) + this._setString(opts, 'dogstatsd.port', options.dogstatsd.port) + } + this._setBoolean(opts, 'dsmEnabled', options.dsmEnabled) + this._setBoolean(opts, 'dynamicInstrumentationEnabled', options.experimental?.dynamicInstrumentationEnabled) + this._setString(opts, 'env', options.env || tags.env) + this._setBoolean(opts, 'experimental.enableGetRumData', options.experimental?.enableGetRumData) + this._setString(opts, 'experimental.exporter', options.experimental?.exporter) + this._setBoolean(opts, 'experimental.runtimeId', options.experimental?.runtimeId) + this._setValue(opts, 'flushInterval', maybeInt(options.flushInterval)) + this._optsUnprocessed.flushInterval = options.flushInterval + this._setValue(opts, 'flushMinSpans', maybeInt(options.flushMinSpans)) + this._optsUnprocessed.flushMinSpans = options.flushMinSpans + this._setArray(opts, 'headerTags', options.headerTags) + this._setString(opts, 'hostname', options.hostname) + this._setString(opts, 'iast.cookieFilterPattern', options.iast?.cookieFilterPattern) + this._setBoolean(opts, 'iast.deduplicationEnabled', options.iast && options.iast.deduplicationEnabled) + this._setBoolean(opts, 'iast.enabled', + options.iast && (options.iast === true || options.iast.enabled === true)) + this._setValue(opts, 'iast.maxConcurrentRequests', + maybeInt(options.iast?.maxConcurrentRequests)) + this._optsUnprocessed['iast.maxConcurrentRequests'] = options.iast?.maxConcurrentRequests + this._setValue(opts, 'iast.maxContextOperations', maybeInt(options.iast?.maxContextOperations)) + this._optsUnprocessed['iast.maxContextOperations'] = options.iast?.maxContextOperations + this._setBoolean(opts, 'iast.redactionEnabled', options.iast?.redactionEnabled) + this._setString(opts, 'iast.redactionNamePattern', options.iast?.redactionNamePattern) + this._setString(opts, 'iast.redactionValuePattern', options.iast?.redactionValuePattern) + const iastRequestSampling = maybeInt(options.iast?.requestSampling) + if (iastRequestSampling > -1 && iastRequestSampling < 101) { + this._setValue(opts, 'iast.requestSampling', iastRequestSampling) + this._optsUnprocessed['iast.requestSampling'] = options.iast?.requestSampling + } + this._setString(opts, 'iast.telemetryVerbosity', options.iast && options.iast.telemetryVerbosity) + this._setBoolean(opts, 'isCiVisibility', options.isCiVisibility) + this._setBoolean(opts, 'logInjection', options.logInjection) + this._setString(opts, 'lookup', options.lookup) + this._setBoolean(opts, 'openAiLogsEnabled', options.openAiLogsEnabled) + this._setValue(opts, 'peerServiceMapping', options.peerServiceMapping) + this._setBoolean(opts, 'plugins', options.plugins) + this._setString(opts, 'port', options.port) + const strProfiling = String(options.profiling) + if (['true', 'false', 'auto'].includes(strProfiling)) { + this._setString(opts, 'profiling.enabled', strProfiling) + } + this._setString(opts, 'protocolVersion', options.protocolVersion) + if (options.remoteConfig) { + this._setValue(opts, 'remoteConfig.pollInterval', maybeFloat(options.remoteConfig.pollInterval)) + this._optsUnprocessed['remoteConfig.pollInterval'] = options.remoteConfig.pollInterval + } + this._setBoolean(opts, 'reportHostname', options.reportHostname) + this._setBoolean(opts, 'runtimeMetrics', options.runtimeMetrics) + this._setArray(opts, 'sampler.spanSamplingRules', reformatSpanSamplingRules(options.spanSamplingRules)) + this._setUnit(opts, 'sampleRate', coalesce(options.sampleRate, options.ingestion.sampleRate)) + const ingestion = options.ingestion || {} + this._setValue(opts, 'sampler.rateLimit', coalesce(options.rateLimit, ingestion.rateLimit)) + this._setSamplingRule(opts, 'sampler.rules', options.samplingRules) + this._setString(opts, 'service', options.service || tags.service) + this._setValue(opts, 'serviceMapping', options.serviceMapping) + this._setString(opts, 'site', options.site) + if (options.spanAttributeSchema) { + this._setString(opts, 'spanAttributeSchema', validateNamingVersion(options.spanAttributeSchema)) + this._optsUnprocessed.spanAttributeSchema = options.spanAttributeSchema + } + this._setBoolean(opts, 'spanRemoveIntegrationFromService', options.spanRemoveIntegrationFromService) + this._setBoolean(opts, 'startupLogs', options.startupLogs) + this._setTags(opts, 'tags', tags) + this._setBoolean(opts, 'traceId128BitGenerationEnabled', options.traceId128BitGenerationEnabled) + this._setBoolean(opts, 'traceId128BitLoggingEnabled', options.traceId128BitLoggingEnabled) + this._setString(opts, 'version', options.version || tags.version) + } - options = Object.assign({ ingestion: {} }, options, opts) + _isCiVisibility () { + return coalesce( + this._optionsArg.isCiVisibility, + this._defaults.isCiVisibility + ) + } - this._setUnit(opts, 'sampleRate', coalesce(options.sampleRate, options.ingestion.sampleRate)) - this._setBoolean(opts, 'logInjection', options.logInjection) - this._setArray(opts, 'headerTags', options.headerTags) + _isCiVisibilityItrEnabled () { + return coalesce( + process.env.DD_CIVISIBILITY_ITR_ENABLED, + true + ) + } + + _getHostname () { + const DD_CIVISIBILITY_AGENTLESS_URL = process.env.DD_CIVISIBILITY_AGENTLESS_URL + const url = DD_CIVISIBILITY_AGENTLESS_URL + ? new URL(DD_CIVISIBILITY_AGENTLESS_URL) + : getAgentUrl(this._getTraceAgentUrl(), this._optionsArg) + const DD_AGENT_HOST = coalesce( + this._optionsArg.hostname, + process.env.DD_AGENT_HOST, + process.env.DD_TRACE_AGENT_HOSTNAME, + '127.0.0.1' + ) + return DD_AGENT_HOST || (url && url.hostname) + } + + _getSpanComputePeerService () { + const DD_TRACE_SPAN_ATTRIBUTE_SCHEMA = validateNamingVersion( + coalesce( + this._optionsArg.spanAttributeSchema, + process.env.DD_TRACE_SPAN_ATTRIBUTE_SCHEMA + ) + ) + + const peerServiceSet = ( + this._optionsArg.hasOwnProperty('spanComputePeerService') || + process.env.hasOwnProperty('DD_TRACE_PEER_SERVICE_DEFAULTS_ENABLED') + ) + const peerServiceValue = coalesce( + this._optionsArg.spanComputePeerService, + process.env.DD_TRACE_PEER_SERVICE_DEFAULTS_ENABLED + ) + + const spanComputePeerService = ( + DD_TRACE_SPAN_ATTRIBUTE_SCHEMA === 'v0' + // In v0, peer service is computed only if it is explicitly set to true + ? peerServiceSet && isTrue(peerServiceValue) + // In >v0, peer service is false only if it is explicitly set to false + : (peerServiceSet ? !isFalse(peerServiceValue) : true) + ) + + return spanComputePeerService + } + + _isCiVisibilityGitUploadEnabled () { + return coalesce( + process.env.DD_CIVISIBILITY_GIT_UPLOAD_ENABLED, + true + ) + } + + _isCiVisibilityManualApiEnabled () { + return coalesce( + process.env.DD_CIVISIBILITY_MANUAL_API_ENABLED, + true + ) + } + + _isTraceStatsComputationEnabled () { + return coalesce( + this._optionsArg.stats, + process.env.DD_TRACE_STATS_COMPUTATION_ENABLED, + getIsGCPFunction() || getIsAzureFunction() + ) + } + + _getTraceAgentUrl () { + return coalesce( + this._optionsArg.url, + process.env.DD_TRACE_AGENT_URL, + process.env.DD_TRACE_URL, + null + ) + } + + // handles values calculated from a mixture of options and env vars + _applyCalculated () { + const calc = setHiddenProperty(this, '_calculated', {}) + + const { + DD_CIVISIBILITY_AGENTLESS_URL, + DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED, + DD_CIVISIBILITY_FLAKY_RETRY_ENABLED, + DD_CIVISIBILITY_FLAKY_RETRY_COUNT, + DD_TEST_SESSION_NAME, + DD_AGENTLESS_LOG_SUBMISSION_ENABLED + } = process.env + + if (DD_CIVISIBILITY_AGENTLESS_URL) { + this._setValue(calc, 'url', new URL(DD_CIVISIBILITY_AGENTLESS_URL)) + } else { + this._setValue(calc, 'url', getAgentUrl(this._getTraceAgentUrl(), this._optionsArg)) + } + if (this._isCiVisibility()) { + this._setBoolean(calc, 'isEarlyFlakeDetectionEnabled', + coalesce(DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED, true)) + this._setBoolean(calc, 'isFlakyTestRetriesEnabled', + coalesce(DD_CIVISIBILITY_FLAKY_RETRY_ENABLED, true)) + this._setValue(calc, 'flakyTestRetriesCount', coalesce(maybeInt(DD_CIVISIBILITY_FLAKY_RETRY_COUNT), 5)) + this._setBoolean(calc, 'isIntelligentTestRunnerEnabled', isTrue(this._isCiVisibilityItrEnabled())) + this._setBoolean(calc, 'isManualApiEnabled', !isFalse(this._isCiVisibilityManualApiEnabled())) + this._setString(calc, 'ciVisibilityTestSessionName', DD_TEST_SESSION_NAME) + this._setBoolean(calc, 'ciVisAgentlessLogSubmissionEnabled', isTrue(DD_AGENTLESS_LOG_SUBMISSION_ENABLED)) + } + this._setString(calc, 'dogstatsd.hostname', this._getHostname()) + this._setBoolean(calc, 'isGitUploadEnabled', + calc.isIntelligentTestRunnerEnabled && !isFalse(this._isCiVisibilityGitUploadEnabled())) + this._setBoolean(calc, 'spanComputePeerService', this._getSpanComputePeerService()) + this._setBoolean(calc, 'stats.enabled', this._isTraceStatsComputationEnabled()) + const defaultPropagationStyle = this._getDefaultPropagationStyle(this._optionsArg) + this._setValue(calc, 'tracePropagationStyle.inject', propagationStyle( + 'inject', + this._optionsArg.tracePropagationStyle + )) + this._setValue(calc, 'tracePropagationStyle.extract', propagationStyle( + 'extract', + this._optionsArg.tracePropagationStyle + )) + if (defaultPropagationStyle.length > 2) { + calc['tracePropagationStyle.inject'] = calc['tracePropagationStyle.inject'] || defaultPropagationStyle + calc['tracePropagationStyle.extract'] = calc['tracePropagationStyle.extract'] || defaultPropagationStyle + } + + const iastEnabled = coalesce(this._options['iast.enabled'], this._env['iast.enabled']) + const profilingEnabled = coalesce(this._options['profiling.enabled'], this._env['profiling.enabled']) + const injectionIncludesProfiler = (this._env.injectionEnabled || []).includes('profiler') + if (iastEnabled || ['auto', 'true'].includes(profilingEnabled) || injectionIncludesProfiler) { + this._setBoolean(calc, 'telemetry.logCollection', true) + } } _applyRemote (options) { - const opts = this._remote = this._remote || {} + const opts = setHiddenProperty(this, '_remote', this._remote || {}) + setHiddenProperty(this, '_remoteUnprocessed', {}) + const tags = {} const headerTags = options.tracing_header_tags ? options.tracing_header_tags.map(tag => { return tag.tag_name ? `${tag.header}:${tag.tag_name}` : tag.header }) : undefined + tagger.add(tags, options.tracing_tags) + if (Object.keys(tags).length) tags['runtime-id'] = runtimeId + this._setUnit(opts, 'sampleRate', options.tracing_sampling_rate) this._setBoolean(opts, 'logInjection', options.log_injection_enabled) this._setArray(opts, 'headerTags', headerTags) + this._setTags(opts, 'tags', tags) + this._setBoolean(opts, 'tracing', options.tracing_enabled) + this._remoteUnprocessed['sampler.rules'] = options.tracing_sampling_rules + this._setSamplingRule(opts, 'sampler.rules', this._reformatTags(options.tracing_sampling_rules)) + } + + _reformatTags (samplingRules) { + for (const rule of (samplingRules || [])) { + const reformattedTags = {} + if (rule.tags) { + for (const tag of (rule.tags || {})) { + reformattedTags[tag.key] = tag.value_glob + } + rule.tags = reformattedTags + } + } + return samplingRules } _setBoolean (obj, name, value) { @@ -810,12 +1159,16 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?) } _setArray (obj, name, value) { - if (value === null || value === undefined) { + if (value == null) { return this._setValue(obj, name, null) } if (typeof value === 'string') { - value = value && value.split(',') + value = value.split(',').map(item => { + // Trim each item and remove whitespace around the colon + const [key, val] = item.split(':').map(part => part.trim()) + return val !== undefined ? `${key}:${val}` : key + }) } if (Array.isArray(value)) { @@ -823,6 +1176,57 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?) } } + _setIntegerRangeSet (obj, name, value) { + if (value == null) { + return this._setValue(obj, name, null) + } + value = value.split(',') + const result = [] + + value.forEach(val => { + if (val.includes('-')) { + const [start, end] = val.split('-').map(Number) + for (let i = start; i <= end; i++) { + result.push(i) + } + } else { + result.push(Number(val)) + } + }) + this._setValue(obj, name, result) + } + + _setSamplingRule (obj, name, value) { + if (value == null) { + return this._setValue(obj, name, null) + } + + if (typeof value === 'string') { + value = value.split(',') + } + + if (Array.isArray(value)) { + value = value.map(rule => { + return remapify(rule, { + sample_rate: 'sampleRate' + }) + }) + this._setValue(obj, name, value) + } + } + + _setString (obj, name, value) { + obj[name] = value ? String(value) : undefined // unset for empty strings + } + + _setTags (obj, name, value) { + if (!value || Object.keys(value).length === 0) { + return this._setValue(obj, name, null) + } + + this._setValue(obj, name, value) + } + _setValue (obj, name, value) { obj[name] = value } @@ -830,22 +1234,30 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?) // TODO: Report origin changes and errors to telemetry. // TODO: Deeply merge configurations. // TODO: Move change tracking to telemetry. + // for telemetry reporting, `name`s in `containers` need to be keys from: + // eslint-disable-next-line max-len + // https://github.com/DataDog/dd-go/blob/prod/trace/apps/tracer-telemetry-intake/telemetry-payload/static/config_norm_rules.json _merge () { - const containers = [this._remote, this._options, this._env, this._defaults] - const origins = ['remote_config', 'code', 'env_var', 'default'] + const containers = [this._remote, this._options, this._env, this._calculated, this._defaults] + const origins = ['remote_config', 'code', 'env_var', 'calculated', 'default'] + const unprocessedValues = [this._remoteUnprocessed, this._optsUnprocessed, this._envUnprocessed, {}, {}] const changes = [] for (const name in this._defaults) { for (let i = 0; i < containers.length; i++) { const container = containers[i] - const origin = origins[i] + const value = container[name] - if ((container[name] !== null && container[name] !== undefined) || container === this._defaults) { - if (this[name] === container[name] && this.hasOwnProperty(name)) break + if ((value !== null && value !== undefined) || container === this._defaults) { + if (get(this, name) === value && has(this, name)) break - const value = this[name] = container[name] + set(this, name, value) - changes.push({ name, value, origin }) + changes.push({ + name, + value: unprocessedValues[i][name] || value, + origin: origins[i] + }) break } @@ -853,11 +1265,20 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?) } this.sampler.sampleRate = this.sampleRate - updateConfig(changes, this) } } +function maybeInt (number) { + const parsed = parseInt(number) + return isNaN(parsed) ? undefined : parsed +} + +function maybeFloat (number) { + const parsed = parseFloat(number) + return isNaN(parsed) ? undefined : parsed +} + function getAgentUrl (url, options) { if (url) return new URL(url) @@ -875,4 +1296,13 @@ function getAgentUrl (url, options) { } } +function setHiddenProperty (obj, name, value) { + Object.defineProperty(obj, name, { + value, + enumerable: false, + writable: true + }) + return obj[name] +} + module.exports = Config diff --git a/packages/dd-trace/src/constants.js b/packages/dd-trace/src/constants.js index 89dcbdf4c7e..a242f717a37 100644 --- a/packages/dd-trace/src/constants.js +++ b/packages/dd-trace/src/constants.js @@ -15,6 +15,8 @@ module.exports = { SAMPLING_MECHANISM_MANUAL: 4, SAMPLING_MECHANISM_APPSEC: 5, SAMPLING_MECHANISM_SPAN: 8, + SAMPLING_MECHANISM_REMOTE_USER: 11, + SAMPLING_MECHANISM_REMOTE_DYNAMIC: 12, SPAN_SAMPLING_MECHANISM: '_dd.span_sampling.mechanism', SPAN_SAMPLING_RULE_RATE: '_dd.span_sampling.rule_rate', SPAN_SAMPLING_MAX_PER_SECOND: '_dd.span_sampling.max_per_second', @@ -30,5 +32,19 @@ module.exports = { PEER_SERVICE_SOURCE_KEY: '_dd.peer.service.source', PEER_SERVICE_REMAP_KEY: '_dd.peer.service.remapped_from', SCI_REPOSITORY_URL: '_dd.git.repository_url', - SCI_COMMIT_SHA: '_dd.git.commit.sha' + SCI_COMMIT_SHA: '_dd.git.commit.sha', + APM_TRACING_ENABLED_KEY: '_dd.apm.enabled', + APPSEC_PROPAGATION_KEY: '_dd.p.appsec', + PAYLOAD_TAG_REQUEST_PREFIX: 'aws.request.body', + PAYLOAD_TAG_RESPONSE_PREFIX: 'aws.response.body', + PAYLOAD_TAGGING_MAX_TAGS: 758, + SCHEMA_DEFINITION: 'schema.definition', + SCHEMA_WEIGHT: 'schema.weight', + SCHEMA_TYPE: 'schema.type', + SCHEMA_ID: 'schema.id', + SCHEMA_TOPIC: 'schema.topic', + SCHEMA_OPERATION: 'schema.operation', + SCHEMA_NAME: 'schema.name', + GRPC_CLIENT_ERROR_STATUSES: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], + GRPC_SERVER_ERROR_STATUSES: [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] } diff --git a/packages/dd-trace/src/data_streams.js b/packages/dd-trace/src/data_streams.js new file mode 100644 index 00000000000..ae3d4c22316 --- /dev/null +++ b/packages/dd-trace/src/data_streams.js @@ -0,0 +1,44 @@ +const DataStreamsContext = require('./data_streams_context') + +class DataStreamsCheckpointer { + constructor (tracer) { + this.tracer = tracer + this.config = tracer._config + this.dsmProcessor = tracer._dataStreamsProcessor + } + + setProduceCheckpoint (type, target, carrier) { + if (!this.config.dsmEnabled) return + + const ctx = this.dsmProcessor.setCheckpoint( + ['type:' + type, 'topic:' + target, 'direction:out', 'manual_checkpoint:true'], + null, + DataStreamsContext.getDataStreamsContext(), + null + ) + DataStreamsContext.setDataStreamsContext(ctx) + + this.tracer.inject(ctx, 'text_map_dsm', carrier) + } + + setConsumeCheckpoint (type, source, carrier) { + if (!this.config.dsmEnabled) return + + const parentCtx = this.tracer.extract('text_map_dsm', carrier) + DataStreamsContext.setDataStreamsContext(parentCtx) + + const ctx = this.dsmProcessor.setCheckpoint( + ['type:' + type, 'topic:' + source, 'direction:in', 'manual_checkpoint:true'], + null, + parentCtx, + null + ) + DataStreamsContext.setDataStreamsContext(ctx) + + return ctx + } +} + +module.exports = { + DataStreamsCheckpointer +} diff --git a/packages/dd-trace/src/data_streams_context.js b/packages/dd-trace/src/data_streams_context.js index 94152fe11d9..e3c62d35e25 100644 --- a/packages/dd-trace/src/data_streams_context.js +++ b/packages/dd-trace/src/data_streams_context.js @@ -1,4 +1,5 @@ const { storage } = require('../../datadog-core') +const log = require('./log') function getDataStreamsContext () { const store = storage.getStore() @@ -6,7 +7,9 @@ function getDataStreamsContext () { } function setDataStreamsContext (dataStreamsContext) { - storage.enterWith({ ...(storage.getStore()), dataStreamsContext }) + log.debug(() => `Setting new DSM Context: ${JSON.stringify(dataStreamsContext)}.`) + + if (dataStreamsContext) storage.enterWith({ ...(storage.getStore()), dataStreamsContext }) } module.exports = { diff --git a/packages/dd-trace/src/datastreams/fnv.js b/packages/dd-trace/src/datastreams/fnv.js new file mode 100644 index 00000000000..c226ec40cd4 --- /dev/null +++ b/packages/dd-trace/src/datastreams/fnv.js @@ -0,0 +1,23 @@ +const FNV_64_PRIME = BigInt('0x100000001B3') +const FNV1_64_INIT = BigInt('0xCBF29CE484222325') + +function fnv (data, hvalInit, fnvPrime, fnvSize) { + let hval = hvalInit + for (const byte of data) { + hval = (hval * fnvPrime) % fnvSize + hval = hval ^ BigInt(byte) + } + return hval +} + +function fnv64 (data) { + if (!Buffer.isBuffer(data)) { + data = Buffer.from(data, 'utf-8') + } + const byteArray = new Uint8Array(data) + return fnv(byteArray, FNV1_64_INIT, FNV_64_PRIME, BigInt(2) ** BigInt(64)) +} + +module.exports = { + fnv64 +} diff --git a/packages/dd-trace/src/datastreams/pathway.js b/packages/dd-trace/src/datastreams/pathway.js index 9865bfee7a7..ed2f6cc85f8 100644 --- a/packages/dd-trace/src/datastreams/pathway.js +++ b/packages/dd-trace/src/datastreams/pathway.js @@ -4,22 +4,32 @@ const crypto = require('crypto') const { encodeVarint, decodeVarint } = require('./encoding') const LRUCache = require('lru-cache') +const log = require('../log') +const pick = require('../../../datadog-core/src/utils/src/pick') const options = { max: 500 } const cache = new LRUCache(options) +const CONTEXT_PROPAGATION_KEY = 'dd-pathway-ctx' +const CONTEXT_PROPAGATION_KEY_BASE64 = 'dd-pathway-ctx-base64' + +const logKeys = [CONTEXT_PROPAGATION_KEY, CONTEXT_PROPAGATION_KEY_BASE64] + function shaHash (checkpointString) { const hash = crypto.createHash('md5').update(checkpointString).digest('hex').slice(0, 16) return Buffer.from(hash, 'hex') } function computeHash (service, env, edgeTags, parentHash) { - const key = `${service}${env}` + edgeTags.join('') + parentHash.toString() + edgeTags.sort() + const hashableEdgeTags = edgeTags.filter(item => item !== 'manual_checkpoint:true') + + const key = `${service}${env}` + hashableEdgeTags.join('') + parentHash.toString() if (cache.get(key)) { return cache.get(key) } - const currentHash = shaHash(`${service}${env}` + edgeTags.join('')) - const buf = Buffer.concat([ currentHash, parentHash ], 16) + const currentHash = shaHash(`${service}${env}` + hashableEdgeTags.join('')) + const buf = Buffer.concat([currentHash, parentHash], 16) const val = shaHash(buf.toString()) cache.set(key, val) return val @@ -33,6 +43,11 @@ function encodePathwayContext (dataStreamsContext) { ], 20) } +function encodePathwayContextBase64 (dataStreamsContext) { + const encodedPathway = encodePathwayContext(dataStreamsContext) + return encodedPathway.toString('base64') +} + function decodePathwayContext (pathwayContext) { if (pathwayContext == null || pathwayContext.length < 8) { return null @@ -51,8 +66,60 @@ function decodePathwayContext (pathwayContext) { return { hash: pathwayHash, pathwayStartNs: pathwayStartMs * 1e6, edgeStartNs: edgeStartMs * 1e6 } } +function decodePathwayContextBase64 (pathwayContext) { + if (pathwayContext == null || pathwayContext.length < 8) { + return + } + if (Buffer.isBuffer(pathwayContext)) { + pathwayContext = pathwayContext.toString() + } + const encodedPathway = Buffer.from(pathwayContext, 'base64') + return decodePathwayContext(encodedPathway) +} + +class DsmPathwayCodec { + // we use a class for encoding / decoding in case we update our encoding/decoding. A class will make updates easier + // instead of using individual functions. + static encode (dataStreamsContext, carrier) { + if (!dataStreamsContext || !dataStreamsContext.hash) { + return + } + carrier[CONTEXT_PROPAGATION_KEY_BASE64] = encodePathwayContextBase64(dataStreamsContext) + + log.debug(() => `Injected into DSM carrier: ${JSON.stringify(pick(carrier, logKeys))}.`) + } + + static decode (carrier) { + log.debug(() => `Attempting extract from DSM carrier: ${JSON.stringify(pick(carrier, logKeys))}.`) + + if (carrier == null) return + + let ctx + if (CONTEXT_PROPAGATION_KEY_BASE64 in carrier) { + // decode v2 encoding of base64 + ctx = decodePathwayContextBase64(carrier[CONTEXT_PROPAGATION_KEY_BASE64]) + } else if (CONTEXT_PROPAGATION_KEY in carrier) { + try { + // decode v1 encoding + ctx = decodePathwayContext(carrier[CONTEXT_PROPAGATION_KEY]) + } catch { + // pass + } + // cover case where base64 context was received under wrong key + if (!ctx && CONTEXT_PROPAGATION_KEY in carrier) { + ctx = decodePathwayContextBase64(carrier[CONTEXT_PROPAGATION_KEY]) + } + } + + return ctx + } +} + module.exports = { computePathwayHash: computeHash, encodePathwayContext, - decodePathwayContext + decodePathwayContext, + encodePathwayContextBase64, + decodePathwayContextBase64, + DsmPathwayCodec } diff --git a/packages/dd-trace/src/datastreams/processor.js b/packages/dd-trace/src/datastreams/processor.js index 601d81441d8..d036af805a7 100644 --- a/packages/dd-trace/src/datastreams/processor.js +++ b/packages/dd-trace/src/datastreams/processor.js @@ -4,16 +4,18 @@ const pkg = require('../../../../package.json') const Uint64 = require('int64-buffer').Uint64BE const { LogCollapsingLowestDenseDDSketch } = require('@datadog/sketches-js') -const { encodePathwayContext } = require('./pathway') +const { DsmPathwayCodec } = require('./pathway') const { DataStreamsWriter } = require('./writer') const { computePathwayHash } = require('./pathway') const { types } = require('util') const { PATHWAY_HASH } = require('../../../../ext/tags') +const { SchemaBuilder } = require('./schemas/schema_builder') +const { SchemaSampler } = require('./schemas/schema_sampler') +const log = require('../log') const ENTRY_PARENT_HASH = Buffer.from('0000000000000000', 'hex') const HIGH_ACCURACY_DISTRIBUTION = 0.0075 -const CONTEXT_PROPAGATION_KEY = 'dd-pathway-ctx' class StatsPoint { constructor (hash, parentHash, edgeTags) { @@ -45,14 +47,73 @@ class StatsPoint { } } -class StatsBucket extends Map { +class Backlog { + constructor ({ offset, ...tags }) { + this._tags = Object.keys(tags).sort().map(key => `${key}:${tags[key]}`) + this._hash = this._tags.join(',') + this._offset = offset + } + + get hash () { return this._hash } + + get offset () { return this._offset } + + get tags () { return this._tags } + + encode () { + return { + Tags: this.tags, + Value: this.offset + } + } +} + +class StatsBucket { + constructor () { + this._checkpoints = new Map() + this._backlogs = new Map() + } + + get checkpoints () { + return this._checkpoints + } + + get backlogs () { + return this._backlogs + } + forCheckpoint (checkpoint) { const key = checkpoint.hash - if (!this.has(key)) { - this.set(key, new StatsPoint(checkpoint.hash, checkpoint.parentHash, checkpoint.edgeTags)) // StatsPoint + if (!this._checkpoints.has(key)) { + this._checkpoints.set( + key, new StatsPoint(checkpoint.hash, checkpoint.parentHash, checkpoint.edgeTags) + ) } - return this.get(key) + return this._checkpoints.get(key) + } + + /** + * Conditionally add a backlog to the bucket. If there is currently an offset + * matching the backlog's tags, overwrite the offset IFF the backlog's offset + * is greater than the recorded offset. + * + * @typedef {{[key: string]: string}} BacklogData + * @property {number} offset + * + * @param {BacklogData} backlogData + * @returns {Backlog} + */ + forBacklog (backlogData) { + const backlog = new Backlog(backlogData) + const existingBacklog = this._backlogs.get(backlog.hash) + if (existingBacklog !== undefined) { + if (existingBacklog.offset > backlog.offset) { + return existingBacklog + } + } + this._backlogs.set(backlog.hash, backlog) + return backlog } } @@ -66,6 +127,21 @@ function getSizeOrZero (obj) { if (Buffer.isBuffer(obj)) { return obj.length } + if (Array.isArray(obj) && obj.length > 0) { + if (typeof obj[0] === 'number') return Buffer.from(obj).length + let payloadSize = 0 + obj.forEach(item => { + payloadSize += getSizeOrZero(item) + }) + return payloadSize + } + if (obj !== null && typeof obj === 'object') { + try { + return getHeadersSize(obj) + } catch { + // pass + } + } return 0 } @@ -79,6 +155,11 @@ function getMessageSize (message) { return getSizeOrZero(key) + getSizeOrZero(value) + getHeadersSize(headers) } +function getAmqpMessageSize (message) { + const { headers, content } = message + return getSizeOrZero(content) + getHeadersSize(headers) +} + class TimeBuckets extends Map { forTime (time) { if (!this.has(time)) { @@ -98,7 +179,8 @@ class DataStreamsProcessor { env, tags, version, - service + service, + flushInterval } = {}) { this.writer = new DataStreamsWriter({ hostname, @@ -114,31 +196,45 @@ class DataStreamsProcessor { this.service = service || 'unnamed-nodejs-service' this.version = version || '' this.sequence = 0 + this.flushInterval = flushInterval + this._schemaSamplers = {} if (this.enabled) { - this.timer = setInterval(this.onInterval.bind(this), 10000) + this.timer = setInterval(this.onInterval.bind(this), flushInterval) this.timer.unref() } + process.once('beforeExit', () => this.onInterval()) } onInterval () { - const serialized = this._serializeBuckets() - if (!serialized) return + const { Stats } = this._serializeBuckets() + if (Stats.length === 0) return const payload = { Env: this.env, Service: this.service, - Stats: serialized, + Stats, TracerVersion: pkg.version, Version: this.version, - Lang: 'javascript' + Lang: 'javascript', + Tags: Object.entries(this.tags).map(([key, value]) => `${key}:${value}`) } this.writer.flush(payload) } + /** + * Given a timestamp in nanoseconds, compute and return the closest TimeBucket + * @param {number} timestamp + * @returns {StatsBucket} + */ + bucketFromTimestamp (timestamp) { + const bucketTime = Math.round(timestamp - (timestamp % this.bucketSizeNs)) + const bucket = this.buckets.forTime(bucketTime) + return bucket + } + recordCheckpoint (checkpoint, span = null) { if (!this.enabled) return - const bucketTime = Math.round(checkpoint.currentTimestamp - (checkpoint.currentTimestamp % this.bucketSizeNs)) - this.buckets.forTime(bucketTime) + this.bucketFromTimestamp(checkpoint.currentTimestamp) .forCheckpoint(checkpoint) .addLatencies(checkpoint) // set DSM pathway hash on span to enable related traces feature on DSM tab, convert from buffer to uint64 @@ -177,67 +273,131 @@ class DataStreamsProcessor { closestOppositeDirectionHash = parentHash closestOppositeDirectionEdgeStart = edgeStartNs } + log.debug( + () => `Setting DSM Checkpoint from extracted parent context with hash: ${parentHash} and edge tags: ${edgeTags}` + ) + } else { + log.debug(() => 'Setting DSM Checkpoint with empty parent context.') } const hash = computePathwayHash(this.service, this.env, edgeTags, parentHash) const edgeLatencyNs = nowNs - edgeStartNs const pathwayLatencyNs = nowNs - pathwayStartNs const dataStreamsContext = { - hash: hash, - edgeStartNs: edgeStartNs, - pathwayStartNs: pathwayStartNs, + hash, + edgeStartNs, + pathwayStartNs, previousDirection: direction, - closestOppositeDirectionHash: closestOppositeDirectionHash, - closestOppositeDirectionEdgeStart: closestOppositeDirectionEdgeStart + closestOppositeDirectionHash, + closestOppositeDirectionEdgeStart } if (direction === 'direction:out') { // Add the header for this now, as the callee doesn't have access to context when producing - payloadSize += getSizeOrZero(encodePathwayContext(dataStreamsContext)) - payloadSize += CONTEXT_PROPAGATION_KEY.length + // - 1 to account for extra byte for { + const ddInfoContinued = {} + DsmPathwayCodec.encode(dataStreamsContext, ddInfoContinued) + payloadSize += getSizeOrZero(JSON.stringify(ddInfoContinued)) - 1 } const checkpoint = { currentTimestamp: nowNs, - parentHash: parentHash, - hash: hash, - edgeTags: edgeTags, - edgeLatencyNs: edgeLatencyNs, - pathwayLatencyNs: pathwayLatencyNs, - payloadSize: payloadSize + parentHash, + hash, + edgeTags, + edgeLatencyNs, + pathwayLatencyNs, + payloadSize } this.recordCheckpoint(checkpoint, span) return dataStreamsContext } + recordOffset ({ timestamp, ...backlogData }) { + if (!this.enabled) return + return this.bucketFromTimestamp(timestamp) + .forBacklog(backlogData) + } + + setOffset (offsetObj) { + if (!this.enabled) return + const nowNs = Date.now() * 1e6 + const backlogData = { + ...offsetObj, + timestamp: nowNs + } + this.recordOffset(backlogData) + } + _serializeBuckets () { + // TimeBuckets const serializedBuckets = [] - for (const [ timeNs, bucket ] of this.buckets.entries()) { + for (const [timeNs, bucket] of this.buckets.entries()) { const points = [] - for (const stats of bucket.values()) { + // bucket: StatsBucket + // stats: StatsPoint + for (const stats of bucket._checkpoints.values()) { points.push(stats.encode()) } + const backlogs = [] + for (const backlog of bucket._backlogs.values()) { + backlogs.push(backlog.encode()) + } serializedBuckets.push({ Start: new Uint64(timeNs), Duration: new Uint64(this.bucketSizeNs), - Stats: points + Stats: points, + Backlogs: backlogs }) } this.buckets.clear() - return serializedBuckets + return { + Stats: serializedBuckets + } + } + + setUrl (url) { + this.writer.setUrl(url) + } + + trySampleSchema (topic) { + const nowMs = Date.now() + + if (!this._schemaSamplers[topic]) { + this._schemaSamplers[topic] = new SchemaSampler() + } + + const sampler = this._schemaSamplers[topic] + return sampler.trySample(nowMs) + } + + canSampleSchema (topic) { + const nowMs = Date.now() + + if (!this._schemaSamplers[topic]) { + this._schemaSamplers[topic] = new SchemaSampler() + } + + const sampler = this._schemaSamplers[topic] + return sampler.canSample(nowMs) + } + + getSchema (schemaName, iterator) { + return SchemaBuilder.getSchema(schemaName, iterator) } } module.exports = { - DataStreamsProcessor: DataStreamsProcessor, - StatsPoint: StatsPoint, - StatsBucket: StatsBucket, + DataStreamsProcessor, + StatsPoint, + StatsBucket, + Backlog, TimeBuckets, getMessageSize, getHeadersSize, getSizeOrZero, - ENTRY_PARENT_HASH, - CONTEXT_PROPAGATION_KEY + getAmqpMessageSize, + ENTRY_PARENT_HASH } diff --git a/packages/dd-trace/src/datastreams/schemas/schema.js b/packages/dd-trace/src/datastreams/schemas/schema.js new file mode 100644 index 00000000000..4378e37d080 --- /dev/null +++ b/packages/dd-trace/src/datastreams/schemas/schema.js @@ -0,0 +1,8 @@ +class Schema { + constructor (definition, id) { + this.definition = definition + this.id = id + } +} + +module.exports = { Schema } diff --git a/packages/dd-trace/src/datastreams/schemas/schema_builder.js b/packages/dd-trace/src/datastreams/schemas/schema_builder.js new file mode 100644 index 00000000000..092f5b45101 --- /dev/null +++ b/packages/dd-trace/src/datastreams/schemas/schema_builder.js @@ -0,0 +1,133 @@ +const LRUCache = require('lru-cache') +const { fnv64 } = require('../fnv') +const { Schema } = require('./schema') + +const maxDepth = 10 +const maxProperties = 1000 +const CACHE = new LRUCache({ max: 256 }) + +class SchemaBuilder { + constructor (iterator) { + this.schema = new OpenApiSchema() + this.iterator = iterator + this.properties = 0 + } + + static getCache () { + return CACHE + } + + static getSchemaDefinition (schema) { + const noNones = convertToJsonCompatible(schema) + const definition = jsonStringify(noNones) + const id = fnv64(Buffer.from(definition, 'utf-8')).toString() + return new Schema(definition, id) + } + + static getSchema (schemaName, iterator, builder) { + if (!CACHE.has(schemaName)) { + CACHE.set(schemaName, (builder ?? new SchemaBuilder(iterator)).build()) + } + return CACHE.get(schemaName) + } + + build () { + this.iterator.iterateOverSchema(this) + return this.schema + } + + addProperty (schemaName, fieldName, isArray, type, description, ref, format, enumValues) { + if (this.properties >= maxProperties) { + return false + } + this.properties += 1 + let property = new OpenApiSchema.PROPERTY(type, description, ref, format, enumValues, null) + if (isArray) { + property = new OpenApiSchema.PROPERTY('array', null, null, null, null, property) + } + this.schema.components.schemas[schemaName].properties[fieldName] = property + return true + } + + shouldExtractSchema (schemaName, depth) { + if (depth > maxDepth) { + return false + } + if (schemaName in this.schema.components.schemas) { + return false + } + this.schema.components.schemas[schemaName] = new OpenApiSchema.SCHEMA() + return true + } +} + +class OpenApiSchema { + constructor () { + this.openapi = '3.0.0' + this.components = new OpenApiComponents() + } +} + +OpenApiSchema.SCHEMA = class { + constructor () { + this.type = 'object' + this.properties = {} + } +} + +OpenApiSchema.PROPERTY = class { + constructor (type, description = null, ref = null, format = null, enumValues = null, items = null) { + this.type = type + this.description = description + this.$ref = ref + this.format = format + this.enum = enumValues + this.items = items + } +} + +class OpenApiComponents { + constructor () { + this.schemas = {} + } +} + +function convertToJsonCompatible (obj) { + if (Array.isArray(obj)) { + return obj.filter(item => item !== null).map(item => convertToJsonCompatible(item)) + } else if (obj && typeof obj === 'object') { + const jsonObj = {} + for (const [key, value] of Object.entries(obj)) { + if (value !== null) { + jsonObj[key] = convertToJsonCompatible(value) + } + } + return jsonObj + } + return obj +} + +function convertKey (key) { + if (key === 'enumValues') { + return 'enum' + } + return key +} + +function jsonStringify (obj, indent = 2) { + // made to stringify json exactly similar to python / java in order for hashing to be the same + const jsonString = JSON.stringify(obj, (_, value) => value, indent) + return jsonString.replace(/^ +/gm, ' ') // Replace leading spaces with single space + .replace(/\n/g, '') // Remove newlines + .replace(/{ /g, '{') // Remove space after '{' + .replace(/ }/g, '}') // Remove space before '}' + .replace(/\[ /g, '[') // Remove space after '[' + .replace(/ \]/g, ']') // Remove space before ']' +} + +module.exports = { + SchemaBuilder, + OpenApiSchema, + convertToJsonCompatible, + convertKey +} diff --git a/packages/dd-trace/src/datastreams/schemas/schema_sampler.js b/packages/dd-trace/src/datastreams/schemas/schema_sampler.js new file mode 100644 index 00000000000..903a4ea1dec --- /dev/null +++ b/packages/dd-trace/src/datastreams/schemas/schema_sampler.js @@ -0,0 +1,29 @@ +const SAMPLE_INTERVAL_MILLIS = 30 * 1000 + +class SchemaSampler { + constructor () { + this.weight = 0 + this.lastSampleMs = 0 + } + + trySample (currentTimeMs) { + if (currentTimeMs >= this.lastSampleMs + SAMPLE_INTERVAL_MILLIS) { + if (currentTimeMs >= this.lastSampleMs + SAMPLE_INTERVAL_MILLIS) { + this.lastSampleMs = currentTimeMs + const weight = this.weight + this.weight = 0 + return weight + } + } + return 0 + } + + canSample (currentTimeMs) { + this.weight += 1 + return currentTimeMs >= this.lastSampleMs + SAMPLE_INTERVAL_MILLIS + } +} + +module.exports = { + SchemaSampler +} diff --git a/packages/dd-trace/src/datastreams/writer.js b/packages/dd-trace/src/datastreams/writer.js index acc7a51a3b3..f8c9e021ecc 100644 --- a/packages/dd-trace/src/datastreams/writer.js +++ b/packages/dd-trace/src/datastreams/writer.js @@ -15,13 +15,10 @@ function makeRequest (data, url, cb) { 'Datadog-Meta-Tracer-Version': pkg.version, 'Content-Type': 'application/msgpack', 'Content-Encoding': 'gzip' - } + }, + url } - options.protocol = url.protocol - options.hostname = url.hostname - options.port = url.port - log.debug(() => `Request to the intake: ${JSON.stringify(options)}`) request(data, options, (err, res) => { @@ -59,6 +56,15 @@ class DataStreamsWriter { }) }) } + + setUrl (url) { + try { + url = new URL(url) + this._url = url + } catch (e) { + log.warn(e.stack) + } + } } module.exports = { diff --git a/packages/dd-trace/src/debugger/devtools_client/config.js b/packages/dd-trace/src/debugger/devtools_client/config.js new file mode 100644 index 00000000000..838a1a76cca --- /dev/null +++ b/packages/dd-trace/src/debugger/devtools_client/config.js @@ -0,0 +1,26 @@ +'use strict' + +const { workerData: { config: parentConfig, parentThreadId, configPort } } = require('node:worker_threads') +const { format } = require('node:url') +const log = require('../../log') + +const config = module.exports = { + runtimeId: parentConfig.tags['runtime-id'], + service: parentConfig.service, + commitSHA: parentConfig.commitSHA, + repositoryUrl: parentConfig.repositoryUrl, + parentThreadId +} + +updateUrl(parentConfig) + +configPort.on('message', updateUrl) +configPort.on('messageerror', (err) => log.error(err)) + +function updateUrl (updates) { + config.url = updates.url || format({ + protocol: 'http:', + hostname: updates.hostname || 'localhost', + port: updates.port + }) +} diff --git a/packages/dd-trace/src/debugger/devtools_client/index.js b/packages/dd-trace/src/debugger/devtools_client/index.js new file mode 100644 index 00000000000..4675b61d725 --- /dev/null +++ b/packages/dd-trace/src/debugger/devtools_client/index.js @@ -0,0 +1,112 @@ +'use strict' + +const { randomUUID } = require('crypto') +const { breakpoints } = require('./state') +const session = require('./session') +const { getLocalStateForCallFrame } = require('./snapshot') +const send = require('./send') +const { getScriptUrlFromId } = require('./state') +const { ackEmitting, ackError } = require('./status') +const { parentThreadId } = require('./config') +const log = require('../../log') +const { version } = require('../../../../../package.json') + +require('./remote_config') + +// There doesn't seem to be an official standard for the content of these fields, so we're just populating them with +// something that should be useful to a Node.js developer. +const threadId = parentThreadId === 0 ? `pid:${process.pid}` : `pid:${process.pid};tid:${parentThreadId}` +const threadName = parentThreadId === 0 ? 'MainThread' : `WorkerThread:${parentThreadId}` + +session.on('Debugger.paused', async ({ params }) => { + const start = process.hrtime.bigint() + const timestamp = Date.now() + + let captureSnapshotForProbe = null + let maxReferenceDepth, maxCollectionSize, maxLength + const probes = params.hitBreakpoints.map((id) => { + const probe = breakpoints.get(id) + if (probe.captureSnapshot) { + captureSnapshotForProbe = probe + maxReferenceDepth = highestOrUndefined(probe.capture.maxReferenceDepth, maxReferenceDepth) + maxCollectionSize = highestOrUndefined(probe.capture.maxCollectionSize, maxCollectionSize) + maxLength = highestOrUndefined(probe.capture.maxLength, maxLength) + } + return probe + }) + + let processLocalState + if (captureSnapshotForProbe !== null) { + try { + // TODO: Create unique states for each affected probe based on that probes unique `capture` settings (DEBUG-2863) + processLocalState = await getLocalStateForCallFrame( + params.callFrames[0], + { maxReferenceDepth, maxCollectionSize, maxLength } + ) + } catch (err) { + // TODO: This error is not tied to a specific probe, but to all probes with `captureSnapshot: true`. + // However, in 99,99% of cases, there will be just a single probe, so I guess this simplification is ok? + ackError(err, captureSnapshotForProbe) // TODO: Ok to continue after sending ackError? + } + } + + await session.post('Debugger.resume') + const diff = process.hrtime.bigint() - start // TODO: Recored as telemetry (DEBUG-2858) + + log.debug(`Finished processing breakpoints - main thread paused for: ${Number(diff) / 1000000} ms`) + + const logger = { + // We can safely use `location.file` from the first probe in the array, since all probes hit by `hitBreakpoints` + // must exist in the same file since the debugger can only pause the main thread in one location. + name: probes[0].location.file, // name of the class/type/file emitting the snapshot + method: params.callFrames[0].functionName, // name of the method/function emitting the snapshot + version, + thread_id: threadId, + thread_name: threadName + } + + const stack = params.callFrames.map((frame) => { + let fileName = getScriptUrlFromId(frame.location.scriptId) + if (fileName.startsWith('file://')) fileName = fileName.substr(7) // TODO: This might not be required + return { + fileName, + function: frame.functionName, + lineNumber: frame.location.lineNumber + 1, // Beware! lineNumber is zero-indexed + columnNumber: frame.location.columnNumber + 1 // Beware! columnNumber is zero-indexed + } + }) + + // TODO: Send multiple probes in one HTTP request as an array (DEBUG-2848) + for (const probe of probes) { + const snapshot = { + id: randomUUID(), + timestamp, + probe: { + id: probe.id, + version: probe.version, + location: probe.location + }, + stack, + language: 'javascript' + } + + if (probe.captureSnapshot) { + const state = processLocalState() + if (state) { + snapshot.captures = { + lines: { [probe.location.lines[0]]: { locals: state } } + } + } + } + + // TODO: Process template (DEBUG-2628) + send(probe.template, logger, snapshot, (err) => { + if (err) log.error(err) + else ackEmitting(probe) + }) + } +}) + +function highestOrUndefined (num, max) { + return num === undefined ? max : Math.max(num, max ?? 0) +} diff --git a/packages/dd-trace/src/debugger/devtools_client/inspector_promises_polyfill.js b/packages/dd-trace/src/debugger/devtools_client/inspector_promises_polyfill.js new file mode 100644 index 00000000000..bb4b0340be6 --- /dev/null +++ b/packages/dd-trace/src/debugger/devtools_client/inspector_promises_polyfill.js @@ -0,0 +1,23 @@ +'use strict' + +const { builtinModules } = require('node:module') + +if (builtinModules.includes('inspector/promises')) { + module.exports = require('node:inspector/promises') +} else { + const inspector = require('node:inspector') + const { promisify } = require('node:util') + + // The rest of the code in this file is lifted from: + // https://github.com/nodejs/node/blob/1d4d76ff3fb08f9a0c55a1d5530b46c4d5d550c7/lib/inspector/promises.js + class Session extends inspector.Session { + constructor () { super() } // eslint-disable-line no-useless-constructor + } + + Session.prototype.post = promisify(inspector.Session.prototype.post) + + module.exports = { + ...inspector, + Session + } +} diff --git a/packages/dd-trace/src/debugger/devtools_client/remote_config.js b/packages/dd-trace/src/debugger/devtools_client/remote_config.js new file mode 100644 index 00000000000..8a7d7386e33 --- /dev/null +++ b/packages/dd-trace/src/debugger/devtools_client/remote_config.js @@ -0,0 +1,164 @@ +'use strict' + +const { workerData: { rcPort } } = require('node:worker_threads') +const { findScriptFromPartialPath, probes, breakpoints } = require('./state') +const session = require('./session') +const { ackReceived, ackInstalled, ackError } = require('./status') +const log = require('../../log') + +let sessionStarted = false + +// Example log line probe (simplified): +// { +// id: '100c9a5c-45ad-49dc-818b-c570d31e11d1', +// version: 0, +// type: 'LOG_PROBE', +// where: { sourceFile: 'index.js', lines: ['25'] }, // only use first array element +// template: 'Hello World 2', +// segments: [...], +// captureSnapshot: true, +// capture: { maxReferenceDepth: 1 }, +// sampling: { snapshotsPerSecond: 1 }, +// evaluateAt: 'EXIT' // only used for method probes +// } +// +// Example log method probe (simplified): +// { +// id: 'd692ee6d-5734-4df7-9d86-e3bc6449cc8c', +// version: 0, +// type: 'LOG_PROBE', +// where: { typeName: 'index.js', methodName: 'handlerA' }, +// template: 'Executed index.js.handlerA, it took {@duration}ms', +// segments: [...], +// captureSnapshot: false, +// capture: { maxReferenceDepth: 3 }, +// sampling: { snapshotsPerSecond: 5000 }, +// evaluateAt: 'EXIT' // only used for method probes +// } +rcPort.on('message', async ({ action, conf: probe, ackId }) => { + try { + await processMsg(action, probe) + rcPort.postMessage({ ackId }) + } catch (err) { + rcPort.postMessage({ ackId, error: err }) + ackError(err, probe) + } +}) +rcPort.on('messageerror', (err) => log.error(err)) + +async function start () { + sessionStarted = true + return session.post('Debugger.enable') // return instead of await to reduce number of promises created +} + +async function stop () { + sessionStarted = false + return session.post('Debugger.disable') // return instead of await to reduce number of promises created +} + +async function processMsg (action, probe) { + log.debug(`Received request to ${action} ${probe.type} probe (id: ${probe.id}, version: ${probe.version})`) + + if (action !== 'unapply') ackReceived(probe) + + if (probe.type !== 'LOG_PROBE') { + throw new Error(`Unsupported probe type: ${probe.type} (id: ${probe.id}, version: ${probe.version})`) + } + if (!probe.where.sourceFile && !probe.where.lines) { + throw new Error( + // eslint-disable-next-line max-len + `Unsupported probe insertion point! Only line-based probes are supported (id: ${probe.id}, version: ${probe.version})` + ) + } + + // This lock is to ensure that we don't get the following race condition: + // + // When a breakpoint is being removed and there are no other breakpoints, we disable the debugger by calling + // `Debugger.disable` to free resources. However, if a new breakpoint is being added around the same time, we might + // have a race condition where the new breakpoint thinks that the debugger is already enabled because the removal of + // the other breakpoint hasn't had a chance to call `Debugger.disable` yet. Then once the code that's adding the new + // breakpoints tries to call `Debugger.setBreakpoint` it fails because in the meantime `Debugger.disable` was called. + // + // If the code is ever refactored to not tear down the debugger if there's no active breakpoints, we can safely remove + // this lock. + const release = await lock() + + try { + switch (action) { + case 'unapply': + await removeBreakpoint(probe) + break + case 'apply': + await addBreakpoint(probe) + break + case 'modify': + // TODO: Modify existing probe instead of removing it (DEBUG-2817) + await removeBreakpoint(probe) + await addBreakpoint(probe) + break + default: + throw new Error( + // eslint-disable-next-line max-len + `Cannot process probe ${probe.id} (version: ${probe.version}) - unknown remote configuration action: ${action}` + ) + } + } finally { + release() + } +} + +async function addBreakpoint (probe) { + if (!sessionStarted) await start() + + const file = probe.where.sourceFile + const line = Number(probe.where.lines[0]) // Tracer doesn't support multiple-line breakpoints + + // Optimize for sending data to /debugger/v1/input endpoint + probe.location = { file, lines: [String(line)] } + delete probe.where + + // TODO: Inbetween `await session.post('Debugger.enable')` and here, the scripts are parsed and cached. + // Maybe there's a race condition here or maybe we're guraenteed that `await session.post('Debugger.enable')` will + // not continue untill all scripts have been parsed? + const script = findScriptFromPartialPath(file) + if (!script) throw new Error(`No loaded script found for ${file} (probe: ${probe.id}, version: ${probe.version})`) + const [path, scriptId] = script + + log.debug(`Adding breakpoint at ${path}:${line} (probe: ${probe.id}, version: ${probe.version})`) + + const { breakpointId } = await session.post('Debugger.setBreakpoint', { + location: { + scriptId, + lineNumber: line - 1 // Beware! lineNumber is zero-indexed + } + }) + + probes.set(probe.id, breakpointId) + breakpoints.set(breakpointId, probe) + + ackInstalled(probe) +} + +async function removeBreakpoint ({ id }) { + if (!sessionStarted) { + // We should not get in this state, but abort if we do, so the code doesn't fail unexpected + throw Error(`Cannot remove probe ${id}: Debugger not started`) + } + if (!probes.has(id)) { + throw Error(`Unknown probe id: ${id}`) + } + + const breakpointId = probes.get(id) + await session.post('Debugger.removeBreakpoint', { breakpointId }) + probes.delete(id) + breakpoints.delete(breakpointId) + + if (breakpoints.size === 0) await stop() +} + +async function lock () { + if (lock.p) await lock.p + let resolve + lock.p = new Promise((_resolve) => { resolve = _resolve }).then(() => { lock.p = null }) + return resolve +} diff --git a/packages/dd-trace/src/debugger/devtools_client/send.js b/packages/dd-trace/src/debugger/devtools_client/send.js new file mode 100644 index 00000000000..593c3ea235d --- /dev/null +++ b/packages/dd-trace/src/debugger/devtools_client/send.js @@ -0,0 +1,41 @@ +'use strict' + +const { hostname: getHostname } = require('os') +const { stringify } = require('querystring') + +const config = require('./config') +const request = require('../../exporters/common/request') +const { GIT_COMMIT_SHA, GIT_REPOSITORY_URL } = require('../../plugins/util/tags') + +module.exports = send + +const ddsource = 'dd_debugger' +const hostname = getHostname() +const service = config.service + +const ddtags = [ + [GIT_COMMIT_SHA, config.commitSHA], + [GIT_REPOSITORY_URL, config.repositoryUrl] +].map((pair) => pair.join(':')).join(',') + +const path = `/debugger/v1/input?${stringify({ ddtags })}` + +function send (message, logger, snapshot, cb) { + const opts = { + method: 'POST', + url: config.url, + path, + headers: { 'Content-Type': 'application/json; charset=utf-8' } + } + + const payload = { + ddsource, + hostname, + service, + message, + logger, + 'debugger.snapshot': snapshot + } + + request(JSON.stringify(payload), opts, cb) +} diff --git a/packages/dd-trace/src/debugger/devtools_client/session.js b/packages/dd-trace/src/debugger/devtools_client/session.js new file mode 100644 index 00000000000..3cda2322b36 --- /dev/null +++ b/packages/dd-trace/src/debugger/devtools_client/session.js @@ -0,0 +1,7 @@ +'use strict' + +const inspector = require('./inspector_promises_polyfill') + +const session = module.exports = new inspector.Session() + +session.connectToMainThread() diff --git a/packages/dd-trace/src/debugger/devtools_client/snapshot/collector.js b/packages/dd-trace/src/debugger/devtools_client/snapshot/collector.js new file mode 100644 index 00000000000..14f6db9727f --- /dev/null +++ b/packages/dd-trace/src/debugger/devtools_client/snapshot/collector.js @@ -0,0 +1,182 @@ +'use strict' + +const { collectionSizeSym } = require('./symbols') +const session = require('../session') + +const LEAF_SUBTYPES = new Set(['date', 'regexp']) +const ITERABLE_SUBTYPES = new Set(['map', 'set', 'weakmap', 'weakset']) + +module.exports = { + getRuntimeObject: getObject +} + +// TODO: Can we speed up thread pause time by calling mutiple Runtime.getProperties in parallel when possible? +// The most simple solution would be to swich from an async/await approach to a callback based approach, in which case +// each lookup will just finish in its own time and traverse the child nodes when the event loop allows it. +// Alternatively, use `Promise.all` or something like that, but the code would probably be more complex. + +async function getObject (objectId, opts, depth = 0, collection = false) { + const { result, privateProperties } = await session.post('Runtime.getProperties', { + objectId, + ownProperties: true // exclude inherited properties + }) + + if (collection) { + // Trim the collection if it's too large. + // Collections doesn't contain private properties, so the code in this block doesn't have to deal with it. + removeNonEnumerableProperties(result) // remove the `length` property + const size = result.length + if (size > opts.maxCollectionSize) { + result.splice(opts.maxCollectionSize) + result[collectionSizeSym] = size + } + } else if (privateProperties) { + result.push(...privateProperties) + } + + return traverseGetPropertiesResult(result, opts, depth) +} + +async function traverseGetPropertiesResult (props, opts, depth) { + // TODO: Decide if we should filter out non-enumerable properties or not: + // props = props.filter((e) => e.enumerable) + + if (depth >= opts.maxReferenceDepth) return props + + for (const prop of props) { + if (prop.value === undefined) continue + const { value: { type, objectId, subtype } } = prop + if (type === 'object') { + if (objectId === undefined) continue // if `subtype` is "null" + if (LEAF_SUBTYPES.has(subtype)) continue // don't waste time with these subtypes + prop.value.properties = await getObjectProperties(subtype, objectId, opts, depth) + } else if (type === 'function') { + prop.value.properties = await getFunctionProperties(objectId, opts, depth + 1) + } + } + + return props +} + +async function getObjectProperties (subtype, objectId, opts, depth) { + if (ITERABLE_SUBTYPES.has(subtype)) { + return getIterable(objectId, opts, depth) + } else if (subtype === 'promise') { + return getInternalProperties(objectId, opts, depth) + } else if (subtype === 'proxy') { + return getProxy(objectId, opts, depth) + } else if (subtype === 'arraybuffer') { + return getArrayBuffer(objectId, opts, depth) + } else { + return getObject(objectId, opts, depth + 1, subtype === 'array' || subtype === 'typedarray') + } +} + +// TODO: The following extra information from `internalProperties` might be relevant to include for functions: +// - Bound function: `[[TargetFunction]]`, `[[BoundThis]]` and `[[BoundArgs]]` +// - Non-bound function: `[[FunctionLocation]]`, and `[[Scopes]]` +async function getFunctionProperties (objectId, opts, depth) { + let { result } = await session.post('Runtime.getProperties', { + objectId, + ownProperties: true // exclude inherited properties + }) + + // For legacy reasons (I assume) functions has a `prototype` property besides the internal `[[Prototype]]` + result = result.filter(({ name }) => name !== 'prototype') + + return traverseGetPropertiesResult(result, opts, depth) +} + +async function getIterable (objectId, opts, depth) { + // TODO: If the iterable has any properties defined on the object directly, instead of in its collection, they will + // exist in the return value below in the `result` property. We currently do not collect these. + const { internalProperties } = await session.post('Runtime.getProperties', { + objectId, + ownProperties: true // exclude inherited properties + }) + + let entry = internalProperties[1] + if (entry.name !== '[[Entries]]') { + // Currently `[[Entries]]` is the last of 2 elements, but in case this ever changes, fall back to searching + entry = internalProperties.findLast(({ name }) => name === '[[Entries]]') + } + + // Skip the `[[Entries]]` level and go directly to the content of the iterable + const { result } = await session.post('Runtime.getProperties', { + objectId: entry.value.objectId, + ownProperties: true // exclude inherited properties + }) + + removeNonEnumerableProperties(result) // remove the `length` property + const size = result.length + if (size > opts.maxCollectionSize) { + result.splice(opts.maxCollectionSize) + result[collectionSizeSym] = size + } + + return traverseGetPropertiesResult(result, opts, depth) +} + +async function getInternalProperties (objectId, opts, depth) { + const { internalProperties } = await session.post('Runtime.getProperties', { + objectId, + ownProperties: true // exclude inherited properties + }) + + // We want all internal properties except the prototype + const props = internalProperties.filter(({ name }) => name !== '[[Prototype]]') + + return traverseGetPropertiesResult(props, opts, depth) +} + +async function getProxy (objectId, opts, depth) { + const { internalProperties } = await session.post('Runtime.getProperties', { + objectId, + ownProperties: true // exclude inherited properties + }) + + // TODO: If we do not skip the proxy wrapper, we can add a `revoked` boolean + let entry = internalProperties[1] + if (entry.name !== '[[Target]]') { + // Currently `[[Target]]` is the last of 2 elements, but in case this ever changes, fall back to searching + entry = internalProperties.findLast(({ name }) => name === '[[Target]]') + } + + // Skip the `[[Target]]` level and go directly to the target of the Proxy + const { result } = await session.post('Runtime.getProperties', { + objectId: entry.value.objectId, + ownProperties: true // exclude inherited properties + }) + + return traverseGetPropertiesResult(result, opts, depth) +} + +// Support for ArrayBuffer is a bit trickly because the internal structure stored in `internalProperties` is not +// documented and is not straight forward. E.g. ArrayBuffer(3) will internally contain both Int8Array(3) and +// UInt8Array(3), whereas ArrayBuffer(8) internally contains both Int8Array(8), Uint8Array(8), Int16Array(4), and +// Int32Array(2) - all representing the same data in different ways. +async function getArrayBuffer (objectId, opts, depth) { + const { internalProperties } = await session.post('Runtime.getProperties', { + objectId, + ownProperties: true // exclude inherited properties + }) + + // Use Uint8 to make it easy to convert to a string later. + const entry = internalProperties.find(({ name }) => name === '[[Uint8Array]]') + + // Skip the `[[Uint8Array]]` level and go directly to the content of the ArrayBuffer + const { result } = await session.post('Runtime.getProperties', { + objectId: entry.value.objectId, + ownProperties: true // exclude inherited properties + }) + + return traverseGetPropertiesResult(result, opts, depth) +} + +function removeNonEnumerableProperties (props) { + for (let i = 0; i < props.length; i++) { + if (props[i].enumerable === false) { + props.splice(i--, 1) + } + } +} diff --git a/packages/dd-trace/src/debugger/devtools_client/snapshot/index.js b/packages/dd-trace/src/debugger/devtools_client/snapshot/index.js new file mode 100644 index 00000000000..cca7aa43bae --- /dev/null +++ b/packages/dd-trace/src/debugger/devtools_client/snapshot/index.js @@ -0,0 +1,35 @@ +'use strict' + +const { getRuntimeObject } = require('./collector') +const { processRawState } = require('./processor') + +const DEFAULT_MAX_REFERENCE_DEPTH = 3 +const DEFAULT_MAX_COLLECTION_SIZE = 100 +const DEFAULT_MAX_LENGTH = 255 + +module.exports = { + getLocalStateForCallFrame +} + +async function getLocalStateForCallFrame ( + callFrame, + { + maxReferenceDepth = DEFAULT_MAX_REFERENCE_DEPTH, + maxCollectionSize = DEFAULT_MAX_COLLECTION_SIZE, + maxLength = DEFAULT_MAX_LENGTH + } = {} +) { + const rawState = [] + let processedState = null + + for (const scope of callFrame.scopeChain) { + if (scope.type === 'global') continue // The global scope is too noisy + rawState.push(...await getRuntimeObject(scope.object.objectId, { maxReferenceDepth, maxCollectionSize })) + } + + // Deplay calling `processRawState` so the caller gets a chance to resume the main thread before processing `rawState` + return () => { + processedState = processedState ?? processRawState(rawState, maxLength) + return processedState + } +} diff --git a/packages/dd-trace/src/debugger/devtools_client/snapshot/processor.js b/packages/dd-trace/src/debugger/devtools_client/snapshot/processor.js new file mode 100644 index 00000000000..9ded1477441 --- /dev/null +++ b/packages/dd-trace/src/debugger/devtools_client/snapshot/processor.js @@ -0,0 +1,241 @@ +'use strict' + +const { collectionSizeSym } = require('./symbols') + +module.exports = { + processRawState: processProperties +} + +// Matches classes in source code, no matter how it's written: +// - Named: class MyClass {} +// - Anonymous: class {} +// - Named, with odd whitespace: class\n\t MyClass\n{} +// - Anonymous, with odd whitespace: class\n{} +const CLASS_REGEX = /^class\s([^{]*)/ + +function processProperties (props, maxLength) { + const result = {} + + for (const prop of props) { + // TODO: Hack to avoid periods in keys, as EVP doesn't support that. A better solution can be implemented later + result[prop.name.replaceAll('.', '_')] = getPropertyValue(prop, maxLength) + } + + return result +} + +function getPropertyValue (prop, maxLength) { + // Special case for getters and setters which does not have a value property + if ('get' in prop) { + const hasGet = prop.get.type !== 'undefined' + const hasSet = prop.set.type !== 'undefined' + if (hasGet && hasSet) return { type: 'getter/setter' } + if (hasGet) return { type: 'getter' } + if (hasSet) return { type: 'setter' } + } + + switch (prop.value?.type) { + case 'object': + return getObjectValue(prop.value, maxLength) + case 'function': + return toFunctionOrClass(prop.value, maxLength) + case undefined: // TODO: Add test for when a prop has no value. I think it's if it's defined after the breakpoint? + case 'undefined': + return { type: 'undefined' } + case 'string': + return toString(prop.value.value, maxLength) + case 'number': + return { type: 'number', value: prop.value.description } // use `descripton` to get it as string + case 'boolean': + return { type: 'boolean', value: prop.value.value === true ? 'true' : 'false' } + case 'symbol': + return { type: 'symbol', value: prop.value.description } + case 'bigint': + return { type: 'bigint', value: prop.value.description.slice(0, -1) } // remove trailing `n` + default: + // As of this writing, the Chrome DevTools Protocol doesn't allow any other types than the ones listed above, but + // in the future new ones might be added. + return { type: prop.value.type, notCapturedReason: 'Unsupported property type' } + } +} + +function getObjectValue (obj, maxLength) { + switch (obj.subtype) { + case undefined: + return toObject(obj.className, obj.properties, maxLength) + case 'array': + return toArray(obj.className, obj.properties, maxLength) + case 'null': + return { type: 'null', isNull: true } + // case 'node': // TODO: What does this subtype represent? + case 'regexp': + return { type: obj.className, value: obj.description } + case 'date': + // TODO: This looses millisecond resolution, as that's not retained in the `.toString()` representation contained + // in the `description` field. Unfortunately that's all we get from the Chrome DevTools Protocol. + return { type: obj.className, value: `${new Date(obj.description).toISOString().slice(0, -5)}Z` } + case 'map': + return toMap(obj.className, obj.properties, maxLength) + case 'set': + return toSet(obj.className, obj.properties, maxLength) + case 'weakmap': + return toMap(obj.className, obj.properties, maxLength) + case 'weakset': + return toSet(obj.className, obj.properties, maxLength) + // case 'iterator': // TODO: I've not been able to trigger this subtype + case 'generator': + // Use `subtype` instead of `className` to make it obvious it's a generator + return toObject(obj.subtype, obj.properties, maxLength) + case 'error': + // TODO: Convert stack trace to array to avoid string trunctation or disable truncation in this case? + return toObject(obj.className, obj.properties, maxLength) + case 'proxy': + // Use `desciption` instead of `className` as the `type` to get type of target object (`Proxy(Error)` vs `proxy`) + return toObject(obj.description, obj.properties, maxLength) + case 'promise': + return toObject(obj.className, obj.properties, maxLength) + case 'typedarray': + return toArray(obj.className, obj.properties, maxLength) + case 'arraybuffer': + return toArrayBuffer(obj.className, obj.properties, maxLength) + // case 'dataview': // TODO: Looks like the internal ArrayBuffer is only accessible via the `buffer` getter + // case 'webassemblymemory': // TODO: Looks like the internal ArrayBuffer is only accessible via the `buffer` getter + // case 'wasmvalue': // TODO: I've not been able to trigger this subtype + default: + // As of this writing, the Chrome DevTools Protocol doesn't allow any other subtypes than the ones listed above, + // but in the future new ones might be added. + return { type: obj.subtype, notCapturedReason: 'Unsupported object type' } + } +} + +function toFunctionOrClass (value, maxLength) { + const classMatch = value.description.match(CLASS_REGEX) + + if (classMatch === null) { + // This is a function + // TODO: Would it make sense to detect if it's an arrow function or not? + return toObject(value.className, value.properties, maxLength) + } else { + // This is a class + const className = classMatch[1].trim() + return { type: className ? `class ${className}` : 'class' } + } +} + +function toString (str, maxLength) { + const size = str.length + + if (size <= maxLength) { + return { type: 'string', value: str } + } + + return { + type: 'string', + value: str.substr(0, maxLength), + truncated: true, + size + } +} + +function toObject (type, props, maxLength) { + if (props === undefined) return notCapturedDepth(type) + return { type, fields: processProperties(props, maxLength) } +} + +function toArray (type, elements, maxLength) { + if (elements === undefined) return notCapturedDepth(type) + + // Perf: Create array of expected size in advance (expect that it contains only one non-enumrable element) + const result = { type, elements: new Array(elements.length) } + + setNotCaptureReasonOnCollection(result, elements) + + let i = 0 + for (const elm of elements) { + result.elements[i++] = getPropertyValue(elm, maxLength) + } + + return result +} + +function toMap (type, pairs, maxLength) { + if (pairs === undefined) return notCapturedDepth(type) + + // Perf: Create array of expected size in advance + const result = { type, entries: new Array(pairs.length) } + + setNotCaptureReasonOnCollection(result, pairs) + + let i = 0 + for (const pair of pairs) { + // The following code is based on assumptions made when researching the output of the Chrome DevTools Protocol. + // There doesn't seem to be any documentation to back it up: + // + // `pair.value` is a special wrapper-object with subtype `internal#entry`. This can be skipped and we can go + // directly to its children, of which there will always be exactly two, the first containing the key, and the + // second containing the value of this entry of the Map. + const key = getPropertyValue(pair.value.properties[0], maxLength) + const val = getPropertyValue(pair.value.properties[1], maxLength) + result.entries[i++] = [key, val] + } + + return result +} + +function toSet (type, values, maxLength) { + if (values === undefined) return notCapturedDepth(type) + + // Perf: Create array of expected size in advance (expect that it contains only one non-enumrable element) + const result = { type, elements: new Array(values.length) } + + setNotCaptureReasonOnCollection(result, values) + + let i = 0 + for (const value of values) { + // The following code is based on assumptions made when researching the output of the Chrome DevTools Protocol. + // There doesn't seem to be any documentation to back it up: + // + // `value.value` is a special wrapper-object with subtype `internal#entry`. This can be skipped and we can go + // directly to its children, of which there will always be exactly one, which contain the actual value in this entry + // of the Set. + result.elements[i++] = getPropertyValue(value.value.properties[0], maxLength) + } + + return result +} + +function toArrayBuffer (type, bytes, maxLength) { + if (bytes === undefined) return notCapturedDepth(type) + + const size = bytes.length + + if (size > maxLength) { + return { + type, + value: arrayBufferToString(bytes, maxLength), + truncated: true, + size: bytes.length + } + } else { + return { type, value: arrayBufferToString(bytes, size) } + } +} + +function arrayBufferToString (bytes, size) { + const buf = Buffer.allocUnsafe(size) + for (let i = 0; i < size; i++) { + buf[i] = bytes[i].value.value + } + return buf.toString() +} + +function setNotCaptureReasonOnCollection (result, collection) { + if (collectionSizeSym in collection) { + result.notCapturedReason = 'collectionSize' + result.size = collection[collectionSizeSym] + } +} + +function notCapturedDepth (type) { + return { type, notCapturedReason: 'depth' } +} diff --git a/packages/dd-trace/src/debugger/devtools_client/snapshot/symbols.js b/packages/dd-trace/src/debugger/devtools_client/snapshot/symbols.js new file mode 100644 index 00000000000..99efc36e5f6 --- /dev/null +++ b/packages/dd-trace/src/debugger/devtools_client/snapshot/symbols.js @@ -0,0 +1,5 @@ +'use stict' + +module.exports = { + collectionSizeSym: Symbol('datadog.collectionSize') +} diff --git a/packages/dd-trace/src/debugger/devtools_client/state.js b/packages/dd-trace/src/debugger/devtools_client/state.js new file mode 100644 index 00000000000..8be9c808369 --- /dev/null +++ b/packages/dd-trace/src/debugger/devtools_client/state.js @@ -0,0 +1,53 @@ +'use strict' + +const session = require('./session') + +const scriptIds = [] +const scriptUrls = new Map() + +module.exports = { + probes: new Map(), + breakpoints: new Map(), + + /** + * Find the matching script that can be inspected based on a partial path. + * + * Algorithm: Find the sortest url that ends in the requested path. + * + * Will identify the correct script as long as Node.js doesn't load a module from a `node_modules` folder outside the + * project root. If so, there's a risk that this path is shorter than the expected path inside the project root. + * Example of mismatch where path = `index.js`: + * + * Expected match: /www/code/my-projects/demo-project1/index.js + * Actual shorter match: /www/node_modules/dd-trace/index.js + * + * To fix this, specify a more unique file path, e.g `demo-project1/index.js` instead of `index.js` + * + * @param {string} path + * @returns {[string, string] | undefined} + */ + findScriptFromPartialPath (path) { + return scriptIds + .filter(([url]) => url.endsWith(path)) + .sort(([a], [b]) => a.length - b.length)[0] + }, + + getScriptUrlFromId (id) { + return scriptUrls.get(id) + } +} + +// Known params.url protocols: +// - `node:` - Ignored, as we don't want to instrument Node.js internals +// - `wasm:` - Ignored, as we don't support instrumenting WebAssembly +// - `file:` - Regular on-disk file +// Unknown params.url values: +// - `structured-stack` - Not sure what this is, but should just be ignored +// - `` - Not sure what this is, but should just be ignored +// TODO: Event fired for all files, every time debugger is enabled. So when we disable it, we need to reset the state +session.on('Debugger.scriptParsed', ({ params }) => { + scriptUrls.set(params.scriptId, params.url) + if (params.url.startsWith('file:')) { + scriptIds.push([params.url, params.scriptId]) + } +}) diff --git a/packages/dd-trace/src/debugger/devtools_client/status.js b/packages/dd-trace/src/debugger/devtools_client/status.js new file mode 100644 index 00000000000..e4ba10d8c55 --- /dev/null +++ b/packages/dd-trace/src/debugger/devtools_client/status.js @@ -0,0 +1,109 @@ +'use strict' + +const LRUCache = require('lru-cache') +const config = require('./config') +const request = require('../../exporters/common/request') +const FormData = require('../../exporters/common/form-data') +const log = require('../../log') + +module.exports = { + ackReceived, + ackInstalled, + ackEmitting, + ackError +} + +const ddsource = 'dd_debugger' +const service = config.service +const runtimeId = config.runtimeId + +const cache = new LRUCache({ + ttl: 1000 * 60 * 60, // 1 hour + // Unfortunate requirement when using LRUCache: + // It will emit a warning unless `ttlAutopurge`, `max`, or `maxSize` is set when using `ttl`. + // TODO: Consider alternative as this is NOT performant :( + ttlAutopurge: true +}) + +const STATUSES = { + RECEIVED: 'RECEIVED', + INSTALLED: 'INSTALLED', + EMITTING: 'EMITTING', + ERROR: 'ERROR', + BLOCKED: 'BLOCKED' // TODO: Implement once support for allow list, deny list or max probe limit has been added +} + +function ackReceived ({ id: probeId, version }) { + onlyUniqueUpdates( + STATUSES.RECEIVED, probeId, version, + () => send(statusPayload(probeId, version, STATUSES.RECEIVED)) + ) +} + +function ackInstalled ({ id: probeId, version }) { + onlyUniqueUpdates( + STATUSES.INSTALLED, probeId, version, + () => send(statusPayload(probeId, version, STATUSES.INSTALLED)) + ) +} + +function ackEmitting ({ id: probeId, version }) { + onlyUniqueUpdates( + STATUSES.EMITTING, probeId, version, + () => send(statusPayload(probeId, version, STATUSES.EMITTING)) + ) +} + +function ackError (err, { id: probeId, version }) { + log.error(err) + + onlyUniqueUpdates(STATUSES.ERROR, probeId, version, () => { + const payload = statusPayload(probeId, version, STATUSES.ERROR) + + payload.debugger.diagnostics.exception = { + type: err.code, + message: err.message, + stacktrace: err.stack + } + + send(payload) + }) +} + +function send (payload) { + const form = new FormData() + + form.append( + 'event', + JSON.stringify(payload), + { filename: 'event.json', contentType: 'application/json; charset=utf-8' } + ) + + const options = { + method: 'POST', + url: config.url, + path: '/debugger/v1/diagnostics', + headers: form.getHeaders() + } + + request(form, options, (err) => { + if (err) log.error(err) + }) +} + +function statusPayload (probeId, version, status) { + return { + ddsource, + service, + debugger: { + diagnostics: { probeId, runtimeId, version, status } + } + } +} + +function onlyUniqueUpdates (type, id, version, fn) { + const key = `${type}-${id}-${version}` + if (cache.has(key)) return + fn() + cache.set(key) +} diff --git a/packages/dd-trace/src/debugger/index.js b/packages/dd-trace/src/debugger/index.js new file mode 100644 index 00000000000..3638119c6f1 --- /dev/null +++ b/packages/dd-trace/src/debugger/index.js @@ -0,0 +1,99 @@ +'use strict' + +const { join } = require('path') +const { Worker, MessageChannel, threadId: parentThreadId } = require('worker_threads') +const log = require('../log') + +let worker = null +let configChannel = null +let ackId = 0 + +const { NODE_OPTIONS, ...env } = process.env + +module.exports = { + start, + configure +} + +function start (config, rc) { + if (worker !== null) return + + log.debug('Starting Dynamic Instrumentation client...') + + const rcAckCallbacks = new Map() + const rcChannel = new MessageChannel() + configChannel = new MessageChannel() + + rc.setProductHandler('LIVE_DEBUGGING', (action, conf, id, ack) => { + rcAckCallbacks.set(++ackId, ack) + rcChannel.port2.postMessage({ action, conf, ackId }) + }) + + rcChannel.port2.on('message', ({ ackId, error }) => { + const ack = rcAckCallbacks.get(ackId) + if (ack === undefined) { + // This should never happen, but just in case something changes in the future, we should guard against it + log.error(`Received an unknown ackId: ${ackId}`) + if (error) log.error(error) + return + } + ack(error) + rcAckCallbacks.delete(ackId) + }) + rcChannel.port2.on('messageerror', (err) => log.error(err)) + + worker = new Worker( + join(__dirname, 'devtools_client', 'index.js'), + { + execArgv: [], // Avoid worker thread inheriting the `-r` command line argument + env, // Avoid worker thread inheriting the `NODE_OPTIONS` environment variable (in case it contains `-r`) + workerData: { + config: serializableConfig(config), + parentThreadId, + rcPort: rcChannel.port1, + configPort: configChannel.port1 + }, + transferList: [rcChannel.port1, configChannel.port1] + } + ) + + worker.unref() + + worker.on('online', () => { + log.debug(`Dynamic Instrumentation worker thread started successfully (thread id: ${worker.threadId})`) + }) + + worker.on('error', (err) => log.error(err)) + worker.on('messageerror', (err) => log.error(err)) + + worker.on('exit', (code) => { + const error = new Error(`Dynamic Instrumentation worker thread exited unexpectedly with code ${code}`) + + log.error(error) + + // Be nice, clean up now that the worker thread encounted an issue and we can't continue + rc.removeProductHandler('LIVE_DEBUGGING') + worker.removeAllListeners() + configChannel = null + for (const ackId of rcAckCallbacks.keys()) { + rcAckCallbacks.get(ackId)(error) + rcAckCallbacks.delete(ackId) + } + }) +} + +function configure (config) { + if (configChannel === null) return + configChannel.port2.postMessage(serializableConfig(config)) +} + +// TODO: Refactor the Config class so it never produces any config objects that are incompatible with MessageChannel +function serializableConfig (config) { + // URL objects cannot be serialized over the MessageChannel, so we need to convert them to strings first + if (config.url instanceof URL) { + config = { ...config } + config.url = config.url.toString() + } + + return config +} diff --git a/packages/dd-trace/src/dogstatsd.js b/packages/dd-trace/src/dogstatsd.js index 65a1dd618d7..ba84de71341 100644 --- a/packages/dd-trace/src/dogstatsd.js +++ b/packages/dd-trace/src/dogstatsd.js @@ -12,6 +12,7 @@ const MAX_BUFFER_SIZE = 1024 // limit from the agent const TYPE_COUNTER = 'c' const TYPE_GAUGE = 'g' const TYPE_DISTRIBUTION = 'd' +const TYPE_HISTOGRAM = 'h' class DogStatsDClient { constructor (options = {}) { @@ -46,6 +47,10 @@ class DogStatsDClient { this._add(stat, value, TYPE_DISTRIBUTION, tags) } + histogram (stat, value, tags) { + this._add(stat, value, TYPE_HISTOGRAM, tags) + } + flush () { const queue = this._enqueue() @@ -67,16 +72,14 @@ class DogStatsDClient { request(buffer, this._httpOptions, (err) => { if (err) { log.error('HTTP error from agent: ' + err.stack) - if (err.status) { + if (err.status === 404) { // Inside this if-block, we have connectivity to the agent, but // we're not getting a 200 from the proxy endpoint. If it's a 404, // then we know we'll never have the endpoint, so just clear out the // options. Either way, we can give UDP a try. - if (err.status === 404) { - this._httpOptions = null - } - this._sendUdp(queue) + this._httpOptions = null } + this._sendUdp(queue) } }) } @@ -182,16 +185,6 @@ class DogStatsDClient { } } -class NoopDogStatsDClient { - gauge () { } - - increment () { } - - distribution () { } - - flush () { } -} - // This is a simplified user-facing proxy to the underlying DogStatsDClient instance class CustomMetrics { constructor (config) { @@ -231,6 +224,14 @@ class CustomMetrics { ) } + histogram (stat, value, tags) { + return this.dogstatsd.histogram( + stat, + value, + CustomMetrics.tagTranslator(tags) + ) + } + flush () { return this.dogstatsd.flush() } @@ -254,6 +255,5 @@ class CustomMetrics { module.exports = { DogStatsDClient, - NoopDogStatsDClient, CustomMetrics } diff --git a/packages/dd-trace/src/encode/0.4.js b/packages/dd-trace/src/encode/0.4.js index 7cf1ee0b2ef..02d96cb8a26 100644 --- a/packages/dd-trace/src/encode/0.4.js +++ b/packages/dd-trace/src/encode/0.4.js @@ -83,13 +83,17 @@ class AgentEncoder { span = formatSpan(span) bytes.reserve(1) - if (span.type) { + if (span.type && span.meta_struct) { + bytes.buffer[bytes.length++] = 0x8d + } else if (span.type || span.meta_struct) { bytes.buffer[bytes.length++] = 0x8c + } else { + bytes.buffer[bytes.length++] = 0x8b + } + if (span.type) { this._encodeString(bytes, 'type') this._encodeString(bytes, span.type) - } else { - bytes.buffer[bytes.length++] = 0x8b } this._encodeString(bytes, 'trace_id') @@ -114,6 +118,10 @@ class AgentEncoder { this._encodeMap(bytes, span.meta) this._encodeString(bytes, 'metrics') this._encodeMap(bytes, span.metrics) + if (span.meta_struct) { + this._encodeString(bytes, 'meta_struct') + this._encodeMetaStruct(bytes, span.meta_struct) + } } } @@ -263,6 +271,84 @@ class AgentEncoder { } } + _encodeMetaStruct (bytes, value) { + const keys = Array.isArray(value) ? [] : Object.keys(value) + const validKeys = keys.filter(key => { + const v = value[key] + return typeof v === 'string' || + typeof v === 'number' || + (v !== null && typeof v === 'object') + }) + + this._encodeMapPrefix(bytes, validKeys.length) + + for (const key of validKeys) { + const v = value[key] + this._encodeString(bytes, key) + this._encodeObjectAsByteArray(bytes, v) + } + } + + _encodeObjectAsByteArray (bytes, value) { + const prefixLength = 5 + const offset = bytes.length + + bytes.reserve(prefixLength) + bytes.length += prefixLength + + this._encodeObject(bytes, value) + + // we should do it after encoding the object to know the real length + const length = bytes.length - offset - prefixLength + bytes.buffer[offset] = 0xc6 + bytes.buffer[offset + 1] = length >> 24 + bytes.buffer[offset + 2] = length >> 16 + bytes.buffer[offset + 3] = length >> 8 + bytes.buffer[offset + 4] = length + } + + _encodeObject (bytes, value, circularReferencesDetector = new Set()) { + circularReferencesDetector.add(value) + if (Array.isArray(value)) { + this._encodeObjectAsArray(bytes, value, circularReferencesDetector) + } else if (value !== null && typeof value === 'object') { + this._encodeObjectAsMap(bytes, value, circularReferencesDetector) + } else if (typeof value === 'string' || typeof value === 'number') { + this._encodeValue(bytes, value) + } + } + + _encodeObjectAsMap (bytes, value, circularReferencesDetector) { + const keys = Object.keys(value) + const validKeys = keys.filter(key => { + const v = value[key] + return typeof v === 'string' || + typeof v === 'number' || + (v !== null && typeof v === 'object' && !circularReferencesDetector.has(v)) + }) + + this._encodeMapPrefix(bytes, validKeys.length) + + for (const key of validKeys) { + const v = value[key] + this._encodeString(bytes, key) + this._encodeObject(bytes, v, circularReferencesDetector) + } + } + + _encodeObjectAsArray (bytes, value, circularReferencesDetector) { + const validValue = value.filter(item => + typeof item === 'string' || + typeof item === 'number' || + (item !== null && typeof item === 'object' && !circularReferencesDetector.has(item))) + + this._encodeArrayPrefix(bytes, validValue) + + for (const item of validValue) { + this._encodeObject(bytes, item, circularReferencesDetector) + } + } + _cacheString (value) { if (!(value in this._stringMap)) { this._stringCount++ diff --git a/packages/dd-trace/src/encode/agentless-ci-visibility.js b/packages/dd-trace/src/encode/agentless-ci-visibility.js index c0111d19679..dea15182323 100644 --- a/packages/dd-trace/src/encode/agentless-ci-visibility.js +++ b/packages/dd-trace/src/encode/agentless-ci-visibility.js @@ -2,14 +2,21 @@ const { truncateSpan, normalizeSpan } = require('./tags-processors') const { AgentEncoder } = require('./0.4') const { version: ddTraceVersion } = require('../../../../package.json') -const id = require('../../../dd-trace/src/id') -const ENCODING_VERSION = 1 +const { ITR_CORRELATION_ID } = require('../../src/plugins/util/test') +const id = require('../../src/id') +const { + distributionMetric, + TELEMETRY_ENDPOINT_PAYLOAD_SERIALIZATION_MS, + TELEMETRY_ENDPOINT_PAYLOAD_EVENTS_COUNT +} = require('../ci-visibility/telemetry') +const ENCODING_VERSION = 1 const ALLOWED_CONTENT_TYPES = ['test_session_end', 'test_module_end', 'test_suite_end', 'test'] const TEST_SUITE_KEYS_LENGTH = 12 const TEST_MODULE_KEYS_LENGTH = 11 const TEST_SESSION_KEYS_LENGTH = 10 +const TEST_AND_SPAN_KEYS_LENGTH = 11 const INTAKE_SOFT_LIMIT = 2 * 1024 * 1024 // 2MB @@ -36,11 +43,23 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder { // length of `payload.events` when calling `makePayload` this._eventCount = 0 + this.metadataTags = {} + this.reset() } + setMetadataTags (tags) { + this.metadataTags = tags + } + _encodeTestSuite (bytes, content) { - this._encodeMapPrefix(bytes, TEST_SUITE_KEYS_LENGTH) + let keysLength = TEST_SUITE_KEYS_LENGTH + const itrCorrelationId = content.meta[ITR_CORRELATION_ID] + if (itrCorrelationId) { + keysLength++ + } + + this._encodeMapPrefix(bytes, keysLength) this._encodeString(bytes, 'type') this._encodeString(bytes, content.type) @@ -53,6 +72,12 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder { this._encodeString(bytes, 'test_suite_id') this._encodeId(bytes, content.span_id) + if (itrCorrelationId) { + this._encodeString(bytes, ITR_CORRELATION_ID) + this._encodeString(bytes, itrCorrelationId) + delete content.meta[ITR_CORRELATION_ID] + } + this._encodeString(bytes, 'error') this._encodeNumber(bytes, content.error) this._encodeString(bytes, 'name') @@ -127,9 +152,7 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder { } _encodeEventContent (bytes, content) { - const keysLength = Object.keys(content).length - - let totalKeysLength = keysLength + let totalKeysLength = TEST_AND_SPAN_KEYS_LENGTH if (content.meta.test_session_id) { totalKeysLength = totalKeysLength + 1 } @@ -139,6 +162,13 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder { if (content.meta.test_suite_id) { totalKeysLength = totalKeysLength + 1 } + const itrCorrelationId = content.meta[ITR_CORRELATION_ID] + if (itrCorrelationId) { + totalKeysLength = totalKeysLength + 1 + } + if (content.type) { + totalKeysLength = totalKeysLength + 1 + } this._encodeMapPrefix(bytes, totalKeysLength) if (content.type) { this._encodeString(bytes, 'type') @@ -189,6 +219,12 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder { delete content.meta.test_suite_id } + if (itrCorrelationId) { + this._encodeString(bytes, ITR_CORRELATION_ID) + this._encodeString(bytes, itrCorrelationId) + delete content.meta[ITR_CORRELATION_ID] + } + this._encodeString(bytes, 'meta') this._encodeMap(bytes, content.meta) this._encodeString(bytes, 'metrics') @@ -247,6 +283,12 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder { } _encode (bytes, trace) { + if (this._isReset) { + this._encodePayloadStart(bytes) + this._isReset = false + } + const startTime = Date.now() + const rawEvents = trace.map(formatSpan) const testSessionEvents = rawEvents.filter( @@ -261,9 +303,15 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder { for (const event of events) { this._encodeEvent(bytes, event) } + distributionMetric( + TELEMETRY_ENDPOINT_PAYLOAD_SERIALIZATION_MS, + { endpoint: 'test_cycle' }, + Date.now() - startTime + ) } makePayload () { + distributionMetric(TELEMETRY_ENDPOINT_PAYLOAD_EVENTS_COUNT, { endpoint: 'test_cycle' }, this._eventCount) const bytes = this._traceBytes const eventsOffset = this._eventsOffset const eventsCount = this._eventCount @@ -290,9 +338,10 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder { version: ENCODING_VERSION, metadata: { '*': { - 'language': 'javascript', - 'library_version': ddTraceVersion - } + language: 'javascript', + library_version: ddTraceVersion + }, + ...this.metadataTags }, events: [] } @@ -311,6 +360,22 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder { this._encodeMapPrefix(bytes, Object.keys(payload.metadata).length) this._encodeString(bytes, '*') this._encodeMap(bytes, payload.metadata['*']) + if (payload.metadata.test) { + this._encodeString(bytes, 'test') + this._encodeMap(bytes, payload.metadata.test) + } + if (payload.metadata.test_suite_end) { + this._encodeString(bytes, 'test_suite_end') + this._encodeMap(bytes, payload.metadata.test_suite_end) + } + if (payload.metadata.test_module_end) { + this._encodeString(bytes, 'test_module_end') + this._encodeMap(bytes, payload.metadata.test_module_end) + } + if (payload.metadata.test_session_end) { + this._encodeString(bytes, 'test_session_end') + this._encodeMap(bytes, payload.metadata.test_session_end) + } this._encodeString(bytes, 'events') // Get offset of the events list to update the length of the array when calling `makePayload` this._eventsOffset = bytes.length @@ -321,7 +386,7 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder { reset () { this._reset() this._eventCount = 0 - this._encodePayloadStart(this._traceBytes) + this._isReset = true } } diff --git a/packages/dd-trace/src/encode/coverage-ci-visibility.js b/packages/dd-trace/src/encode/coverage-ci-visibility.js index a877e11e864..bdf4b17a3cc 100644 --- a/packages/dd-trace/src/encode/coverage-ci-visibility.js +++ b/packages/dd-trace/src/encode/coverage-ci-visibility.js @@ -2,6 +2,11 @@ const { AgentEncoder } = require('./0.4') const Chunk = require('./chunk') +const { + distributionMetric, + TELEMETRY_ENDPOINT_PAYLOAD_SERIALIZATION_MS, + TELEMETRY_ENDPOINT_PAYLOAD_EVENTS_COUNT +} = require('../ci-visibility/telemetry') const FormData = require('../exporters/common/form-data') const COVERAGE_PAYLOAD_VERSION = 2 @@ -21,8 +26,16 @@ class CoverageCIVisibilityEncoder extends AgentEncoder { } encode (coverage) { + const startTime = Date.now() + this._coveragesCount++ this.encodeCodeCoverage(this._coverageBytes, coverage) + + distributionMetric( + TELEMETRY_ENDPOINT_PAYLOAD_SERIALIZATION_MS, + { endpoint: 'code_coverage' }, + Date.now() - startTime + ) } encodeCodeCoverage (bytes, coverage) { @@ -73,6 +86,7 @@ class CoverageCIVisibilityEncoder extends AgentEncoder { } makePayload () { + distributionMetric(TELEMETRY_ENDPOINT_PAYLOAD_EVENTS_COUNT, { endpoint: 'code_coverage' }, this._coveragesCount) const bytes = this._coverageBytes const coveragesOffset = this._coveragesOffset @@ -94,7 +108,7 @@ class CoverageCIVisibilityEncoder extends AgentEncoder { 'coverage1', buffer, { - filename: `coverage1.msgpack`, + filename: 'coverage1.msgpack', contentType: 'application/msgpack' } ) diff --git a/packages/dd-trace/src/exporter.js b/packages/dd-trace/src/exporter.js index 042e43910dc..02d50c3b57e 100644 --- a/packages/dd-trace/src/exporter.js +++ b/packages/dd-trace/src/exporter.js @@ -18,7 +18,9 @@ module.exports = name => { case exporters.AGENT_PROXY: return require('./ci-visibility/exporters/agent-proxy') case exporters.JEST_WORKER: - return require('./ci-visibility/exporters/jest-worker') + case exporters.CUCUMBER_WORKER: + case exporters.MOCHA_WORKER: + return require('./ci-visibility/exporters/test-worker') default: return inAWSLambda && !usingLambdaExtension ? require('./exporters/log') : require('./exporters/agent') } diff --git a/packages/dd-trace/src/exporters/agent/index.js b/packages/dd-trace/src/exporters/agent/index.js index c617d27e89b..b2f25eeda99 100644 --- a/packages/dd-trace/src/exporters/agent/index.js +++ b/packages/dd-trace/src/exporters/agent/index.js @@ -7,7 +7,7 @@ const Writer = require('./writer') class AgentExporter { constructor (config, prioritySampler) { this._config = config - const { url, hostname, port, lookup, protocolVersion, stats = {} } = config + const { url, hostname, port, lookup, protocolVersion, stats = {}, appsec } = config this._url = url || new URL(format({ protocol: 'http:', hostname: hostname || 'localhost', @@ -15,7 +15,7 @@ class AgentExporter { })) const headers = {} - if (stats.enabled) { + if (stats.enabled || appsec?.standalone?.enabled) { headers['Datadog-Client-Computed-Stats'] = 'yes' } diff --git a/packages/dd-trace/src/exporters/common/agent-info-exporter.js b/packages/dd-trace/src/exporters/common/agent-info-exporter.js index 9d1c45195bc..923b7eef0ef 100644 --- a/packages/dd-trace/src/exporters/common/agent-info-exporter.js +++ b/packages/dd-trace/src/exporters/common/agent-info-exporter.js @@ -1,6 +1,7 @@ const { URL, format } = require('url') const request = require('./request') +const { incrementCountMetric, TELEMETRY_EVENTS_ENQUEUED_FOR_SERIALIZATION } = require('../../ci-visibility/telemetry') function fetchAgentInfo (url, callback) { request('', { @@ -49,6 +50,9 @@ class AgentInfoExporter { } _export (payload, writer = this._writer, timerKey = '_timer') { + if (this._config.isCiVisibility) { + incrementCountMetric(TELEMETRY_EVENTS_ENQUEUED_FOR_SERIALIZATION, {}, payload.length) + } writer.append(payload) const { flushInterval } = this._config diff --git a/packages/dd-trace/src/exporters/common/form-data.js b/packages/dd-trace/src/exporters/common/form-data.js index b20e97b8864..dacd495b160 100644 --- a/packages/dd-trace/src/exporters/common/form-data.js +++ b/packages/dd-trace/src/exporters/common/form-data.js @@ -21,6 +21,10 @@ class FormData extends Readable { } } + size () { + return this._data.reduce((size, chunk) => size + chunk.length, 0) + } + getHeaders () { return { 'Content-Type': 'multipart/form-data; boundary=' + this._boundary } } diff --git a/packages/dd-trace/src/exporters/common/request.js b/packages/dd-trace/src/exporters/common/request.js index c59976edb50..ab8b697eef6 100644 --- a/packages/dd-trace/src/exporters/common/request.js +++ b/packages/dd-trace/src/exporters/common/request.js @@ -6,7 +6,9 @@ const { Readable } = require('stream') const http = require('http') const https = require('https') -const { parse: urlParse } = require('url') +const zlib = require('zlib') + +const { urlToHttpOptions } = require('./url-to-http-options-polyfill') const docker = require('./docker') const { httpAgent, httpsAgent } = require('./agents') const { storage } = require('../../../../datadog-core') @@ -17,39 +19,14 @@ const containerId = docker.id() let activeRequests = 0 -// TODO: Replace with `url.urlToHttpOptions` when supported by all versions -function urlToOptions (url) { - const agent = url.agent || http.globalAgent - const options = { - protocol: url.protocol || agent.protocol, - hostname: typeof url.hostname === 'string' && url.hostname.startsWith('[') - ? url.hostname.slice(1, -1) - : url.hostname || - url.host || - 'localhost', - hash: url.hash, - search: url.search, - pathname: url.pathname, - path: `${url.pathname || ''}${url.search || ''}`, - href: url.href - } - if (url.port !== '') { - options.port = Number(url.port) - } - if (url.username || url.password) { - options.auth = `${url.username}:${url.password}` - } - return options -} +function parseUrl (urlObjOrString) { + if (typeof urlObjOrString === 'object') return urlToHttpOptions(urlObjOrString) -function fromUrlString (urlString) { - const url = typeof urlToHttpOptions === 'function' - ? urlToOptions(new URL(urlString)) - : urlParse(urlString) + const url = urlToHttpOptions(new URL(urlObjOrString)) - // Add the 'hostname' back if we're using named pipes - if (url.protocol === 'unix:' && url.host === '.') { - const udsPath = urlString.replace(/^unix:/, '') + // Special handling if we're using named pipes on Windows + if (url.protocol === 'unix:' && url.hostname === '.') { + const udsPath = urlObjOrString.slice(5) url.path = udsPath url.pathname = udsPath } @@ -63,7 +40,7 @@ function request (data, options, callback) { } if (options.url) { - const url = typeof options.url === 'object' ? urlToOptions(options.url) : fromUrlString(options.url) + const url = parseUrl(options.url) if (url.protocol === 'unix:') { options.socketPath = url.pathname } else { @@ -93,16 +70,31 @@ function request (data, options, callback) { options.agent = isSecure ? httpsAgent : httpAgent const onResponse = res => { - let responseData = '' + const chunks = [] res.setTimeout(timeout) - res.on('data', chunk => { responseData += chunk }) + res.on('data', chunk => { + chunks.push(chunk) + }) res.on('end', () => { activeRequests-- + const buffer = Buffer.concat(chunks) if (res.statusCode >= 200 && res.statusCode <= 299) { - callback(null, responseData, res.statusCode) + const isGzip = res.headers['content-encoding'] === 'gzip' + if (isGzip) { + zlib.gunzip(buffer, (err, result) => { + if (err) { + log.error(`Could not gunzip response: ${err.message}`) + callback(null, '', res.statusCode) + } else { + callback(null, result.toString(), res.statusCode) + } + }) + } else { + callback(null, buffer.toString(), res.statusCode) + } } else { let errorMessage = '' try { @@ -114,6 +106,7 @@ function request (data, options, callback) { } catch (e) { // ignore error } + const responseData = buffer.toString() if (responseData) { errorMessage += ` Response from the endpoint: "${responseData}"` } @@ -164,7 +157,7 @@ function request (data, options, callback) { } function byteLength (data) { - return data.length > 0 ? data.reduce((prev, next) => prev + next.length, 0) : 0 + return data.length > 0 ? data.reduce((prev, next) => prev + Buffer.byteLength(next, 'utf8'), 0) : 0 } Object.defineProperty(request, 'writable', { diff --git a/packages/dd-trace/src/exporters/common/url-to-http-options-polyfill.js b/packages/dd-trace/src/exporters/common/url-to-http-options-polyfill.js new file mode 100644 index 00000000000..4ba6b337b08 --- /dev/null +++ b/packages/dd-trace/src/exporters/common/url-to-http-options-polyfill.js @@ -0,0 +1,31 @@ +'use strict' + +const { urlToHttpOptions } = require('url') + +// TODO: Remove `urlToHttpOptions` polyfill once we drop support for the older Cypress versions that uses a built-in +// version of Node.js doesn't include that function. +module.exports = { + urlToHttpOptions: urlToHttpOptions ?? function (url) { + const { hostname, pathname, port, username, password, search } = url + const options = { + __proto__: null, + ...url, // In case the url object was extended by the user. + protocol: url.protocol, + hostname: typeof hostname === 'string' && hostname.startsWith('[') + ? hostname.slice(1, -1) + : hostname, + hash: url.hash, + search, + pathname, + path: `${pathname || ''}${search || ''}`, + href: url.href + } + if (port !== '') { + options.port = Number(port) + } + if (username || password) { + options.auth = `${decodeURIComponent(username)}:${decodeURIComponent(password)}` + } + return options + } +} diff --git a/packages/dd-trace/src/exporters/span-stats/writer.js b/packages/dd-trace/src/exporters/span-stats/writer.js index 11cc09128c4..3ece6d221b4 100644 --- a/packages/dd-trace/src/exporters/span-stats/writer.js +++ b/packages/dd-trace/src/exporters/span-stats/writer.js @@ -1,4 +1,3 @@ - const { SpanStatsEncoder } = require('../../encode/span-stats') const pkg = require('../../../../../package.json') diff --git a/packages/dd-trace/src/external-logger/src/index.js b/packages/dd-trace/src/external-logger/src/index.js index 20a6466874d..aa21b20b6e7 100644 --- a/packages/dd-trace/src/external-logger/src/index.js +++ b/packages/dd-trace/src/external-logger/src/index.js @@ -51,11 +51,11 @@ class ExternalLogger { const payload = { ...log, - 'timestamp': Date.now(), - 'hostname': log.hostname || this.hostname, - 'ddsource': log.ddsource || this.ddsource, - 'service': log.service || this.service, - 'ddtags': logTags || undefined + timestamp: Date.now(), + hostname: log.hostname || this.hostname, + ddsource: log.ddsource || this.ddsource, + service: log.service || this.service, + ddtags: logTags || undefined } this.enqueue(payload) diff --git a/packages/dd-trace/src/flare/file.js b/packages/dd-trace/src/flare/file.js new file mode 100644 index 00000000000..00388e14c5b --- /dev/null +++ b/packages/dd-trace/src/flare/file.js @@ -0,0 +1,44 @@ +'use strict' + +const { Writable } = require('stream') + +const INITIAL_SIZE = 64 * 1024 + +class FlareFile extends Writable { + constructor () { + super() + + this.length = 0 + + this._buffer = Buffer.alloc(INITIAL_SIZE) + } + + get data () { + return this._buffer.subarray(0, this.length) + } + + _write (chunk, encoding, callback) { + const length = Buffer.byteLength(chunk) + + this._reserve(length) + + if (Buffer.isBuffer(chunk)) { + this.length += chunk.copy(this._buffer, this.length) + } else { + this.length += this._buffer.write(chunk, encoding) + } + + callback() + } + + _reserve (length) { + while (this.length + length > this._buffer.length) { + const buffer = Buffer.alloc(this.length * 2) + + this._buffer.copy(buffer) + this._buffer = buffer + } + } +} + +module.exports = FlareFile diff --git a/packages/dd-trace/src/flare/index.js b/packages/dd-trace/src/flare/index.js new file mode 100644 index 00000000000..70ec4ccd75e --- /dev/null +++ b/packages/dd-trace/src/flare/index.js @@ -0,0 +1,98 @@ +'use strict' + +const log = require('../log') +const startupLog = require('../startup-log') +const FlareFile = require('./file') +const { LogChannel } = require('../log/channels') +const request = require('../exporters/common/request') +const FormData = require('../exporters/common/form-data') + +const MAX_LOG_SIZE = 12 * 1024 * 1024 // 12MB soft limit +const TIMEOUT = 20 * 1000 * 60 + +let logChannel = null +let tracerLogs = null +let timer +let tracerConfig = null + +const logger = { + debug: (msg) => recordLog(msg), + info: (msg) => recordLog(msg), + warn: (msg) => recordLog(msg), + error: (err) => recordLog(err.stack) +} + +const flare = { + enable (tracerConfig_) { + tracerConfig = tracerConfig_ + }, + + disable () { + tracerConfig = null + + flare.cleanup() + }, + + prepare (logLevel) { + if (!tracerConfig) return + + logChannel?.unsubscribe(logger) + logChannel = new LogChannel(logLevel) + logChannel.subscribe(logger) + tracerLogs = tracerLogs || new FlareFile() + timer = timer || setTimeout(flare.cleanup, TIMEOUT) + }, + + send (task) { + if (!tracerConfig) return + + const tracerInfo = new FlareFile() + + tracerInfo.write(JSON.stringify(startupLog.tracerInfo(), null, 2)) + + flare._sendFile(task, tracerInfo, 'tracer_info.txt') + flare._sendFile(task, tracerLogs, 'tracer_logs.txt') + + flare.cleanup() + }, + + cleanup () { + logChannel?.unsubscribe(logger) + timer = clearTimeout(timer) + logChannel = null + tracerLogs = null + }, + + _sendFile (task, file, filename) { + if (!file) return + + const form = new FormData() + + form.append('case_id', task.case_id) + form.append('hostname', task.hostname) + form.append('email', task.user_handle) + form.append('source', 'tracer_nodejs') + form.append('flare_file', file.data, { filename }) + + request(form, { + url: tracerConfig.url, + hostname: tracerConfig.hostname, + port: tracerConfig.port, + method: 'POST', + path: '/tracer_flare/v1', + headers: form.getHeaders() + }, (err) => { + if (err) { + log.error(err) + } + }) + } +} + +function recordLog (msg) { + if (tracerLogs.length > MAX_LOG_SIZE) return + + tracerLogs.write(`${msg}\n`) // TODO: gzip +} + +module.exports = flare diff --git a/packages/dd-trace/src/format.js b/packages/dd-trace/src/format.js index cbe41458a83..1b7b86d17f0 100644 --- a/packages/dd-trace/src/format.js +++ b/packages/dd-trace/src/format.js @@ -33,6 +33,8 @@ const map = { function format (span) { const formatted = formatSpan(span) + extractSpanLinks(formatted, span) + extractSpanEvents(formatted, span) extractRootTags(formatted, span) extractChunkTags(formatted, span) extractTags(formatted, span) @@ -51,9 +53,11 @@ function formatSpan (span) { resource: String(spanContext._name), error: 0, meta: {}, + meta_struct: span.meta_struct, metrics: {}, start: Math.round(span._startTime * 1e6), - duration: Math.round(span._duration * 1e6) + duration: Math.round(span._duration * 1e6), + links: [] } } @@ -64,6 +68,44 @@ function setSingleSpanIngestionTags (span, options) { addTag({}, span.metrics, SPAN_SAMPLING_MAX_PER_SECOND, options.maxPerSecond) } +function extractSpanLinks (trace, span) { + const links = [] + if (span._links) { + for (const link of span._links) { + const { context, attributes } = link + const formattedLink = {} + + formattedLink.trace_id = context.toTraceId(true) + formattedLink.span_id = context.toSpanId(true) + + if (attributes && Object.keys(attributes).length > 0) { + formattedLink.attributes = attributes + } + if (context?._sampling?.priority >= 0) formattedLink.flags = context._sampling.priority > 0 ? 1 : 0 + if (context?._tracestate) formattedLink.tracestate = context._tracestate.toString() + + links.push(formattedLink) + } + } + if (links.length > 0) { trace.meta['_dd.span_links'] = JSON.stringify(links) } +} + +function extractSpanEvents (trace, span) { + const events = [] + if (span._events) { + for (const event of span._events) { + const formattedEvent = { + name: event.name, + time_unix_nano: Math.round(event.startTime * 1e6), + attributes: event.attributes && Object.keys(event.attributes).length > 0 ? event.attributes : undefined + } + + events.push(formattedEvent) + } + } + if (events.length > 0) { trace.meta.events = JSON.stringify(events) } +} + function extractTags (trace, span) { const context = span.context() const origin = context._trace.origin @@ -84,7 +126,6 @@ function extractTags (trace, span) { for (const tag in tags) { switch (tag) { - case 'operation.name': case 'service.name': case 'span.type': case 'resource.name': @@ -111,7 +152,10 @@ function extractTags (trace, span) { case ERROR_STACK: // HACK: remove when implemented in the backend if (context._name !== 'fs.operation') { - trace.error = 1 + // HACK: to ensure otel.recordException does not influence trace.error + if (tags.setTraceError) { + trace.error = 1 + } } else { break } @@ -119,7 +163,6 @@ function extractTags (trace, span) { addTag(trace.meta, trace.metrics, tag, tags[tag]) } } - setSingleSpanIngestionTags(trace, context._spanSampling) addTag(trace.meta, trace.metrics, 'language', 'javascript') diff --git a/packages/dd-trace/src/index.js b/packages/dd-trace/src/index.js index a72986a840b..d761efc1885 100644 --- a/packages/dd-trace/src/index.js +++ b/packages/dd-trace/src/index.js @@ -5,6 +5,10 @@ const { isFalse } = require('./util') // Global `jest` is only present in Jest workers. const inJestWorker = typeof jest !== 'undefined' -module.exports = isFalse(process.env.DD_TRACE_ENABLED) || inJestWorker +const ddTraceDisabled = process.env.DD_TRACE_ENABLED + ? isFalse(process.env.DD_TRACE_ENABLED) + : String(process.env.OTEL_TRACES_EXPORTER).toLowerCase() === 'none' + +module.exports = ddTraceDisabled || inJestWorker ? require('./noop/proxy') : require('./proxy') diff --git a/packages/dd-trace/src/lambda/handler.js b/packages/dd-trace/src/lambda/handler.js index 88457ce9d07..ac86901bdc3 100644 --- a/packages/dd-trace/src/lambda/handler.js +++ b/packages/dd-trace/src/lambda/handler.js @@ -93,6 +93,7 @@ exports.datadog = function datadog (lambdaHandler) { return res }) } + clearTimeout(__lambdaTimeout) return result } } diff --git a/packages/dd-trace/src/lambda/index.js b/packages/dd-trace/src/lambda/index.js index d7e253b05a9..6d51c16eec2 100644 --- a/packages/dd-trace/src/lambda/index.js +++ b/packages/dd-trace/src/lambda/index.js @@ -2,4 +2,15 @@ const { registerLambdaHook } = require('./runtime/ritm') -registerLambdaHook() +/** + * It is safe to do it this way, since customers will never be expected to disable + * this specific instrumentation through the init config object. + */ +const _DD_TRACE_DISABLED_INSTRUMENTATIONS = process.env.DD_TRACE_DISABLED_INSTRUMENTATIONS || '' +const _disabledInstrumentations = new Set( + _DD_TRACE_DISABLED_INSTRUMENTATIONS ? _DD_TRACE_DISABLED_INSTRUMENTATIONS.split(',') : [] +) + +if (!_disabledInstrumentations.has('lambda')) { + registerLambdaHook() +} diff --git a/packages/dd-trace/src/lambda/runtime/ritm.js b/packages/dd-trace/src/lambda/runtime/ritm.js index 72bfedb9253..4dd27713a0b 100644 --- a/packages/dd-trace/src/lambda/runtime/ritm.js +++ b/packages/dd-trace/src/lambda/runtime/ritm.js @@ -87,7 +87,7 @@ const registerLambdaHook = () => { const lambdaTaskRoot = process.env.LAMBDA_TASK_ROOT const originalLambdaHandler = process.env.DD_LAMBDA_HANDLER - if (originalLambdaHandler !== undefined) { + if (originalLambdaHandler !== undefined && lambdaTaskRoot !== undefined) { const [moduleRoot, moduleAndHandler] = _extractModuleRootAndHandler(originalLambdaHandler) const [_module] = _extractModuleNameAndHandlerPath(moduleAndHandler) diff --git a/packages/dd-trace/src/log/channels.js b/packages/dd-trace/src/log/channels.js index 0bf84871b34..545fef4195a 100644 --- a/packages/dd-trace/src/log/channels.js +++ b/packages/dd-trace/src/log/channels.js @@ -3,44 +3,69 @@ const { channel } = require('dc-polyfill') const Level = { - Debug: 'debug', - Info: 'info', - Warn: 'warn', - Error: 'error' + trace: 20, + debug: 20, + info: 30, + warn: 40, + error: 50, + critical: 50, + off: 100 } -const defaultLevel = Level.Debug +const debugChannel = channel('datadog:log:debug') +const infoChannel = channel('datadog:log:info') +const warnChannel = channel('datadog:log:warn') +const errorChannel = channel('datadog:log:error') -// based on: https://github.com/trentm/node-bunyan#levels -const logChannels = { - [Level.Debug]: createLogChannel(Level.Debug, 20), - [Level.Info]: createLogChannel(Level.Info, 30), - [Level.Warn]: createLogChannel(Level.Warn, 40), - [Level.Error]: createLogChannel(Level.Error, 50) -} +const defaultLevel = Level.debug -function createLogChannel (name, logLevel) { - const logChannel = channel(`datadog:log:${name}`) - logChannel.logLevel = logLevel - return logChannel +function getChannelLogLevel (level) { + return level && typeof level === 'string' + ? Level[level.toLowerCase().trim()] || defaultLevel + : defaultLevel } -function getChannelLogLevel (level) { - let logChannel - if (level && typeof level === 'string') { - logChannel = logChannels[level.toLowerCase().trim()] || logChannels[defaultLevel] - } else { - logChannel = logChannels[defaultLevel] +class LogChannel { + constructor (level) { + this._level = getChannelLogLevel(level) + } + + subscribe (logger) { + if (Level.debug >= this._level) { + debugChannel.subscribe(logger.debug) + } + if (Level.info >= this._level) { + infoChannel.subscribe(logger.info) + } + if (Level.warn >= this._level) { + warnChannel.subscribe(logger.warn) + } + if (Level.error >= this._level) { + errorChannel.subscribe(logger.error) + } + } + + unsubscribe (logger) { + if (debugChannel.hasSubscribers) { + debugChannel.unsubscribe(logger.debug) + } + if (infoChannel.hasSubscribers) { + infoChannel.unsubscribe(logger.info) + } + if (warnChannel.hasSubscribers) { + warnChannel.unsubscribe(logger.warn) + } + if (errorChannel.hasSubscribers) { + errorChannel.unsubscribe(logger.error) + } } - return logChannel.logLevel } module.exports = { - Level, - getChannelLogLevel, + LogChannel, - debugChannel: logChannels[Level.Debug], - infoChannel: logChannels[Level.Info], - warnChannel: logChannels[Level.Warn], - errorChannel: logChannels[Level.Error] + debugChannel, + infoChannel, + warnChannel, + errorChannel } diff --git a/packages/dd-trace/src/log/index.js b/packages/dd-trace/src/log/index.js index 3cb7bd294ca..726d7d1e5e7 100644 --- a/packages/dd-trace/src/log/index.js +++ b/packages/dd-trace/src/log/index.js @@ -1,5 +1,7 @@ 'use strict' +const coalesce = require('koalas') +const { isTrue } = require('../util') const { debugChannel, infoChannel, warnChannel, errorChannel } = require('./channels') const logWriter = require('./writer') @@ -20,13 +22,29 @@ function processMsg (msg) { return typeof msg === 'function' ? msg() : msg } +const config = { + enabled: false, + logger: undefined, + logLevel: 'debug' +} + const log = { + /** + * @returns Read-only version of logging config. To modify config, call `log.use` and `log.toggle` + */ + getConfig () { + return { ...config } + }, + use (logger) { + config.logger = logger logWriter.use(logger) return this }, toggle (enabled, logLevel) { + config.enabled = enabled + config.logLevel = logLevel logWriter.toggle(enabled, logLevel) return this }, @@ -76,4 +94,18 @@ const log = { log.reset() +const enabled = isTrue(coalesce( + process.env.DD_TRACE_DEBUG, + process.env.OTEL_LOG_LEVEL === 'debug', + config.enabled +)) + +const logLevel = coalesce( + process.env.DD_TRACE_LOG_LEVEL, + process.env.OTEL_LOG_LEVEL, + config.logLevel +) + +log.toggle(enabled, logLevel) + module.exports = log diff --git a/packages/dd-trace/src/log/writer.js b/packages/dd-trace/src/log/writer.js index 798d6269f14..bc4a5b20621 100644 --- a/packages/dd-trace/src/log/writer.js +++ b/packages/dd-trace/src/log/writer.js @@ -1,8 +1,7 @@ 'use strict' const { storage } = require('../../../datadog-core') -const { getChannelLogLevel, debugChannel, infoChannel, warnChannel, errorChannel } = require('./channels') - +const { LogChannel } = require('./channels') const defaultLogger = { debug: msg => console.debug(msg), /* eslint-disable-line no-console */ info: msg => console.info(msg), /* eslint-disable-line no-console */ @@ -12,7 +11,7 @@ const defaultLogger = { let enabled = false let logger = defaultLogger -let logLevel = getChannelLogLevel() +let logChannel = new LogChannel() function withNoop (fn) { const store = storage.getStore() @@ -23,45 +22,21 @@ function withNoop (fn) { } function unsubscribeAll () { - if (debugChannel.hasSubscribers) { - debugChannel.unsubscribe(onDebug) - } - if (infoChannel.hasSubscribers) { - infoChannel.unsubscribe(onInfo) - } - if (warnChannel.hasSubscribers) { - warnChannel.unsubscribe(onWarn) - } - if (errorChannel.hasSubscribers) { - errorChannel.unsubscribe(onError) - } + logChannel.unsubscribe({ debug, info, warn, error }) } -function toggleSubscription (enable) { +function toggleSubscription (enable, level) { unsubscribeAll() if (enable) { - if (debugChannel.logLevel >= logLevel) { - debugChannel.subscribe(onDebug) - } - if (infoChannel.logLevel >= logLevel) { - infoChannel.subscribe(onInfo) - } - if (warnChannel.logLevel >= logLevel) { - warnChannel.subscribe(onWarn) - } - if (errorChannel.logLevel >= logLevel) { - errorChannel.subscribe(onError) - } + logChannel = new LogChannel(level) + logChannel.subscribe({ debug, info, warn, error }) } } function toggle (enable, level) { - if (level !== undefined) { - logLevel = getChannelLogLevel(level) - } enabled = enable - toggleSubscription(enabled) + toggleSubscription(enabled, level) } function use (newLogger) { @@ -73,26 +48,9 @@ function use (newLogger) { function reset () { logger = defaultLogger enabled = false - logLevel = getChannelLogLevel() toggleSubscription(false) } -function onError (err) { - if (enabled) error(err) -} - -function onWarn (message) { - if (enabled) warn(message) -} - -function onInfo (message) { - if (enabled) info(message) -} - -function onDebug (message) { - if (enabled) debug(message) -} - function error (err) { if (typeof err !== 'object' || !err) { err = String(err) diff --git a/packages/dd-trace/src/noop/dogstatsd.js b/packages/dd-trace/src/noop/dogstatsd.js new file mode 100644 index 00000000000..899ac11e228 --- /dev/null +++ b/packages/dd-trace/src/noop/dogstatsd.js @@ -0,0 +1,11 @@ +module.exports = class NoopDogStatsDClient { + increment () { } + + gauge () { } + + distribution () { } + + histogram () { } + + flush () { } +} diff --git a/packages/dd-trace/src/noop/proxy.js b/packages/dd-trace/src/noop/proxy.js index db6e39392c9..417cb846f8d 100644 --- a/packages/dd-trace/src/noop/proxy.js +++ b/packages/dd-trace/src/noop/proxy.js @@ -2,14 +2,17 @@ const NoopTracer = require('./tracer') const NoopAppsecSdk = require('../appsec/sdk/noop') +const NoopDogStatsDClient = require('./dogstatsd') const noop = new NoopTracer() const noopAppsec = new NoopAppsecSdk() +const noopDogStatsDClient = new NoopDogStatsDClient() class Tracer { constructor () { this._tracer = noop this.appsec = noopAppsec + this.dogstatsd = noopDogStatsDClient } init () { diff --git a/packages/dd-trace/src/noop/span.js b/packages/dd-trace/src/noop/span.js index 3c5fac81b1b..bee3ce11702 100644 --- a/packages/dd-trace/src/noop/span.js +++ b/packages/dd-trace/src/noop/span.js @@ -18,6 +18,7 @@ class NoopSpan { getBaggageItem (key) {} setTag (key, value) { return this } addTags (keyValueMap) { return this } + addLink (link) { return this } log () { return this } logEvent () {} finish (finishTime) {} diff --git a/packages/dd-trace/src/opentelemetry/context_manager.js b/packages/dd-trace/src/opentelemetry/context_manager.js index 03e9bf8f647..fba84eef9f4 100644 --- a/packages/dd-trace/src/opentelemetry/context_manager.js +++ b/packages/dd-trace/src/opentelemetry/context_manager.js @@ -2,61 +2,46 @@ const { AsyncLocalStorage } = require('async_hooks') const { trace, ROOT_CONTEXT } = require('@opentelemetry/api') +const DataDogSpanContext = require('../opentracing/span_context') const SpanContext = require('./span_context') const tracer = require('../../') -// Horrible hack to acquire the otherwise inaccessible SPAN_KEY so we can redirect it... -// This is used for getting the current span context in OpenTelemetry, but the SPAN_KEY value is -// not exposed as it's meant to be read-only from outside the module. We want to hijack this logic -// so we can instead get the span context from the datadog context manager instead. -let SPAN_KEY -trace.getSpan({ - getValue (key) { - SPAN_KEY = key - } -}) - -// Whenever a value is acquired from the context map we should mostly delegate to the real getter, -// but when accessing the current span we should hijack that access to instead provide a fake span -// which we can use to get an OTel span context wrapping the datadog active scope span context. -function wrappedGetValue (target) { - return (key) => { - if (key === SPAN_KEY) { - return { - spanContext () { - const activeSpan = tracer.scope().active() - const context = activeSpan && activeSpan.context() - return new SpanContext(context) - } - } - } - return target.getValue(key) - } -} - class ContextManager { constructor () { this._store = new AsyncLocalStorage() } active () { - const active = this._store.getStore() || ROOT_CONTEXT + const activeSpan = tracer.scope().active() + const store = this._store.getStore() + const context = (activeSpan && activeSpan.context()) || store || ROOT_CONTEXT - return new Proxy(active, { - get (target, key) { - return key === 'getValue' ? wrappedGetValue(target) : target[key] - } - }) + if (!(context instanceof DataDogSpanContext)) { + return context + } + + if (!context._otelSpanContext) { + const newSpanContext = new SpanContext(context) + context._otelSpanContext = newSpanContext + } + if (store && trace.getSpanContext(store) === context._otelSpanContext) { + return store + } + return trace.setSpanContext(store || ROOT_CONTEXT, context._otelSpanContext) } with (context, fn, thisArg, ...args) { const span = trace.getSpan(context) const ddScope = tracer.scope() - return ddScope.activate(span._ddSpan, () => { + const run = () => { const cb = thisArg == null ? fn : fn.bind(thisArg) return this._store.run(context, cb, ...args) - }) + } + if (span && span._ddSpan) { + return ddScope.activate(span._ddSpan, run) + } + return run() } bind (context, target) { @@ -66,9 +51,7 @@ class ContextManager { } } - // Not part of the spec but the Node.js API expects these enable () {} disable () {} } - module.exports = ContextManager diff --git a/packages/dd-trace/src/opentelemetry/span.js b/packages/dd-trace/src/opentelemetry/span.js index 2ff7a37c577..d2c216c138e 100644 --- a/packages/dd-trace/src/opentelemetry/span.js +++ b/packages/dd-trace/src/opentelemetry/span.js @@ -20,6 +20,20 @@ function hrTimeToMilliseconds (time) { return time[0] * 1e3 + time[1] / 1e6 } +function isTimeInput (startTime) { + if (typeof startTime === 'number') { + return true + } + if (startTime instanceof Date) { + return true + } + if (Array.isArray(startTime) && startTime.length === 2 && + typeof startTime[0] === 'number' && typeof startTime[1] === 'number') { + return true + } + return false +} + const spanKindNames = { [api.SpanKind.INTERNAL]: kinds.INTERNAL, [api.SpanKind.SERVER]: kinds.SERVER, @@ -128,11 +142,12 @@ class Span { context: spanContext._ddContext, startTime, hostname: _tracer._hostname, - integrationName: 'otel', + integrationName: parentTracer?._isOtelLibrary ? 'otel.library' : 'otel', tags: { [SERVICE_NAME]: _tracer._service, [RESOURCE_NAME]: spanName - } + }, + links }, _tracer._debug) if (attributes) { @@ -148,7 +163,6 @@ class Span { // math for computing opentracing timestamps is apparently lossy... this.startTime = hrStartTime this.kind = kind - this.links = links this._spanProcessor.onStart(this, context) } @@ -161,9 +175,11 @@ class Span { get resource () { return this._parentTracer.resource } + get instrumentationLibrary () { return this._parentTracer.instrumentationLibrary } + get _spanProcessor () { return this._parentTracer.getActiveSpanProcessor() } @@ -177,17 +193,27 @@ class Span { } setAttribute (key, value) { + if (key === 'http.response.status_code') { + this._ddSpan.setTag('http.status_code', value.toString()) + } + this._ddSpan.setTag(key, value) return this } setAttributes (attributes) { + if ('http.response.status_code' in attributes) { + attributes['http.status_code'] = attributes['http.response.status_code'].toString() + } + this._ddSpan.addTags(attributes) return this } - addEvent (name, attributesOrStartTime, startTime) { - api.diag.warn('Events not supported') + addLink (context, attributes) { + // extract dd context + const ddSpanContext = context._ddContext + this._ddSpan.addLink(ddSpanContext, attributes) return this } @@ -227,12 +253,29 @@ class Span { return this.ended === false } - recordException (exception) { + addEvent (name, attributesOrStartTime, startTime) { + startTime = attributesOrStartTime && isTimeInput(attributesOrStartTime) ? attributesOrStartTime : startTime + const hrStartTime = timeInputToHrTime(startTime || (performance.now() + timeOrigin)) + startTime = hrTimeToMilliseconds(hrStartTime) + + this._ddSpan.addEvent(name, attributesOrStartTime, startTime) + return this + } + + recordException (exception, timeInput) { + // HACK: identifier is added so that trace.error remains unchanged after a call to otel.recordException this._ddSpan.addTags({ [ERROR_TYPE]: exception.name, [ERROR_MESSAGE]: exception.message, - [ERROR_STACK]: exception.stack + [ERROR_STACK]: exception.stack, + doNotSetTraceError: true }) + const attributes = {} + if (exception.message) attributes['exception.message'] = exception.message + if (exception.type) attributes['exception.type'] = exception.type + if (exception.escaped) attributes['exception.escaped'] = exception.escaped + if (exception.stack) attributes['exception.stacktrace'] = exception.stack + this.addEvent(exception.name, attributes, timeInput) } get duration () { @@ -240,7 +283,7 @@ class Span { } get ended () { - return typeof this.duration !== 'undefined' + return this.duration !== undefined } } diff --git a/packages/dd-trace/src/opentelemetry/span_context.js b/packages/dd-trace/src/opentelemetry/span_context.js index f070ba525c2..06c9b26f8a4 100644 --- a/packages/dd-trace/src/opentelemetry/span_context.js +++ b/packages/dd-trace/src/opentelemetry/span_context.js @@ -24,11 +24,11 @@ class SpanContext { } get traceId () { - return this._ddContext._traceId.toString(16) + return this._ddContext.toTraceId(true) } get spanId () { - return this._ddContext._spanId.toString(16) + return this._ddContext.toSpanId(true) } get traceFlags () { diff --git a/packages/dd-trace/src/opentelemetry/tracer.js b/packages/dd-trace/src/opentelemetry/tracer.js index d3422300ef2..bf2a0c3f86b 100644 --- a/packages/dd-trace/src/opentelemetry/tracer.js +++ b/packages/dd-trace/src/opentelemetry/tracer.js @@ -7,6 +7,7 @@ const Sampler = require('./sampler') const Span = require('./span') const id = require('../id') const SpanContext = require('./span_context') +const TextMapPropagator = require('../opentracing/propagation/text_map') class Tracer { constructor (library, config, tracerProvider) { @@ -15,12 +16,32 @@ class Tracer { this._tracerProvider = tracerProvider // Is there a reason this is public? this.instrumentationLibrary = library + this._isOtelLibrary = library?.name?.startsWith('@opentelemetry/instrumentation-') + this._spanLimits = {} } get resource () { return this._tracerProvider.resource } + _createSpanContextFromParent (parentSpanContext) { + return new SpanContext({ + traceId: parentSpanContext._traceId, + spanId: id(), + parentId: parentSpanContext._spanId, + sampling: parentSpanContext._sampling, + baggageItems: Object.assign({}, parentSpanContext._baggageItems), + trace: parentSpanContext._trace, + tracestate: parentSpanContext._tracestate + }) + } + + // Extracted method to create span context for a new span + _createSpanContextForNewSpan (context) { + const { traceId, spanId, traceFlags, traceState } = context + return TextMapPropagator._convertOtelContextToDatadog(traceId, spanId, traceFlags, traceState) + } + startSpan (name, options = {}, context = api.context.active()) { // remove span from context in case a root span is requested via options if (options.root) { @@ -28,21 +49,11 @@ class Tracer { } const parentSpan = api.trace.getSpan(context) const parentSpanContext = parentSpan && parentSpan.spanContext() - let spanContext - // TODO: Need a way to get 128-bit trace IDs for the validity check API to work... - // if (parent && api.trace.isSpanContextValid(parent)) { - if (parentSpanContext && parentSpanContext.traceId) { - const parent = parentSpanContext._ddContext - spanContext = new SpanContext({ - traceId: parent._traceId, - spanId: id(), - parentId: parent._spanId, - sampling: parent._sampling, - baggageItems: Object.assign({}, parent._baggageItems), - trace: parent._trace, - tracestate: parent._tracestate - }) + if (parentSpanContext && api.trace.isSpanContextValid(parentSpanContext)) { + spanContext = parentSpanContext._ddContext + ? this._createSpanContextFromParent(parentSpanContext._ddContext) + : this._createSpanContextForNewSpan(parentSpanContext) } else { spanContext = new SpanContext() } @@ -118,6 +129,11 @@ class Tracer { getActiveSpanProcessor () { return this._tracerProvider.getActiveSpanProcessor() } + + // not used in our codebase but needed for compatibility. See issue #1244 + getSpanLimits () { + return this._spanLimits + } } module.exports = Tracer diff --git a/packages/dd-trace/src/opentelemetry/tracer_provider.js b/packages/dd-trace/src/opentelemetry/tracer_provider.js index 1d4119cbba1..e015cfad7db 100644 --- a/packages/dd-trace/src/opentelemetry/tracer_provider.js +++ b/packages/dd-trace/src/opentelemetry/tracer_provider.js @@ -1,6 +1,7 @@ 'use strict' -const { trace, context } = require('@opentelemetry/api') +const { trace, context, propagation } = require('@opentelemetry/api') +const { W3CTraceContextPropagator } = require('@opentelemetry/core') const tracer = require('../../') @@ -52,6 +53,13 @@ class TracerProvider { if (!trace.setGlobalTracerProvider(this)) { trace.getTracerProvider().setDelegate(this) } + // The default propagator used is the W3C Trace Context propagator, users should be able to pass in others + // as needed + if (config.propagator) { + propagation.setGlobalPropagator(config.propagator) + } else { + propagation.setGlobalPropagator(new W3CTraceContextPropagator()) + } } forceFlush () { diff --git a/packages/dd-trace/src/opentracing/propagation/log.js b/packages/dd-trace/src/opentracing/propagation/log.js index 957bfc113d2..2203f253fb6 100644 --- a/packages/dd-trace/src/opentracing/propagation/log.js +++ b/packages/dd-trace/src/opentracing/propagation/log.js @@ -15,7 +15,7 @@ class LogPropagator { if (spanContext) { if (this._config.traceId128BitLoggingEnabled && spanContext._trace.tags['_dd.p.tid']) { - carrier.dd.trace_id = spanContext._trace.tags['_dd.p.tid'] + spanContext._traceId.toString(16) + carrier.dd.trace_id = spanContext.toTraceId(true) } else { carrier.dd.trace_id = spanContext.toTraceId() } diff --git a/packages/dd-trace/src/opentracing/propagation/text_map.js b/packages/dd-trace/src/opentracing/propagation/text_map.js index 20a257bb61a..42a482853ee 100644 --- a/packages/dd-trace/src/opentracing/propagation/text_map.js +++ b/packages/dd-trace/src/opentracing/propagation/text_map.js @@ -1,13 +1,19 @@ 'use strict' -const pick = require('lodash.pick') +const pick = require('../../../../datadog-core/src/utils/src/pick') const id = require('../../id') const DatadogSpanContext = require('../span_context') +const OtelSpanContext = require('../../opentelemetry/span_context') const log = require('../../log') const TraceState = require('./tracestate') +const tags = require('../../../../../ext/tags') +const { channel } = require('dc-polyfill') const { AUTO_KEEP, AUTO_REJECT, USER_KEEP } = require('../../../../../ext/priority') +const injectCh = channel('dd-trace:span:inject') +const extractCh = channel('dd-trace:span:extract') + const traceKey = 'x-datadog-trace-id' const spanKey = 'x-datadog-parent-id' const originKey = 'x-datadog-origin' @@ -39,6 +45,7 @@ const tracestateTagKeyFilter = /[^\x21-\x2b\x2d-\x3c\x3e-\x7e]/g // Tag values in tracestate replace ',', '~' and ';' with '_' const tracestateTagValueFilter = /[^\x20-\x2b\x2d-\x3a\x3c-\x7d]/g const invalidSegment = /^0+$/ +const zeroTraceId = '0000000000000000' class TextMapPropagator { constructor (config) { @@ -46,12 +53,18 @@ class TextMapPropagator { } inject (spanContext, carrier) { + if (!spanContext || !carrier) return + this._injectBaggageItems(spanContext, carrier) this._injectDatadog(spanContext, carrier) this._injectB3MultipleHeaders(spanContext, carrier) this._injectB3SingleHeader(spanContext, carrier) this._injectTraceparent(spanContext, carrier) + if (injectCh.hasSubscribers) { + injectCh.publish({ spanContext, carrier }) + } + log.debug(() => `Inject into carrier: ${JSON.stringify(pick(carrier, logKeys))}.`) } @@ -60,6 +73,10 @@ class TextMapPropagator { if (!spanContext) return spanContext + if (extractCh.hasSubscribers) { + extractCh.publish({ spanContext, carrier }) + } + log.debug(() => `Extract from carrier: ${JSON.stringify(pick(carrier, logKeys))}.`) return spanContext @@ -171,9 +188,17 @@ class TextMapPropagator { carrier[traceparentKey] = spanContext.toTraceparent() ts.forVendor('dd', state => { + if (!spanContext._isRemote) { + // SpanContext was created by a ddtrace span. + // Last datadog span id should be set to the current span. + state.set('p', spanContext._spanId) + } else if (spanContext._trace.tags[tags.DD_PARENT_ID]) { + // Propagate the last Datadog span id set on the remote span. + state.set('p', spanContext._trace.tags[tags.DD_PARENT_ID]) + } state.set('s', priority) if (mechanism) { - state.set('t.dm', mechanism) + state.set('t.dm', `-${mechanism}`) } if (typeof origin === 'string') { @@ -206,9 +231,55 @@ class TextMapPropagator { return this._config.tracePropagationStyle[mode].includes(name) } + _hasTraceIdConflict (w3cSpanContext, firstSpanContext) { + return w3cSpanContext !== null && + firstSpanContext.toTraceId(true) === w3cSpanContext.toTraceId(true) && + firstSpanContext.toSpanId() !== w3cSpanContext.toSpanId() + } + + _hasParentIdInTags (spanContext) { + return tags.DD_PARENT_ID in spanContext._trace.tags + } + + _updateParentIdFromDdHeaders (carrier, firstSpanContext) { + const ddCtx = this._extractDatadogContext(carrier) + if (ddCtx !== null) { + firstSpanContext._trace.tags[tags.DD_PARENT_ID] = ddCtx._spanId.toString().padStart(16, '0') + } + } + + _resolveTraceContextConflicts (w3cSpanContext, firstSpanContext, carrier) { + if (!this._hasTraceIdConflict(w3cSpanContext, firstSpanContext)) { + return firstSpanContext + } + if (this._hasParentIdInTags(w3cSpanContext)) { + // tracecontext headers contain a p value, ensure this value is sent to backend + firstSpanContext._trace.tags[tags.DD_PARENT_ID] = w3cSpanContext._trace.tags[tags.DD_PARENT_ID] + } else { + // if p value is not present in tracestate, use the parent id from the datadog headers + this._updateParentIdFromDdHeaders(carrier, firstSpanContext) + } + // the span_id in tracecontext takes precedence over the first extracted propagation style + firstSpanContext._spanId = w3cSpanContext._spanId + return firstSpanContext + } + _extractSpanContext (carrier) { + let spanContext = null for (const extractor of this._config.tracePropagationStyle.extract) { - let spanContext = null + // add logic to ensure tracecontext headers takes precedence over other extracted headers + if (spanContext !== null) { + if (this._config.tracePropagationExtractFirst) { + return spanContext + } + if (extractor !== 'tracecontext') { + continue + } + spanContext = this._resolveTraceContextConflicts( + this._extractTraceparentContext(carrier), spanContext, carrier) + break + } + switch (extractor) { case 'datadog': spanContext = this._extractDatadogContext(carrier) @@ -216,21 +287,23 @@ class TextMapPropagator { case 'tracecontext': spanContext = this._extractTraceparentContext(carrier) break - case 'b3': // TODO: should match "b3 single header" in next major - case 'b3multi': - spanContext = this._extractB3MultiContext(carrier) - break + case 'b3' && this + ._config + .tracePropagationStyle + .otelPropagators: // TODO: should match "b3 single header" in next major case 'b3 single header': // TODO: delete in major after singular "b3" spanContext = this._extractB3SingleContext(carrier) break - } - - if (spanContext !== null) { - return spanContext + case 'b3': + case 'b3multi': + spanContext = this._extractB3MultiContext(carrier) + break + default: + log.warn(`Unknown propagation style: ${extractor}`) } } - return this._extractSqsdContext(carrier) + return spanContext || this._extractSqsdContext(carrier) } _extractDatadogContext (carrier) { @@ -279,7 +352,8 @@ class TextMapPropagator { return new DatadogSpanContext({ traceId: id(), spanId: null, - sampling: { priority } + sampling: { priority }, + isRemote: true }) } @@ -311,8 +385,8 @@ class TextMapPropagator { return null } const matches = headerValue.trim().match(traceparentExpr) - if (matches.length) { - const [ version, traceId, spanId, flags, tail ] = matches.slice(1) + if (matches?.length) { + const [version, traceId, spanId, flags, tail] = matches.slice(1) const traceparent = { version } const tracestate = TraceState.fromString(carrier.tracestate) if (invalidSegment.test(traceId)) return null @@ -327,6 +401,7 @@ class TextMapPropagator { const spanContext = new DatadogSpanContext({ traceId: id(traceId, 16), spanId: id(spanId, 16), + isRemote: true, sampling: { priority: parseInt(flags, 10) & 1 ? 1 : 0 }, traceparent, tracestate @@ -337,6 +412,10 @@ class TextMapPropagator { tracestate.forVendor('dd', state => { for (const [key, value] of state.entries()) { switch (key) { + case 'p': { + spanContext._trace.tags[tags.DD_PARENT_ID] = value + break + } case 's': { const priority = parseInt(value, 10) if (!Number.isInteger(priority)) continue @@ -379,7 +458,8 @@ class TextMapPropagator { return new DatadogSpanContext({ traceId: id(carrier[traceKey], radix), - spanId: id(carrier[spanKey], radix) + spanId: id(carrier[spanKey], radix), + isRemote: true }) } @@ -506,7 +586,7 @@ class TextMapPropagator { const tid = traceId.substring(0, 16) - if (tid === '0000000000000000') return + if (tid === zeroTraceId) return spanContext._trace.tags['_dd.p.tid'] = tid } @@ -536,6 +616,65 @@ class TextMapPropagator { return spanContext._traceId.toString(16) } + + static _convertOtelContextToDatadog (traceId, spanId, traceFlag, ts, meta = {}) { + const origin = null + let samplingPriority = traceFlag + + ts = ts?.traceparent || null + + if (ts) { + // Use TraceState.fromString to parse the tracestate header + const traceState = TraceState.fromString(ts) + let ddTraceStateData = null + + // Extract Datadog specific trace state data + traceState.forVendor('dd', (state) => { + ddTraceStateData = state + return state // You might need to adjust this part based on actual logic needed + }) + + if (ddTraceStateData) { + // Assuming ddTraceStateData is now a Map or similar structure containing Datadog trace state data + // Extract values as needed, similar to the original logic + const samplingPriorityTs = ddTraceStateData.get('s') + const origin = ddTraceStateData.get('o') + // Convert Map to object for meta + const otherPropagatedTags = Object.fromEntries(ddTraceStateData.entries()) + + // Update meta and samplingPriority based on extracted values + Object.assign(meta, otherPropagatedTags) + samplingPriority = TextMapPropagator._getSamplingPriority(traceFlag, parseInt(samplingPriorityTs, 10), origin) + } else { + log.debug(`no dd list member in tracestate from incoming request: ${ts}`) + } + } + + const spanContext = new OtelSpanContext({ + traceId: id(traceId, 16), spanId: id(), tags: meta, parentId: id(spanId, 16) + }) + + spanContext._sampling = { priority: samplingPriority } + spanContext._trace = { origin } + return spanContext + } + + static _getSamplingPriority (traceparentSampled, tracestateSamplingPriority, origin = null) { + const fromRumWithoutPriority = !tracestateSamplingPriority && origin === 'rum' + + let samplingPriority + if (!fromRumWithoutPriority && traceparentSampled === 0 && + (!tracestateSamplingPriority || tracestateSamplingPriority >= 0)) { + samplingPriority = 0 + } else if (!fromRumWithoutPriority && traceparentSampled === 1 && + (!tracestateSamplingPriority || tracestateSamplingPriority < 0)) { + samplingPriority = 1 + } else { + samplingPriority = tracestateSamplingPriority + } + + return samplingPriority + } } module.exports = TextMapPropagator diff --git a/packages/dd-trace/src/opentracing/propagation/text_map_dsm.js b/packages/dd-trace/src/opentracing/propagation/text_map_dsm.js new file mode 100644 index 00000000000..d44518fe828 --- /dev/null +++ b/packages/dd-trace/src/opentracing/propagation/text_map_dsm.js @@ -0,0 +1,43 @@ +const pick = require('../../../../datadog-core/src/utils/src/pick') +const log = require('../../log') + +const { DsmPathwayCodec } = require('../../datastreams/pathway') + +const base64Key = 'dd-pathway-ctx-base64' +const logKeys = [base64Key] + +class DSMTextMapPropagator { + constructor (config) { + this.config = config + } + + inject (ctx, carrier) { + if (!this.config.dsmEnabled) return + + this._injectDatadogDSMContext(ctx, carrier) + + log.debug(() => `Inject into carrier (DSM): ${JSON.stringify(pick(carrier, logKeys))}.`) + } + + extract (carrier) { + if (!this.config.dsmEnabled) return + + const dsmContext = this._extractDatadogDSMContext(carrier) + + if (!dsmContext) return dsmContext + + log.debug(() => `Extract from carrier (DSM): ${JSON.stringify(pick(carrier, logKeys))}.`) + return dsmContext + } + + _injectDatadogDSMContext (ctx, carrier) { + DsmPathwayCodec.encode(ctx, carrier) + } + + _extractDatadogDSMContext (carrier) { + const ctx = DsmPathwayCodec.decode(carrier) + return ctx + } +} + +module.exports = DSMTextMapPropagator diff --git a/packages/dd-trace/src/opentracing/span.js b/packages/dd-trace/src/opentracing/span.js index 86e0c5d12ed..723597ff043 100644 --- a/packages/dd-trace/src/opentracing/span.js +++ b/packages/dd-trace/src/opentracing/span.js @@ -13,6 +13,7 @@ const log = require('../log') const { storage } = require('../../../datadog-core') const telemetryMetrics = require('../telemetry/metrics') const { channel } = require('dc-polyfill') +const spanleak = require('../spanleak') const tracerMetrics = telemetryMetrics.manager.namespace('tracers') @@ -25,12 +26,14 @@ const unfinishedRegistry = createRegistry('unfinished') const finishedRegistry = createRegistry('finished') const OTEL_ENABLED = !!process.env.DD_TRACE_OTEL_ENABLED +const ALLOWED = ['string', 'number', 'boolean'] const integrationCounters = { - span_created: {}, - span_finished: {} + spans_created: {}, + spans_finished: {} } +const startCh = channel('dd-trace:span:start') const finishCh = channel('dd-trace:span:finish') function getIntegrationCounter (event, integration) { @@ -64,13 +67,15 @@ class DatadogSpan { this._store = storage.getStore() this._duration = undefined + this._events = [] + // For internal use only. You probably want `context()._name`. // This name property is not updated when the span name changes. // This is necessary for span count metrics. this._name = operationName this._integrationName = fields.integrationName || 'opentracing' - getIntegrationCounter('span_created', this._integrationName).inc() + getIntegrationCounter('spans_created', this._integrationName).inc() this._spanContext = this._createContext(parent, fields) this._spanContext._name = operationName @@ -81,6 +86,9 @@ class DatadogSpan { this._startTime = fields.startTime || this._getTime() + this._links = [] + fields.links && fields.links.forEach(link => this.addLink(link.context, link.attributes)) + if (DD_TRACE_EXPERIMENTAL_SPAN_COUNTS && finishedRegistry) { runtimeMetrics.increment('runtime.node.spans.unfinished') runtimeMetrics.increment('runtime.node.spans.unfinished.by.name', `span_name:${operationName}`) @@ -90,6 +98,11 @@ class DatadogSpan { unfinishedRegistry.register(this, operationName, this) } + spanleak.addSpan(this, operationName) + + if (startCh.hasSubscribers) { + startCh.publish({ span: this, fields }) + } } toString () { @@ -148,6 +161,26 @@ class DatadogSpan { logEvent () {} + addLink (context, attributes) { + this._links.push({ + context: context._ddContext ? context._ddContext : context, + attributes: this._sanitizeAttributes(attributes) + }) + } + + addEvent (name, attributesOrStartTime, startTime) { + const event = { name } + if (attributesOrStartTime) { + if (typeof attributesOrStartTime === 'object') { + event.attributes = this._sanitizeEventAttributes(attributesOrStartTime) + } else { + startTime = attributesOrStartTime + } + } + event.startTime = startTime || this._getTime() + this._events.push(event) + } + finish (finishTime) { if (this._duration !== undefined) { return @@ -159,7 +192,7 @@ class DatadogSpan { } } - getIntegrationCounter('span_finished', this._integrationName).inc() + getIntegrationCounter('spans_finished', this._integrationName).inc() if (DD_TRACE_EXPERIMENTAL_SPAN_COUNTS && finishedRegistry) { runtimeMetrics.decrement('runtime.node.spans.unfinished') @@ -183,6 +216,56 @@ class DatadogSpan { this._processor.process(this) } + _sanitizeAttributes (attributes = {}) { + const sanitizedAttributes = {} + + const addArrayOrScalarAttributes = (key, maybeArray) => { + if (Array.isArray(maybeArray)) { + for (const subkey in maybeArray) { + addArrayOrScalarAttributes(`${key}.${subkey}`, maybeArray[subkey]) + } + } else { + const maybeScalar = maybeArray + if (ALLOWED.includes(typeof maybeScalar)) { + // Wrap the value as a string if it's not already a string + sanitizedAttributes[key] = typeof maybeScalar === 'string' ? maybeScalar : String(maybeScalar) + } else { + log.warn('Dropping span link attribute. It is not of an allowed type') + } + } + } + + Object.entries(attributes).forEach(entry => { + const [key, value] = entry + addArrayOrScalarAttributes(key, value) + }) + return sanitizedAttributes + } + + _sanitizeEventAttributes (attributes = {}) { + const sanitizedAttributes = {} + + for (const key in attributes) { + const value = attributes[key] + if (Array.isArray(value)) { + const newArray = [] + for (const subkey in value) { + if (ALLOWED.includes(typeof value[subkey])) { + newArray.push(value[subkey]) + } else { + log.warn('Dropping span event attribute. It is not of an allowed type') + } + } + sanitizedAttributes[key] = newArray + } else if (ALLOWED.includes(typeof value)) { + sanitizedAttributes[key] = value + } else { + log.warn('Dropping span event attribute. It is not of an allowed type') + } + } + return sanitizedAttributes + } + _createContext (parent, fields) { let spanContext let startTime @@ -226,6 +309,8 @@ class DatadogSpan { if (startTime) { spanContext._trace.startTime = startTime } + // SpanContext was NOT propagated from a remote parent + spanContext._isRemote = false return spanContext } diff --git a/packages/dd-trace/src/opentracing/span_context.js b/packages/dd-trace/src/opentracing/span_context.js index aaa0ae26bc0..207c97080bb 100644 --- a/packages/dd-trace/src/opentracing/span_context.js +++ b/packages/dd-trace/src/opentracing/span_context.js @@ -11,6 +11,7 @@ class DatadogSpanContext { this._traceId = props.traceId this._spanId = props.spanId + this._isRemote = props.isRemote ?? true this._parentId = props.parentId || null this._name = props.name this._isFinished = props.isFinished || false @@ -26,22 +27,29 @@ class DatadogSpanContext { finished: [], tags: {} } + this._otelSpanContext = undefined } - toTraceId () { + toTraceId (get128bitId = false) { + if (get128bitId) { + return this._traceId.toBuffer().length <= 8 && this._trace.tags[TRACE_ID_128] + ? this._trace.tags[TRACE_ID_128] + this._traceId.toString(16).padStart(16, '0') + : this._traceId.toString(16).padStart(32, '0') + } return this._traceId.toString(10) } - toSpanId () { + toSpanId (get128bitId = false) { + if (get128bitId) { + return this._spanId.toString(16).padStart(16, '0') + } return this._spanId.toString(10) } toTraceparent () { const flags = this._sampling.priority >= AUTO_KEEP ? '01' : '00' - const traceId = this._traceId.toBuffer().length <= 8 && this._trace.tags[TRACE_ID_128] - ? this._trace.tags[TRACE_ID_128] + this._traceId.toString(16).padStart(16, '0') - : this._traceId.toString(16).padStart(32, '0') - const spanId = this._spanId.toString(16).padStart(16, '0') + const traceId = this.toTraceId(true) + const spanId = this.toSpanId(true) const version = (this._traceparent && this._traceparent.version) || '00' return `${version}-${traceId}-${spanId}-${flags}` } diff --git a/packages/dd-trace/src/opentracing/tracer.js b/packages/dd-trace/src/opentracing/tracer.js index 2a46d8a8c9a..2d854442cc3 100644 --- a/packages/dd-trace/src/opentracing/tracer.js +++ b/packages/dd-trace/src/opentracing/tracer.js @@ -5,6 +5,7 @@ const Span = require('./span') const SpanProcessor = require('../span_processor') const PrioritySampler = require('../priority_sampler') const TextMapPropagator = require('./propagation/text_map') +const DSMTextMapPropagator = require('./propagation/text_map_dsm') const HttpPropagator = require('./propagation/http') const BinaryPropagator = require('./propagation/binary') const LogPropagator = require('./propagation/log') @@ -19,16 +20,16 @@ const REFERENCE_CHILD_OF = 'child_of' const REFERENCE_FOLLOWS_FROM = 'follows_from' class DatadogTracer { - constructor (config) { + constructor (config, prioritySampler) { const Exporter = getExporter(config.experimental.exporter) + this._config = config this._service = config.service this._version = config.version this._env = config.env - this._tags = config.tags this._logInjection = config.logInjection this._debug = config.debug - this._prioritySampler = new PrioritySampler(config.env, config.sampler) + this._prioritySampler = prioritySampler ?? new PrioritySampler(config.env, config.sampler) this._exporter = new Exporter(config, this._prioritySampler) this._processor = new SpanProcessor(this._exporter, this._prioritySampler, config) this._url = this._exporter._url @@ -38,7 +39,8 @@ class DatadogTracer { [formats.TEXT_MAP]: new TextMapPropagator(config), [formats.HTTP_HEADERS]: new HttpPropagator(config), [formats.BINARY]: new BinaryPropagator(config), - [formats.LOG]: new LogPropagator(config) + [formats.LOG]: new LogPropagator(config), + [formats.TEXT_MAP_DSM]: new DSMTextMapPropagator(config) } if (config.reportHostname) { this._hostname = os.hostname() @@ -50,8 +52,15 @@ class DatadogTracer { ? getContext(options.childOf) : getParent(options.references) + // as per spec, allow the setting of service name through options const tags = { - 'service.name': this._service + 'service.name': options?.tags?.service ? String(options.tags.service) : this._service + } + + // As per unified service tagging spec if a span is created with a service name different from the global + // service name it will not inherit the global version value + if (options?.tags?.service && options.tags.service !== this._service) { + options.tags.version = undefined } const span = new Span(this, this._processor, this._prioritySampler, { @@ -61,23 +70,26 @@ class DatadogTracer { startTime: options.startTime, hostname: this._hostname, traceId128BitGenerationEnabled: this._traceId128BitGenerationEnabled, - integrationName: options.integrationName + integrationName: options.integrationName, + links: options.links }, this._debug) - span.addTags(this._tags) + span.addTags(this._config.tags) span.addTags(options.tags) return span } - inject (spanContext, format, carrier) { - if (spanContext instanceof Span) { - spanContext = spanContext.context() + inject (context, format, carrier) { + if (context instanceof Span) { + context = context.context() } try { - this._prioritySampler.sample(spanContext) - this._propagators[format].inject(spanContext, carrier) + if (format !== 'text_map_dsm') { + this._prioritySampler.sample(context) + } + this._propagators[format].inject(context, carrier) } catch (e) { log.error(e) runtimeMetrics.increment('datadog.tracer.node.inject.errors', true) diff --git a/packages/dd-trace/src/payload-tagging/config/aws.json b/packages/dd-trace/src/payload-tagging/config/aws.json new file mode 100644 index 00000000000..400b25bf670 --- /dev/null +++ b/packages/dd-trace/src/payload-tagging/config/aws.json @@ -0,0 +1,30 @@ +{ + "sns": { + "request": [ + "$.Attributes.KmsMasterKeyId", + "$.Attributes.PlatformCredential", + "$.Attributes.PlatformPrincipal", + "$.Attributes.Token", + "$.AWSAccountId", + "$.Endpoint", + "$.OneTimePassword", + "$.phoneNumber", + "$.PhoneNumber", + "$.Token" + ], + "response": [ + "$.Attributes.KmsMasterKeyId", + "$.Attributes.Token", + "$.Endpoints.*.Token", + "$.PhoneNumber", + "$.PhoneNumbers", + "$.phoneNumbers", + "$.PlatformApplication.*.PlatformCredential", + "$.PlatformApplication.*.PlatformPrincipal", + "$.Subscriptions.*.Endpoint" + ], + "expand": [ + "$.MessageAttributes.*.StringValue" + ] + } +} diff --git a/packages/dd-trace/src/payload-tagging/config/index.js b/packages/dd-trace/src/payload-tagging/config/index.js new file mode 100644 index 00000000000..16ab4dfd814 --- /dev/null +++ b/packages/dd-trace/src/payload-tagging/config/index.js @@ -0,0 +1,30 @@ +const aws = require('./aws.json') +const sdks = { aws } + +function getSDKRules (sdk, requestInput, responseInput) { + return Object.fromEntries( + Object.entries(sdk).map(([service, serviceRules]) => { + return [ + service, + { + request: serviceRules.request.concat(requestInput || []), + response: serviceRules.response.concat(responseInput || []), + expand: serviceRules.expand || [] + } + ] + }) + ) +} + +function appendRules (requestInput, responseInput) { + return Object.fromEntries( + Object.entries(sdks).map(([name, sdk]) => { + return [ + name, + getSDKRules(sdk, requestInput, responseInput) + ] + }) + ) +} + +module.exports = { appendRules } diff --git a/packages/dd-trace/src/payload-tagging/index.js b/packages/dd-trace/src/payload-tagging/index.js new file mode 100644 index 00000000000..71183443443 --- /dev/null +++ b/packages/dd-trace/src/payload-tagging/index.js @@ -0,0 +1,93 @@ +const rfdc = require('rfdc')({ proto: false, circles: false }) + +const { + PAYLOAD_TAG_REQUEST_PREFIX, + PAYLOAD_TAG_RESPONSE_PREFIX +} = require('../constants') + +const jsonpath = require('./jsonpath-plus.js').JSONPath + +const { tagsFromObject } = require('./tagging') + +/** + * Given an identified value, attempt to parse it as JSON if relevant + * + * @param {any} value + * @returns {any} the parsed object if parsing was successful, the input if not + */ +function maybeJSONParseValue (value) { + if (typeof value !== 'string' || value[0] !== '{') { + return value + } + + try { + return JSON.parse(value) + } catch (e) { + return value + } +} + +/** + * Apply expansion to all expansion JSONPath queries + * + * @param {Object} object + * @param {[String]} expansionRules list of JSONPath queries + */ +function expand (object, expansionRules) { + for (const rule of expansionRules) { + jsonpath(rule, object, (value, _type, desc) => { + desc.parent[desc.parentProperty] = maybeJSONParseValue(value) + }) + } +} + +/** + * Apply redaction to all redaction JSONPath queries + * + * @param {Object} object + * @param {[String]} redactionRules + */ +function redact (object, redactionRules) { + for (const rule of redactionRules) { + jsonpath(rule, object, (_value, _type, desc) => { + desc.parent[desc.parentProperty] = 'redacted' + }) + } +} + +/** + * Generate a map of tag names to tag values by performing: + * 1. Attempting to parse identified fields as JSON + * 2. Redacting fields identified by redaction rules + * 3. Flattening the resulting object, producing as many tag name/tag value pairs + * as there are leaf values in the object + * This function performs side-effects on a _copy_ of the input object. + * + * @param {Object} config sdk configuration for the service + * @param {[String]} config.expand expansion rules for the service + * @param {[String]} config.request redaction rules for the request + * @param {[String]} config.response redaction rules for the response + * @param {Object} object the input object to generate tags from + * @param {Object} opts tag generation options + * @param {String} opts.prefix prefix for all generated tags + * @param {number} opts.maxDepth maximum depth to traverse the object + * @returns + */ +function computeTags (config, object, opts) { + const payload = rfdc(object) + const redactionRules = opts.prefix === PAYLOAD_TAG_REQUEST_PREFIX ? config.request : config.response + const expansionRules = config.expand + expand(payload, expansionRules) + redact(payload, redactionRules) + return tagsFromObject(payload, opts) +} + +function tagsFromRequest (config, object, opts) { + return computeTags(config, object, { ...opts, prefix: PAYLOAD_TAG_REQUEST_PREFIX }) +} + +function tagsFromResponse (config, object, opts) { + return computeTags(config, object, { ...opts, prefix: PAYLOAD_TAG_RESPONSE_PREFIX }) +} + +module.exports = { computeTags, tagsFromRequest, tagsFromResponse } diff --git a/packages/dd-trace/src/payload-tagging/jsonpath-plus.js b/packages/dd-trace/src/payload-tagging/jsonpath-plus.js new file mode 100644 index 00000000000..85249b8210d --- /dev/null +++ b/packages/dd-trace/src/payload-tagging/jsonpath-plus.js @@ -0,0 +1,2094 @@ +'use strict'; + +// NOTE(bengl): This file is taken directly from jsonpath-plus@10.0.0 +// +// https://github.com/JSONPath-Plus/JSONPath/blob/a04dcbac760fed48760b09f387874a36f289c3f3/dist/index-node-cjs.cjs +// +// The only changes are: +// - Replace Object.hasOwn with polyfill +// +// This vendoring-and-editing was done to support usage on Node.js 16.0.0, so +// once support for that release line has ended, this can be replaced with a +// direct dependency on jsonpath-plus@^10. See the PR that introduced this file +// for details. More explicitly as a searchable to-do: +// +// TODO(bengl): Replace this with a direct dependency on jsonpath-plus@^10 when +// we drop support for Node 16. + +// NOTE(bengl): Here is the license as distributed with jsonpath-plus@10: +/* +MIT License + +Copyright (c) 2011-2019 Stefan Goessner, Subbu Allamaraju, Mike Brevoort, +Robert Krahn, Brett Zamir, Richard Schneider + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +const hasOwn = Object.hasOwn || ((obj, prop) => Object.prototype.hasOwnProperty.call(obj, prop)); + +var vm = require('vm'); + +/** + * @implements {IHooks} + */ +class Hooks { + /** + * @callback HookCallback + * @this {*|Jsep} this + * @param {Jsep} env + * @returns: void + */ + /** + * Adds the given callback to the list of callbacks for the given hook. + * + * The callback will be invoked when the hook it is registered for is run. + * + * One callback function can be registered to multiple hooks and the same hook multiple times. + * + * @param {string|object} name The name of the hook, or an object of callbacks keyed by name + * @param {HookCallback|boolean} callback The callback function which is given environment variables. + * @param {?boolean} [first=false] Will add the hook to the top of the list (defaults to the bottom) + * @public + */ + add(name, callback, first) { + if (typeof arguments[0] != 'string') { + // Multiple hook callbacks, keyed by name + for (let name in arguments[0]) { + this.add(name, arguments[0][name], arguments[1]); + } + } else { + (Array.isArray(name) ? name : [name]).forEach(function (name) { + this[name] = this[name] || []; + if (callback) { + this[name][first ? 'unshift' : 'push'](callback); + } + }, this); + } + } + + /** + * Runs a hook invoking all registered callbacks with the given environment variables. + * + * Callbacks will be invoked synchronously and in the order in which they were registered. + * + * @param {string} name The name of the hook. + * @param {Object} env The environment variables of the hook passed to all callbacks registered. + * @public + */ + run(name, env) { + this[name] = this[name] || []; + this[name].forEach(function (callback) { + callback.call(env && env.context ? env.context : env, env); + }); + } +} + +/** + * @implements {IPlugins} + */ +class Plugins { + constructor(jsep) { + this.jsep = jsep; + this.registered = {}; + } + + /** + * @callback PluginSetup + * @this {Jsep} jsep + * @returns: void + */ + /** + * Adds the given plugin(s) to the registry + * + * @param {object} plugins + * @param {string} plugins.name The name of the plugin + * @param {PluginSetup} plugins.init The init function + * @public + */ + register(...plugins) { + plugins.forEach(plugin => { + if (typeof plugin !== 'object' || !plugin.name || !plugin.init) { + throw new Error('Invalid JSEP plugin format'); + } + if (this.registered[plugin.name]) { + // already registered. Ignore. + return; + } + plugin.init(this.jsep); + this.registered[plugin.name] = plugin; + }); + } +} + +// JavaScript Expression Parser (JSEP) 1.3.9 + +class Jsep { + /** + * @returns {string} + */ + static get version() { + // To be filled in by the template + return '1.3.9'; + } + + /** + * @returns {string} + */ + static toString() { + return 'JavaScript Expression Parser (JSEP) v' + Jsep.version; + } + // ==================== CONFIG ================================ + /** + * @method addUnaryOp + * @param {string} op_name The name of the unary op to add + * @returns {Jsep} + */ + static addUnaryOp(op_name) { + Jsep.max_unop_len = Math.max(op_name.length, Jsep.max_unop_len); + Jsep.unary_ops[op_name] = 1; + return Jsep; + } + + /** + * @method jsep.addBinaryOp + * @param {string} op_name The name of the binary op to add + * @param {number} precedence The precedence of the binary op (can be a float). Higher number = higher precedence + * @param {boolean} [isRightAssociative=false] whether operator is right-associative + * @returns {Jsep} + */ + static addBinaryOp(op_name, precedence, isRightAssociative) { + Jsep.max_binop_len = Math.max(op_name.length, Jsep.max_binop_len); + Jsep.binary_ops[op_name] = precedence; + if (isRightAssociative) { + Jsep.right_associative.add(op_name); + } else { + Jsep.right_associative.delete(op_name); + } + return Jsep; + } + + /** + * @method addIdentifierChar + * @param {string} char The additional character to treat as a valid part of an identifier + * @returns {Jsep} + */ + static addIdentifierChar(char) { + Jsep.additional_identifier_chars.add(char); + return Jsep; + } + + /** + * @method addLiteral + * @param {string} literal_name The name of the literal to add + * @param {*} literal_value The value of the literal + * @returns {Jsep} + */ + static addLiteral(literal_name, literal_value) { + Jsep.literals[literal_name] = literal_value; + return Jsep; + } + + /** + * @method removeUnaryOp + * @param {string} op_name The name of the unary op to remove + * @returns {Jsep} + */ + static removeUnaryOp(op_name) { + delete Jsep.unary_ops[op_name]; + if (op_name.length === Jsep.max_unop_len) { + Jsep.max_unop_len = Jsep.getMaxKeyLen(Jsep.unary_ops); + } + return Jsep; + } + + /** + * @method removeAllUnaryOps + * @returns {Jsep} + */ + static removeAllUnaryOps() { + Jsep.unary_ops = {}; + Jsep.max_unop_len = 0; + return Jsep; + } + + /** + * @method removeIdentifierChar + * @param {string} char The additional character to stop treating as a valid part of an identifier + * @returns {Jsep} + */ + static removeIdentifierChar(char) { + Jsep.additional_identifier_chars.delete(char); + return Jsep; + } + + /** + * @method removeBinaryOp + * @param {string} op_name The name of the binary op to remove + * @returns {Jsep} + */ + static removeBinaryOp(op_name) { + delete Jsep.binary_ops[op_name]; + if (op_name.length === Jsep.max_binop_len) { + Jsep.max_binop_len = Jsep.getMaxKeyLen(Jsep.binary_ops); + } + Jsep.right_associative.delete(op_name); + return Jsep; + } + + /** + * @method removeAllBinaryOps + * @returns {Jsep} + */ + static removeAllBinaryOps() { + Jsep.binary_ops = {}; + Jsep.max_binop_len = 0; + return Jsep; + } + + /** + * @method removeLiteral + * @param {string} literal_name The name of the literal to remove + * @returns {Jsep} + */ + static removeLiteral(literal_name) { + delete Jsep.literals[literal_name]; + return Jsep; + } + + /** + * @method removeAllLiterals + * @returns {Jsep} + */ + static removeAllLiterals() { + Jsep.literals = {}; + return Jsep; + } + // ==================== END CONFIG ============================ + + /** + * @returns {string} + */ + get char() { + return this.expr.charAt(this.index); + } + + /** + * @returns {number} + */ + get code() { + return this.expr.charCodeAt(this.index); + } + /** + * @param {string} expr a string with the passed in express + * @returns Jsep + */ + constructor(expr) { + // `index` stores the character number we are currently at + // All of the gobbles below will modify `index` as we move along + this.expr = expr; + this.index = 0; + } + + /** + * static top-level parser + * @returns {jsep.Expression} + */ + static parse(expr) { + return new Jsep(expr).parse(); + } + + /** + * Get the longest key length of any object + * @param {object} obj + * @returns {number} + */ + static getMaxKeyLen(obj) { + return Math.max(0, ...Object.keys(obj).map(k => k.length)); + } + + /** + * `ch` is a character code in the next three functions + * @param {number} ch + * @returns {boolean} + */ + static isDecimalDigit(ch) { + return ch >= 48 && ch <= 57; // 0...9 + } + + /** + * Returns the precedence of a binary operator or `0` if it isn't a binary operator. Can be float. + * @param {string} op_val + * @returns {number} + */ + static binaryPrecedence(op_val) { + return Jsep.binary_ops[op_val] || 0; + } + + /** + * Looks for start of identifier + * @param {number} ch + * @returns {boolean} + */ + static isIdentifierStart(ch) { + return ch >= 65 && ch <= 90 || + // A...Z + ch >= 97 && ch <= 122 || + // a...z + ch >= 128 && !Jsep.binary_ops[String.fromCharCode(ch)] || + // any non-ASCII that is not an operator + Jsep.additional_identifier_chars.has(String.fromCharCode(ch)); // additional characters + } + + /** + * @param {number} ch + * @returns {boolean} + */ + static isIdentifierPart(ch) { + return Jsep.isIdentifierStart(ch) || Jsep.isDecimalDigit(ch); + } + + /** + * throw error at index of the expression + * @param {string} message + * @throws + */ + throwError(message) { + const error = new Error(message + ' at character ' + this.index); + error.index = this.index; + error.description = message; + throw error; + } + + /** + * Run a given hook + * @param {string} name + * @param {jsep.Expression|false} [node] + * @returns {?jsep.Expression} + */ + runHook(name, node) { + if (Jsep.hooks[name]) { + const env = { + context: this, + node + }; + Jsep.hooks.run(name, env); + return env.node; + } + return node; + } + + /** + * Runs a given hook until one returns a node + * @param {string} name + * @returns {?jsep.Expression} + */ + searchHook(name) { + if (Jsep.hooks[name]) { + const env = { + context: this + }; + Jsep.hooks[name].find(function (callback) { + callback.call(env.context, env); + return env.node; + }); + return env.node; + } + } + + /** + * Push `index` up to the next non-space character + */ + gobbleSpaces() { + let ch = this.code; + // Whitespace + while (ch === Jsep.SPACE_CODE || ch === Jsep.TAB_CODE || ch === Jsep.LF_CODE || ch === Jsep.CR_CODE) { + ch = this.expr.charCodeAt(++this.index); + } + this.runHook('gobble-spaces'); + } + + /** + * Top-level method to parse all expressions and returns compound or single node + * @returns {jsep.Expression} + */ + parse() { + this.runHook('before-all'); + const nodes = this.gobbleExpressions(); + + // If there's only one expression just try returning the expression + const node = nodes.length === 1 ? nodes[0] : { + type: Jsep.COMPOUND, + body: nodes + }; + return this.runHook('after-all', node); + } + + /** + * top-level parser (but can be reused within as well) + * @param {number} [untilICode] + * @returns {jsep.Expression[]} + */ + gobbleExpressions(untilICode) { + let nodes = [], + ch_i, + node; + while (this.index < this.expr.length) { + ch_i = this.code; + + // Expressions can be separated by semicolons, commas, or just inferred without any + // separators + if (ch_i === Jsep.SEMCOL_CODE || ch_i === Jsep.COMMA_CODE) { + this.index++; // ignore separators + } else { + // Try to gobble each expression individually + if (node = this.gobbleExpression()) { + nodes.push(node); + // If we weren't able to find a binary expression and are out of room, then + // the expression passed in probably has too much + } else if (this.index < this.expr.length) { + if (ch_i === untilICode) { + break; + } + this.throwError('Unexpected "' + this.char + '"'); + } + } + } + return nodes; + } + + /** + * The main parsing function. + * @returns {?jsep.Expression} + */ + gobbleExpression() { + const node = this.searchHook('gobble-expression') || this.gobbleBinaryExpression(); + this.gobbleSpaces(); + return this.runHook('after-expression', node); + } + + /** + * Search for the operation portion of the string (e.g. `+`, `===`) + * Start by taking the longest possible binary operations (3 characters: `===`, `!==`, `>>>`) + * and move down from 3 to 2 to 1 character until a matching binary operation is found + * then, return that binary operation + * @returns {string|boolean} + */ + gobbleBinaryOp() { + this.gobbleSpaces(); + let to_check = this.expr.substr(this.index, Jsep.max_binop_len); + let tc_len = to_check.length; + while (tc_len > 0) { + // Don't accept a binary op when it is an identifier. + // Binary ops that start with a identifier-valid character must be followed + // by a non identifier-part valid character + if (Jsep.binary_ops.hasOwnProperty(to_check) && (!Jsep.isIdentifierStart(this.code) || this.index + to_check.length < this.expr.length && !Jsep.isIdentifierPart(this.expr.charCodeAt(this.index + to_check.length)))) { + this.index += tc_len; + return to_check; + } + to_check = to_check.substr(0, --tc_len); + } + return false; + } + + /** + * This function is responsible for gobbling an individual expression, + * e.g. `1`, `1+2`, `a+(b*2)-Math.sqrt(2)` + * @returns {?jsep.BinaryExpression} + */ + gobbleBinaryExpression() { + let node, biop, prec, stack, biop_info, left, right, i, cur_biop; + + // First, try to get the leftmost thing + // Then, check to see if there's a binary operator operating on that leftmost thing + // Don't gobbleBinaryOp without a left-hand-side + left = this.gobbleToken(); + if (!left) { + return left; + } + biop = this.gobbleBinaryOp(); + + // If there wasn't a binary operator, just return the leftmost node + if (!biop) { + return left; + } + + // Otherwise, we need to start a stack to properly place the binary operations in their + // precedence structure + biop_info = { + value: biop, + prec: Jsep.binaryPrecedence(biop), + right_a: Jsep.right_associative.has(biop) + }; + right = this.gobbleToken(); + if (!right) { + this.throwError("Expected expression after " + biop); + } + stack = [left, biop_info, right]; + + // Properly deal with precedence using [recursive descent](http://www.engr.mun.ca/~theo/Misc/exp_parsing.htm) + while (biop = this.gobbleBinaryOp()) { + prec = Jsep.binaryPrecedence(biop); + if (prec === 0) { + this.index -= biop.length; + break; + } + biop_info = { + value: biop, + prec, + right_a: Jsep.right_associative.has(biop) + }; + cur_biop = biop; + + // Reduce: make a binary expression from the three topmost entries. + const comparePrev = prev => biop_info.right_a && prev.right_a ? prec > prev.prec : prec <= prev.prec; + while (stack.length > 2 && comparePrev(stack[stack.length - 2])) { + right = stack.pop(); + biop = stack.pop().value; + left = stack.pop(); + node = { + type: Jsep.BINARY_EXP, + operator: biop, + left, + right + }; + stack.push(node); + } + node = this.gobbleToken(); + if (!node) { + this.throwError("Expected expression after " + cur_biop); + } + stack.push(biop_info, node); + } + i = stack.length - 1; + node = stack[i]; + while (i > 1) { + node = { + type: Jsep.BINARY_EXP, + operator: stack[i - 1].value, + left: stack[i - 2], + right: node + }; + i -= 2; + } + return node; + } + + /** + * An individual part of a binary expression: + * e.g. `foo.bar(baz)`, `1`, `"abc"`, `(a % 2)` (because it's in parenthesis) + * @returns {boolean|jsep.Expression} + */ + gobbleToken() { + let ch, to_check, tc_len, node; + this.gobbleSpaces(); + node = this.searchHook('gobble-token'); + if (node) { + return this.runHook('after-token', node); + } + ch = this.code; + if (Jsep.isDecimalDigit(ch) || ch === Jsep.PERIOD_CODE) { + // Char code 46 is a dot `.` which can start off a numeric literal + return this.gobbleNumericLiteral(); + } + if (ch === Jsep.SQUOTE_CODE || ch === Jsep.DQUOTE_CODE) { + // Single or double quotes + node = this.gobbleStringLiteral(); + } else if (ch === Jsep.OBRACK_CODE) { + node = this.gobbleArray(); + } else { + to_check = this.expr.substr(this.index, Jsep.max_unop_len); + tc_len = to_check.length; + while (tc_len > 0) { + // Don't accept an unary op when it is an identifier. + // Unary ops that start with a identifier-valid character must be followed + // by a non identifier-part valid character + if (Jsep.unary_ops.hasOwnProperty(to_check) && (!Jsep.isIdentifierStart(this.code) || this.index + to_check.length < this.expr.length && !Jsep.isIdentifierPart(this.expr.charCodeAt(this.index + to_check.length)))) { + this.index += tc_len; + const argument = this.gobbleToken(); + if (!argument) { + this.throwError('missing unaryOp argument'); + } + return this.runHook('after-token', { + type: Jsep.UNARY_EXP, + operator: to_check, + argument, + prefix: true + }); + } + to_check = to_check.substr(0, --tc_len); + } + if (Jsep.isIdentifierStart(ch)) { + node = this.gobbleIdentifier(); + if (Jsep.literals.hasOwnProperty(node.name)) { + node = { + type: Jsep.LITERAL, + value: Jsep.literals[node.name], + raw: node.name + }; + } else if (node.name === Jsep.this_str) { + node = { + type: Jsep.THIS_EXP + }; + } + } else if (ch === Jsep.OPAREN_CODE) { + // open parenthesis + node = this.gobbleGroup(); + } + } + if (!node) { + return this.runHook('after-token', false); + } + node = this.gobbleTokenProperty(node); + return this.runHook('after-token', node); + } + + /** + * Gobble properties of of identifiers/strings/arrays/groups. + * e.g. `foo`, `bar.baz`, `foo['bar'].baz` + * It also gobbles function calls: + * e.g. `Math.acos(obj.angle)` + * @param {jsep.Expression} node + * @returns {jsep.Expression} + */ + gobbleTokenProperty(node) { + this.gobbleSpaces(); + let ch = this.code; + while (ch === Jsep.PERIOD_CODE || ch === Jsep.OBRACK_CODE || ch === Jsep.OPAREN_CODE || ch === Jsep.QUMARK_CODE) { + let optional; + if (ch === Jsep.QUMARK_CODE) { + if (this.expr.charCodeAt(this.index + 1) !== Jsep.PERIOD_CODE) { + break; + } + optional = true; + this.index += 2; + this.gobbleSpaces(); + ch = this.code; + } + this.index++; + if (ch === Jsep.OBRACK_CODE) { + node = { + type: Jsep.MEMBER_EXP, + computed: true, + object: node, + property: this.gobbleExpression() + }; + if (!node.property) { + this.throwError('Unexpected "' + this.char + '"'); + } + this.gobbleSpaces(); + ch = this.code; + if (ch !== Jsep.CBRACK_CODE) { + this.throwError('Unclosed ['); + } + this.index++; + } else if (ch === Jsep.OPAREN_CODE) { + // A function call is being made; gobble all the arguments + node = { + type: Jsep.CALL_EXP, + 'arguments': this.gobbleArguments(Jsep.CPAREN_CODE), + callee: node + }; + } else if (ch === Jsep.PERIOD_CODE || optional) { + if (optional) { + this.index--; + } + this.gobbleSpaces(); + node = { + type: Jsep.MEMBER_EXP, + computed: false, + object: node, + property: this.gobbleIdentifier() + }; + } + if (optional) { + node.optional = true; + } // else leave undefined for compatibility with esprima + + this.gobbleSpaces(); + ch = this.code; + } + return node; + } + + /** + * Parse simple numeric literals: `12`, `3.4`, `.5`. Do this by using a string to + * keep track of everything in the numeric literal and then calling `parseFloat` on that string + * @returns {jsep.Literal} + */ + gobbleNumericLiteral() { + let number = '', + ch, + chCode; + while (Jsep.isDecimalDigit(this.code)) { + number += this.expr.charAt(this.index++); + } + if (this.code === Jsep.PERIOD_CODE) { + // can start with a decimal marker + number += this.expr.charAt(this.index++); + while (Jsep.isDecimalDigit(this.code)) { + number += this.expr.charAt(this.index++); + } + } + ch = this.char; + if (ch === 'e' || ch === 'E') { + // exponent marker + number += this.expr.charAt(this.index++); + ch = this.char; + if (ch === '+' || ch === '-') { + // exponent sign + number += this.expr.charAt(this.index++); + } + while (Jsep.isDecimalDigit(this.code)) { + // exponent itself + number += this.expr.charAt(this.index++); + } + if (!Jsep.isDecimalDigit(this.expr.charCodeAt(this.index - 1))) { + this.throwError('Expected exponent (' + number + this.char + ')'); + } + } + chCode = this.code; + + // Check to make sure this isn't a variable name that start with a number (123abc) + if (Jsep.isIdentifierStart(chCode)) { + this.throwError('Variable names cannot start with a number (' + number + this.char + ')'); + } else if (chCode === Jsep.PERIOD_CODE || number.length === 1 && number.charCodeAt(0) === Jsep.PERIOD_CODE) { + this.throwError('Unexpected period'); + } + return { + type: Jsep.LITERAL, + value: parseFloat(number), + raw: number + }; + } + + /** + * Parses a string literal, staring with single or double quotes with basic support for escape codes + * e.g. `"hello world"`, `'this is\nJSEP'` + * @returns {jsep.Literal} + */ + gobbleStringLiteral() { + let str = ''; + const startIndex = this.index; + const quote = this.expr.charAt(this.index++); + let closed = false; + while (this.index < this.expr.length) { + let ch = this.expr.charAt(this.index++); + if (ch === quote) { + closed = true; + break; + } else if (ch === '\\') { + // Check for all of the common escape codes + ch = this.expr.charAt(this.index++); + switch (ch) { + case 'n': + str += '\n'; + break; + case 'r': + str += '\r'; + break; + case 't': + str += '\t'; + break; + case 'b': + str += '\b'; + break; + case 'f': + str += '\f'; + break; + case 'v': + str += '\x0B'; + break; + default: + str += ch; + } + } else { + str += ch; + } + } + if (!closed) { + this.throwError('Unclosed quote after "' + str + '"'); + } + return { + type: Jsep.LITERAL, + value: str, + raw: this.expr.substring(startIndex, this.index) + }; + } + + /** + * Gobbles only identifiers + * e.g.: `foo`, `_value`, `$x1` + * Also, this function checks if that identifier is a literal: + * (e.g. `true`, `false`, `null`) or `this` + * @returns {jsep.Identifier} + */ + gobbleIdentifier() { + let ch = this.code, + start = this.index; + if (Jsep.isIdentifierStart(ch)) { + this.index++; + } else { + this.throwError('Unexpected ' + this.char); + } + while (this.index < this.expr.length) { + ch = this.code; + if (Jsep.isIdentifierPart(ch)) { + this.index++; + } else { + break; + } + } + return { + type: Jsep.IDENTIFIER, + name: this.expr.slice(start, this.index) + }; + } + + /** + * Gobbles a list of arguments within the context of a function call + * or array literal. This function also assumes that the opening character + * `(` or `[` has already been gobbled, and gobbles expressions and commas + * until the terminator character `)` or `]` is encountered. + * e.g. `foo(bar, baz)`, `my_func()`, or `[bar, baz]` + * @param {number} termination + * @returns {jsep.Expression[]} + */ + gobbleArguments(termination) { + const args = []; + let closed = false; + let separator_count = 0; + while (this.index < this.expr.length) { + this.gobbleSpaces(); + let ch_i = this.code; + if (ch_i === termination) { + // done parsing + closed = true; + this.index++; + if (termination === Jsep.CPAREN_CODE && separator_count && separator_count >= args.length) { + this.throwError('Unexpected token ' + String.fromCharCode(termination)); + } + break; + } else if (ch_i === Jsep.COMMA_CODE) { + // between expressions + this.index++; + separator_count++; + if (separator_count !== args.length) { + // missing argument + if (termination === Jsep.CPAREN_CODE) { + this.throwError('Unexpected token ,'); + } else if (termination === Jsep.CBRACK_CODE) { + for (let arg = args.length; arg < separator_count; arg++) { + args.push(null); + } + } + } + } else if (args.length !== separator_count && separator_count !== 0) { + // NOTE: `&& separator_count !== 0` allows for either all commas, or all spaces as arguments + this.throwError('Expected comma'); + } else { + const node = this.gobbleExpression(); + if (!node || node.type === Jsep.COMPOUND) { + this.throwError('Expected comma'); + } + args.push(node); + } + } + if (!closed) { + this.throwError('Expected ' + String.fromCharCode(termination)); + } + return args; + } + + /** + * Responsible for parsing a group of things within parentheses `()` + * that have no identifier in front (so not a function call) + * This function assumes that it needs to gobble the opening parenthesis + * and then tries to gobble everything within that parenthesis, assuming + * that the next thing it should see is the close parenthesis. If not, + * then the expression probably doesn't have a `)` + * @returns {boolean|jsep.Expression} + */ + gobbleGroup() { + this.index++; + let nodes = this.gobbleExpressions(Jsep.CPAREN_CODE); + if (this.code === Jsep.CPAREN_CODE) { + this.index++; + if (nodes.length === 1) { + return nodes[0]; + } else if (!nodes.length) { + return false; + } else { + return { + type: Jsep.SEQUENCE_EXP, + expressions: nodes + }; + } + } else { + this.throwError('Unclosed ('); + } + } + + /** + * Responsible for parsing Array literals `[1, 2, 3]` + * This function assumes that it needs to gobble the opening bracket + * and then tries to gobble the expressions as arguments. + * @returns {jsep.ArrayExpression} + */ + gobbleArray() { + this.index++; + return { + type: Jsep.ARRAY_EXP, + elements: this.gobbleArguments(Jsep.CBRACK_CODE) + }; + } +} + +// Static fields: +const hooks = new Hooks(); +Object.assign(Jsep, { + hooks, + plugins: new Plugins(Jsep), + // Node Types + // ---------- + // This is the full set of types that any JSEP node can be. + // Store them here to save space when minified + COMPOUND: 'Compound', + SEQUENCE_EXP: 'SequenceExpression', + IDENTIFIER: 'Identifier', + MEMBER_EXP: 'MemberExpression', + LITERAL: 'Literal', + THIS_EXP: 'ThisExpression', + CALL_EXP: 'CallExpression', + UNARY_EXP: 'UnaryExpression', + BINARY_EXP: 'BinaryExpression', + ARRAY_EXP: 'ArrayExpression', + TAB_CODE: 9, + LF_CODE: 10, + CR_CODE: 13, + SPACE_CODE: 32, + PERIOD_CODE: 46, + // '.' + COMMA_CODE: 44, + // ',' + SQUOTE_CODE: 39, + // single quote + DQUOTE_CODE: 34, + // double quotes + OPAREN_CODE: 40, + // ( + CPAREN_CODE: 41, + // ) + OBRACK_CODE: 91, + // [ + CBRACK_CODE: 93, + // ] + QUMARK_CODE: 63, + // ? + SEMCOL_CODE: 59, + // ; + COLON_CODE: 58, + // : + + // Operations + // ---------- + // Use a quickly-accessible map to store all of the unary operators + // Values are set to `1` (it really doesn't matter) + unary_ops: { + '-': 1, + '!': 1, + '~': 1, + '+': 1 + }, + // Also use a map for the binary operations but set their values to their + // binary precedence for quick reference (higher number = higher precedence) + // see [Order of operations](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Operator_Precedence) + binary_ops: { + '||': 1, + '&&': 2, + '|': 3, + '^': 4, + '&': 5, + '==': 6, + '!=': 6, + '===': 6, + '!==': 6, + '<': 7, + '>': 7, + '<=': 7, + '>=': 7, + '<<': 8, + '>>': 8, + '>>>': 8, + '+': 9, + '-': 9, + '*': 10, + '/': 10, + '%': 10 + }, + // sets specific binary_ops as right-associative + right_associative: new Set(), + // Additional valid identifier chars, apart from a-z, A-Z and 0-9 (except on the starting char) + additional_identifier_chars: new Set(['$', '_']), + // Literals + // ---------- + // Store the values to return for the various literals we may encounter + literals: { + 'true': true, + 'false': false, + 'null': null + }, + // Except for `this`, which is special. This could be changed to something like `'self'` as well + this_str: 'this' +}); +Jsep.max_unop_len = Jsep.getMaxKeyLen(Jsep.unary_ops); +Jsep.max_binop_len = Jsep.getMaxKeyLen(Jsep.binary_ops); + +// Backward Compatibility: +const jsep = expr => new Jsep(expr).parse(); +const stdClassProps = Object.getOwnPropertyNames(class Test {}); +Object.getOwnPropertyNames(Jsep).filter(prop => !stdClassProps.includes(prop) && jsep[prop] === undefined).forEach(m => { + jsep[m] = Jsep[m]; +}); +jsep.Jsep = Jsep; // allows for const { Jsep } = require('jsep'); + +const CONDITIONAL_EXP = 'ConditionalExpression'; +var ternary = { + name: 'ternary', + init(jsep) { + // Ternary expression: test ? consequent : alternate + jsep.hooks.add('after-expression', function gobbleTernary(env) { + if (env.node && this.code === jsep.QUMARK_CODE) { + this.index++; + const test = env.node; + const consequent = this.gobbleExpression(); + if (!consequent) { + this.throwError('Expected expression'); + } + this.gobbleSpaces(); + if (this.code === jsep.COLON_CODE) { + this.index++; + const alternate = this.gobbleExpression(); + if (!alternate) { + this.throwError('Expected expression'); + } + env.node = { + type: CONDITIONAL_EXP, + test, + consequent, + alternate + }; + + // check for operators of higher priority than ternary (i.e. assignment) + // jsep sets || at 1, and assignment at 0.9, and conditional should be between them + if (test.operator && jsep.binary_ops[test.operator] <= 0.9) { + let newTest = test; + while (newTest.right.operator && jsep.binary_ops[newTest.right.operator] <= 0.9) { + newTest = newTest.right; + } + env.node.test = newTest.right; + newTest.right = env.node; + env.node = test; + } + } else { + this.throwError('Expected :'); + } + } + }); + } +}; + +// Add default plugins: + +jsep.plugins.register(ternary); + +const FSLASH_CODE = 47; // '/' +const BSLASH_CODE = 92; // '\\' + +var index = { + name: 'regex', + init(jsep) { + // Regex literal: /abc123/ig + jsep.hooks.add('gobble-token', function gobbleRegexLiteral(env) { + if (this.code === FSLASH_CODE) { + const patternIndex = ++this.index; + let inCharSet = false; + while (this.index < this.expr.length) { + if (this.code === FSLASH_CODE && !inCharSet) { + const pattern = this.expr.slice(patternIndex, this.index); + let flags = ''; + while (++this.index < this.expr.length) { + const code = this.code; + if (code >= 97 && code <= 122 // a...z + || code >= 65 && code <= 90 // A...Z + || code >= 48 && code <= 57) { + // 0-9 + flags += this.char; + } else { + break; + } + } + let value; + try { + value = new RegExp(pattern, flags); + } catch (e) { + this.throwError(e.message); + } + env.node = { + type: jsep.LITERAL, + value, + raw: this.expr.slice(patternIndex - 1, this.index) + }; + + // allow . [] and () after regex: /regex/.test(a) + env.node = this.gobbleTokenProperty(env.node); + return env.node; + } + if (this.code === jsep.OBRACK_CODE) { + inCharSet = true; + } else if (inCharSet && this.code === jsep.CBRACK_CODE) { + inCharSet = false; + } + this.index += this.code === BSLASH_CODE ? 2 : 1; + } + this.throwError('Unclosed Regex'); + } + }); + } +}; + +const PLUS_CODE = 43; // + +const MINUS_CODE = 45; // - + +const plugin = { + name: 'assignment', + assignmentOperators: new Set(['=', '*=', '**=', '/=', '%=', '+=', '-=', '<<=', '>>=', '>>>=', '&=', '^=', '|=']), + updateOperators: [PLUS_CODE, MINUS_CODE], + assignmentPrecedence: 0.9, + init(jsep) { + const updateNodeTypes = [jsep.IDENTIFIER, jsep.MEMBER_EXP]; + plugin.assignmentOperators.forEach(op => jsep.addBinaryOp(op, plugin.assignmentPrecedence, true)); + jsep.hooks.add('gobble-token', function gobbleUpdatePrefix(env) { + const code = this.code; + if (plugin.updateOperators.some(c => c === code && c === this.expr.charCodeAt(this.index + 1))) { + this.index += 2; + env.node = { + type: 'UpdateExpression', + operator: code === PLUS_CODE ? '++' : '--', + argument: this.gobbleTokenProperty(this.gobbleIdentifier()), + prefix: true + }; + if (!env.node.argument || !updateNodeTypes.includes(env.node.argument.type)) { + this.throwError(`Unexpected ${env.node.operator}`); + } + } + }); + jsep.hooks.add('after-token', function gobbleUpdatePostfix(env) { + if (env.node) { + const code = this.code; + if (plugin.updateOperators.some(c => c === code && c === this.expr.charCodeAt(this.index + 1))) { + if (!updateNodeTypes.includes(env.node.type)) { + this.throwError(`Unexpected ${env.node.operator}`); + } + this.index += 2; + env.node = { + type: 'UpdateExpression', + operator: code === PLUS_CODE ? '++' : '--', + argument: env.node, + prefix: false + }; + } + } + }); + jsep.hooks.add('after-expression', function gobbleAssignment(env) { + if (env.node) { + // Note: Binaries can be chained in a single expression to respect + // operator precedence (i.e. a = b = 1 + 2 + 3) + // Update all binary assignment nodes in the tree + updateBinariesToAssignments(env.node); + } + }); + function updateBinariesToAssignments(node) { + if (plugin.assignmentOperators.has(node.operator)) { + node.type = 'AssignmentExpression'; + updateBinariesToAssignments(node.left); + updateBinariesToAssignments(node.right); + } else if (!node.operator) { + Object.values(node).forEach(val => { + if (val && typeof val === 'object') { + updateBinariesToAssignments(val); + } + }); + } + } + } +}; + +/* eslint-disable no-bitwise */ + +// register plugins +jsep.plugins.register(index, plugin); +const SafeEval = { + /** + * @param {jsep.Expression} ast + * @param {Record} subs + */ + evalAst(ast, subs) { + switch (ast.type) { + case 'BinaryExpression': + case 'LogicalExpression': + return SafeEval.evalBinaryExpression(ast, subs); + case 'Compound': + return SafeEval.evalCompound(ast, subs); + case 'ConditionalExpression': + return SafeEval.evalConditionalExpression(ast, subs); + case 'Identifier': + return SafeEval.evalIdentifier(ast, subs); + case 'Literal': + return SafeEval.evalLiteral(ast, subs); + case 'MemberExpression': + return SafeEval.evalMemberExpression(ast, subs); + case 'UnaryExpression': + return SafeEval.evalUnaryExpression(ast, subs); + case 'ArrayExpression': + return SafeEval.evalArrayExpression(ast, subs); + case 'CallExpression': + return SafeEval.evalCallExpression(ast, subs); + case 'AssignmentExpression': + return SafeEval.evalAssignmentExpression(ast, subs); + default: + throw SyntaxError('Unexpected expression', ast); + } + }, + evalBinaryExpression(ast, subs) { + const result = { + '||': (a, b) => a || b(), + '&&': (a, b) => a && b(), + '|': (a, b) => a | b(), + '^': (a, b) => a ^ b(), + '&': (a, b) => a & b(), + // eslint-disable-next-line eqeqeq + '==': (a, b) => a == b(), + // eslint-disable-next-line eqeqeq + '!=': (a, b) => a != b(), + '===': (a, b) => a === b(), + '!==': (a, b) => a !== b(), + '<': (a, b) => a < b(), + '>': (a, b) => a > b(), + '<=': (a, b) => a <= b(), + '>=': (a, b) => a >= b(), + '<<': (a, b) => a << b(), + '>>': (a, b) => a >> b(), + '>>>': (a, b) => a >>> b(), + '+': (a, b) => a + b(), + '-': (a, b) => a - b(), + '*': (a, b) => a * b(), + '/': (a, b) => a / b(), + '%': (a, b) => a % b() + }[ast.operator](SafeEval.evalAst(ast.left, subs), () => SafeEval.evalAst(ast.right, subs)); + return result; + }, + evalCompound(ast, subs) { + let last; + for (let i = 0; i < ast.body.length; i++) { + if (ast.body[i].type === 'Identifier' && ['var', 'let', 'const'].includes(ast.body[i].name) && ast.body[i + 1] && ast.body[i + 1].type === 'AssignmentExpression') { + // var x=2; is detected as + // [{Identifier var}, {AssignmentExpression x=2}] + // eslint-disable-next-line @stylistic/max-len -- Long + // eslint-disable-next-line sonarjs/updated-loop-counter -- Convenient + i += 1; + } + const expr = ast.body[i]; + last = SafeEval.evalAst(expr, subs); + } + return last; + }, + evalConditionalExpression(ast, subs) { + if (SafeEval.evalAst(ast.test, subs)) { + return SafeEval.evalAst(ast.consequent, subs); + } + return SafeEval.evalAst(ast.alternate, subs); + }, + evalIdentifier(ast, subs) { + if (ast.name in subs) { + return subs[ast.name]; + } + throw ReferenceError(`${ast.name} is not defined`); + }, + evalLiteral(ast) { + return ast.value; + }, + evalMemberExpression(ast, subs) { + const prop = ast.computed ? SafeEval.evalAst(ast.property) // `object[property]` + : ast.property.name; // `object.property` property is Identifier + const obj = SafeEval.evalAst(ast.object, subs); + const result = obj[prop]; + if (typeof result === 'function') { + return result.bind(obj); // arrow functions aren't affected by bind. + } + return result; + }, + evalUnaryExpression(ast, subs) { + const result = { + '-': a => -SafeEval.evalAst(a, subs), + '!': a => !SafeEval.evalAst(a, subs), + '~': a => ~SafeEval.evalAst(a, subs), + // eslint-disable-next-line no-implicit-coercion + '+': a => +SafeEval.evalAst(a, subs) + }[ast.operator](ast.argument); + return result; + }, + evalArrayExpression(ast, subs) { + return ast.elements.map(el => SafeEval.evalAst(el, subs)); + }, + evalCallExpression(ast, subs) { + const args = ast.arguments.map(arg => SafeEval.evalAst(arg, subs)); + const func = SafeEval.evalAst(ast.callee, subs); + return func(...args); + }, + evalAssignmentExpression(ast, subs) { + if (ast.left.type !== 'Identifier') { + throw SyntaxError('Invalid left-hand side in assignment'); + } + const id = ast.left.name; + const value = SafeEval.evalAst(ast.right, subs); + subs[id] = value; + return subs[id]; + } +}; + +/** + * A replacement for NodeJS' VM.Script which is also {@link https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP | Content Security Policy} friendly. + */ +class SafeScript { + /** + * @param {string} expr Expression to evaluate + */ + constructor(expr) { + this.code = expr; + this.ast = jsep(this.code); + } + + /** + * @param {object} context Object whose items will be added + * to evaluation + * @returns {EvaluatedResult} Result of evaluated code + */ + runInNewContext(context) { + const keyMap = { + ...context + }; + return SafeEval.evalAst(this.ast, keyMap); + } +} + +/* eslint-disable camelcase, unicorn/prefer-string-replace-all, + unicorn/prefer-at */ + + +/** + * @typedef {null|boolean|number|string|object|GenericArray} JSONObject + */ + +/** + * @typedef {any} AnyItem + */ + +/** + * @typedef {any} AnyResult + */ + +/** + * Copies array and then pushes item into it. + * @param {GenericArray} arr Array to copy and into which to push + * @param {AnyItem} item Array item to add (to end) + * @returns {GenericArray} Copy of the original array + */ +function push(arr, item) { + arr = arr.slice(); + arr.push(item); + return arr; +} +/** + * Copies array and then unshifts item into it. + * @param {AnyItem} item Array item to add (to beginning) + * @param {GenericArray} arr Array to copy and into which to unshift + * @returns {GenericArray} Copy of the original array + */ +function unshift(item, arr) { + arr = arr.slice(); + arr.unshift(item); + return arr; +} + +/** + * Caught when JSONPath is used without `new` but rethrown if with `new` + * @extends Error + */ +class NewError extends Error { + /** + * @param {AnyResult} value The evaluated scalar value + */ + constructor(value) { + super('JSONPath should not be called with "new" (it prevents return ' + 'of (unwrapped) scalar values)'); + this.avoidNew = true; + this.value = value; + this.name = 'NewError'; + } +} + +/** +* @typedef {object} ReturnObject +* @property {string} path +* @property {JSONObject} value +* @property {object|GenericArray} parent +* @property {string} parentProperty +*/ + +/** +* @callback JSONPathCallback +* @param {string|object} preferredOutput +* @param {"value"|"property"} type +* @param {ReturnObject} fullRetObj +* @returns {void} +*/ + +/** +* @callback OtherTypeCallback +* @param {JSONObject} val +* @param {string} path +* @param {object|GenericArray} parent +* @param {string} parentPropName +* @returns {boolean} +*/ + +/** + * @typedef {any} ContextItem + */ + +/** + * @typedef {any} EvaluatedResult + */ + +/** +* @callback EvalCallback +* @param {string} code +* @param {ContextItem} context +* @returns {EvaluatedResult} +*/ + +/** + * @typedef {typeof SafeScript} EvalClass + */ + +/** + * @typedef {object} JSONPathOptions + * @property {JSON} json + * @property {string|string[]} path + * @property {"value"|"path"|"pointer"|"parent"|"parentProperty"| + * "all"} [resultType="value"] + * @property {boolean} [flatten=false] + * @property {boolean} [wrap=true] + * @property {object} [sandbox={}] + * @property {EvalCallback|EvalClass|'safe'|'native'| + * boolean} [eval = 'safe'] + * @property {object|GenericArray|null} [parent=null] + * @property {string|null} [parentProperty=null] + * @property {JSONPathCallback} [callback] + * @property {OtherTypeCallback} [otherTypeCallback] Defaults to + * function which throws on encountering `@other` + * @property {boolean} [autostart=true] + */ + +/** + * @param {string|JSONPathOptions} opts If a string, will be treated as `expr` + * @param {string} [expr] JSON path to evaluate + * @param {JSON} [obj] JSON object to evaluate against + * @param {JSONPathCallback} [callback] Passed 3 arguments: 1) desired payload + * per `resultType`, 2) `"value"|"property"`, 3) Full returned object with + * all payloads + * @param {OtherTypeCallback} [otherTypeCallback] If `@other()` is at the end + * of one's query, this will be invoked with the value of the item, its + * path, its parent, and its parent's property name, and it should return + * a boolean indicating whether the supplied value belongs to the "other" + * type or not (or it may handle transformations and return `false`). + * @returns {JSONPath} + * @class + */ +function JSONPath(opts, expr, obj, callback, otherTypeCallback) { + // eslint-disable-next-line no-restricted-syntax + if (!(this instanceof JSONPath)) { + try { + return new JSONPath(opts, expr, obj, callback, otherTypeCallback); + } catch (e) { + if (!e.avoidNew) { + throw e; + } + return e.value; + } + } + if (typeof opts === 'string') { + otherTypeCallback = callback; + callback = obj; + obj = expr; + expr = opts; + opts = null; + } + const optObj = opts && typeof opts === 'object'; + opts = opts || {}; + this.json = opts.json || obj; + this.path = opts.path || expr; + this.resultType = opts.resultType || 'value'; + this.flatten = opts.flatten || false; + this.wrap = hasOwn(opts, 'wrap') ? opts.wrap : true; + this.sandbox = opts.sandbox || {}; + this.eval = opts.eval === undefined ? 'safe' : opts.eval; + this.ignoreEvalErrors = typeof opts.ignoreEvalErrors === 'undefined' ? false : opts.ignoreEvalErrors; + this.parent = opts.parent || null; + this.parentProperty = opts.parentProperty || null; + this.callback = opts.callback || callback || null; + this.otherTypeCallback = opts.otherTypeCallback || otherTypeCallback || function () { + throw new TypeError('You must supply an otherTypeCallback callback option ' + 'with the @other() operator.'); + }; + if (opts.autostart !== false) { + const args = { + path: optObj ? opts.path : expr + }; + if (!optObj) { + args.json = obj; + } else if ('json' in opts) { + args.json = opts.json; + } + const ret = this.evaluate(args); + if (!ret || typeof ret !== 'object') { + throw new NewError(ret); + } + return ret; + } +} + +// PUBLIC METHODS +JSONPath.prototype.evaluate = function (expr, json, callback, otherTypeCallback) { + let currParent = this.parent, + currParentProperty = this.parentProperty; + let { + flatten, + wrap + } = this; + this.currResultType = this.resultType; + this.currEval = this.eval; + this.currSandbox = this.sandbox; + callback = callback || this.callback; + this.currOtherTypeCallback = otherTypeCallback || this.otherTypeCallback; + json = json || this.json; + expr = expr || this.path; + if (expr && typeof expr === 'object' && !Array.isArray(expr)) { + if (!expr.path && expr.path !== '') { + throw new TypeError('You must supply a "path" property when providing an object ' + 'argument to JSONPath.evaluate().'); + } + if (!hasOwn(expr, 'json')) { + throw new TypeError('You must supply a "json" property when providing an object ' + 'argument to JSONPath.evaluate().'); + } + ({ + json + } = expr); + flatten = hasOwn(expr, 'flatten') ? expr.flatten : flatten; + this.currResultType = hasOwn(expr, 'resultType') ? expr.resultType : this.currResultType; + this.currSandbox = hasOwn(expr, 'sandbox') ? expr.sandbox : this.currSandbox; + wrap = hasOwn(expr, 'wrap') ? expr.wrap : wrap; + this.currEval = hasOwn(expr, 'eval') ? expr.eval : this.currEval; + callback = hasOwn(expr, 'callback') ? expr.callback : callback; + this.currOtherTypeCallback = hasOwn(expr, 'otherTypeCallback') ? expr.otherTypeCallback : this.currOtherTypeCallback; + currParent = hasOwn(expr, 'parent') ? expr.parent : currParent; + currParentProperty = hasOwn(expr, 'parentProperty') ? expr.parentProperty : currParentProperty; + expr = expr.path; + } + currParent = currParent || null; + currParentProperty = currParentProperty || null; + if (Array.isArray(expr)) { + expr = JSONPath.toPathString(expr); + } + if (!expr && expr !== '' || !json) { + return undefined; + } + const exprList = JSONPath.toPathArray(expr); + if (exprList[0] === '$' && exprList.length > 1) { + exprList.shift(); + } + this._hasParentSelector = null; + const result = this._trace(exprList, json, ['$'], currParent, currParentProperty, callback).filter(function (ea) { + return ea && !ea.isParentSelector; + }); + if (!result.length) { + return wrap ? [] : undefined; + } + if (!wrap && result.length === 1 && !result[0].hasArrExpr) { + return this._getPreferredOutput(result[0]); + } + return result.reduce((rslt, ea) => { + const valOrPath = this._getPreferredOutput(ea); + if (flatten && Array.isArray(valOrPath)) { + rslt = rslt.concat(valOrPath); + } else { + rslt.push(valOrPath); + } + return rslt; + }, []); +}; + +// PRIVATE METHODS + +JSONPath.prototype._getPreferredOutput = function (ea) { + const resultType = this.currResultType; + switch (resultType) { + case 'all': + { + const path = Array.isArray(ea.path) ? ea.path : JSONPath.toPathArray(ea.path); + ea.pointer = JSONPath.toPointer(path); + ea.path = typeof ea.path === 'string' ? ea.path : JSONPath.toPathString(ea.path); + return ea; + } + case 'value': + case 'parent': + case 'parentProperty': + return ea[resultType]; + case 'path': + return JSONPath.toPathString(ea[resultType]); + case 'pointer': + return JSONPath.toPointer(ea.path); + default: + throw new TypeError('Unknown result type'); + } +}; +JSONPath.prototype._handleCallback = function (fullRetObj, callback, type) { + if (callback) { + const preferredOutput = this._getPreferredOutput(fullRetObj); + fullRetObj.path = typeof fullRetObj.path === 'string' ? fullRetObj.path : JSONPath.toPathString(fullRetObj.path); + // eslint-disable-next-line n/callback-return + callback(preferredOutput, type, fullRetObj); + } +}; + +/** + * + * @param {string} expr + * @param {JSONObject} val + * @param {string} path + * @param {object|GenericArray} parent + * @param {string} parentPropName + * @param {JSONPathCallback} callback + * @param {boolean} hasArrExpr + * @param {boolean} literalPriority + * @returns {ReturnObject|ReturnObject[]} + */ +JSONPath.prototype._trace = function (expr, val, path, parent, parentPropName, callback, hasArrExpr, literalPriority) { + // No expr to follow? return path and value as the result of + // this trace branch + let retObj; + if (!expr.length) { + retObj = { + path, + value: val, + parent, + parentProperty: parentPropName, + hasArrExpr + }; + this._handleCallback(retObj, callback, 'value'); + return retObj; + } + const loc = expr[0], + x = expr.slice(1); + + // We need to gather the return value of recursive trace calls in order to + // do the parent sel computation. + const ret = []; + /** + * + * @param {ReturnObject|ReturnObject[]} elems + * @returns {void} + */ + function addRet(elems) { + if (Array.isArray(elems)) { + // This was causing excessive stack size in Node (with or + // without Babel) against our performance test: + // `ret.push(...elems);` + elems.forEach(t => { + ret.push(t); + }); + } else { + ret.push(elems); + } + } + if ((typeof loc !== 'string' || literalPriority) && val && hasOwn(val, loc)) { + // simple case--directly follow property + addRet(this._trace(x, val[loc], push(path, loc), val, loc, callback, hasArrExpr)); + // eslint-disable-next-line unicorn/prefer-switch -- Part of larger `if` + } else if (loc === '*') { + // all child properties + this._walk(val, m => { + addRet(this._trace(x, val[m], push(path, m), val, m, callback, true, true)); + }); + } else if (loc === '..') { + // all descendent parent properties + // Check remaining expression with val's immediate children + addRet(this._trace(x, val, path, parent, parentPropName, callback, hasArrExpr)); + this._walk(val, m => { + // We don't join m and x here because we only want parents, + // not scalar values + if (typeof val[m] === 'object') { + // Keep going with recursive descent on val's + // object children + addRet(this._trace(expr.slice(), val[m], push(path, m), val, m, callback, true)); + } + }); + // The parent sel computation is handled in the frame above using the + // ancestor object of val + } else if (loc === '^') { + // This is not a final endpoint, so we do not invoke the callback here + this._hasParentSelector = true; + return { + path: path.slice(0, -1), + expr: x, + isParentSelector: true + }; + } else if (loc === '~') { + // property name + retObj = { + path: push(path, loc), + value: parentPropName, + parent, + parentProperty: null + }; + this._handleCallback(retObj, callback, 'property'); + return retObj; + } else if (loc === '$') { + // root only + addRet(this._trace(x, val, path, null, null, callback, hasArrExpr)); + } else if (/^(-?\d*):(-?\d*):?(\d*)$/u.test(loc)) { + // [start:end:step] Python slice syntax + addRet(this._slice(loc, x, val, path, parent, parentPropName, callback)); + } else if (loc.indexOf('?(') === 0) { + // [?(expr)] (filtering) + if (this.currEval === false) { + throw new Error('Eval [?(expr)] prevented in JSONPath expression.'); + } + const safeLoc = loc.replace(/^\?\((.*?)\)$/u, '$1'); + // check for a nested filter expression + const nested = /@.?([^?]*)[['](\??\(.*?\))(?!.\)\])[\]']/gu.exec(safeLoc); + if (nested) { + // find if there are matches in the nested expression + // add them to the result set if there is at least one match + this._walk(val, m => { + const npath = [nested[2]]; + const nvalue = nested[1] ? val[m][nested[1]] : val[m]; + const filterResults = this._trace(npath, nvalue, path, parent, parentPropName, callback, true); + if (filterResults.length > 0) { + addRet(this._trace(x, val[m], push(path, m), val, m, callback, true)); + } + }); + } else { + this._walk(val, m => { + if (this._eval(safeLoc, val[m], m, path, parent, parentPropName)) { + addRet(this._trace(x, val[m], push(path, m), val, m, callback, true)); + } + }); + } + } else if (loc[0] === '(') { + // [(expr)] (dynamic property/index) + if (this.currEval === false) { + throw new Error('Eval [(expr)] prevented in JSONPath expression.'); + } + // As this will resolve to a property name (but we don't know it + // yet), property and parent information is relative to the + // parent of the property to which this expression will resolve + addRet(this._trace(unshift(this._eval(loc, val, path[path.length - 1], path.slice(0, -1), parent, parentPropName), x), val, path, parent, parentPropName, callback, hasArrExpr)); + } else if (loc[0] === '@') { + // value type: @boolean(), etc. + let addType = false; + const valueType = loc.slice(1, -2); + switch (valueType) { + case 'scalar': + if (!val || !['object', 'function'].includes(typeof val)) { + addType = true; + } + break; + case 'boolean': + case 'string': + case 'undefined': + case 'function': + if (typeof val === valueType) { + addType = true; + } + break; + case 'integer': + if (Number.isFinite(val) && !(val % 1)) { + addType = true; + } + break; + case 'number': + if (Number.isFinite(val)) { + addType = true; + } + break; + case 'nonFinite': + if (typeof val === 'number' && !Number.isFinite(val)) { + addType = true; + } + break; + case 'object': + if (val && typeof val === valueType) { + addType = true; + } + break; + case 'array': + if (Array.isArray(val)) { + addType = true; + } + break; + case 'other': + addType = this.currOtherTypeCallback(val, path, parent, parentPropName); + break; + case 'null': + if (val === null) { + addType = true; + } + break; + /* c8 ignore next 2 */ + default: + throw new TypeError('Unknown value type ' + valueType); + } + if (addType) { + retObj = { + path, + value: val, + parent, + parentProperty: parentPropName + }; + this._handleCallback(retObj, callback, 'value'); + return retObj; + } + // `-escaped property + } else if (loc[0] === '`' && val && hasOwn(val, loc.slice(1))) { + const locProp = loc.slice(1); + addRet(this._trace(x, val[locProp], push(path, locProp), val, locProp, callback, hasArrExpr, true)); + } else if (loc.includes(',')) { + // [name1,name2,...] + const parts = loc.split(','); + for (const part of parts) { + addRet(this._trace(unshift(part, x), val, path, parent, parentPropName, callback, true)); + } + // simple case--directly follow property + } else if (!literalPriority && val && hasOwn(val, loc)) { + addRet(this._trace(x, val[loc], push(path, loc), val, loc, callback, hasArrExpr, true)); + } + + // We check the resulting values for parent selections. For parent + // selections we discard the value object and continue the trace with the + // current val object + if (this._hasParentSelector) { + for (let t = 0; t < ret.length; t++) { + const rett = ret[t]; + if (rett && rett.isParentSelector) { + const tmp = this._trace(rett.expr, val, rett.path, parent, parentPropName, callback, hasArrExpr); + if (Array.isArray(tmp)) { + ret[t] = tmp[0]; + const tl = tmp.length; + for (let tt = 1; tt < tl; tt++) { + // eslint-disable-next-line @stylistic/max-len -- Long + // eslint-disable-next-line sonarjs/updated-loop-counter -- Convenient + t++; + ret.splice(t, 0, tmp[tt]); + } + } else { + ret[t] = tmp; + } + } + } + } + return ret; +}; +JSONPath.prototype._walk = function (val, f) { + if (Array.isArray(val)) { + const n = val.length; + for (let i = 0; i < n; i++) { + f(i); + } + } else if (val && typeof val === 'object') { + Object.keys(val).forEach(m => { + f(m); + }); + } +}; +JSONPath.prototype._slice = function (loc, expr, val, path, parent, parentPropName, callback) { + if (!Array.isArray(val)) { + return undefined; + } + const len = val.length, + parts = loc.split(':'), + step = parts[2] && Number.parseInt(parts[2]) || 1; + let start = parts[0] && Number.parseInt(parts[0]) || 0, + end = parts[1] && Number.parseInt(parts[1]) || len; + start = start < 0 ? Math.max(0, start + len) : Math.min(len, start); + end = end < 0 ? Math.max(0, end + len) : Math.min(len, end); + const ret = []; + for (let i = start; i < end; i += step) { + const tmp = this._trace(unshift(i, expr), val, path, parent, parentPropName, callback, true); + // Should only be possible to be an array here since first part of + // ``unshift(i, expr)` passed in above would not be empty, nor `~`, + // nor begin with `@` (as could return objects) + // This was causing excessive stack size in Node (with or + // without Babel) against our performance test: `ret.push(...tmp);` + tmp.forEach(t => { + ret.push(t); + }); + } + return ret; +}; +JSONPath.prototype._eval = function (code, _v, _vname, path, parent, parentPropName) { + this.currSandbox._$_parentProperty = parentPropName; + this.currSandbox._$_parent = parent; + this.currSandbox._$_property = _vname; + this.currSandbox._$_root = this.json; + this.currSandbox._$_v = _v; + const containsPath = code.includes('@path'); + if (containsPath) { + this.currSandbox._$_path = JSONPath.toPathString(path.concat([_vname])); + } + const scriptCacheKey = this.currEval + 'Script:' + code; + if (!JSONPath.cache[scriptCacheKey]) { + let script = code.replace(/@parentProperty/gu, '_$_parentProperty').replace(/@parent/gu, '_$_parent').replace(/@property/gu, '_$_property').replace(/@root/gu, '_$_root').replace(/@([.\s)[])/gu, '_$_v$1'); + if (containsPath) { + script = script.replace(/@path/gu, '_$_path'); + } + if (this.currEval === 'safe' || this.currEval === true || this.currEval === undefined) { + JSONPath.cache[scriptCacheKey] = new this.safeVm.Script(script); + } else if (this.currEval === 'native') { + JSONPath.cache[scriptCacheKey] = new this.vm.Script(script); + } else if (typeof this.currEval === 'function' && this.currEval.prototype && hasOwn(this.currEval.prototype, 'runInNewContext')) { + const CurrEval = this.currEval; + JSONPath.cache[scriptCacheKey] = new CurrEval(script); + } else if (typeof this.currEval === 'function') { + JSONPath.cache[scriptCacheKey] = { + runInNewContext: context => this.currEval(script, context) + }; + } else { + throw new TypeError(`Unknown "eval" property "${this.currEval}"`); + } + } + try { + return JSONPath.cache[scriptCacheKey].runInNewContext(this.currSandbox); + } catch (e) { + if (this.ignoreEvalErrors) { + return false; + } + throw new Error('jsonPath: ' + e.message + ': ' + code); + } +}; + +// PUBLIC CLASS PROPERTIES AND METHODS + +// Could store the cache object itself +JSONPath.cache = {}; + +/** + * @param {string[]} pathArr Array to convert + * @returns {string} The path string + */ +JSONPath.toPathString = function (pathArr) { + const x = pathArr, + n = x.length; + let p = '$'; + for (let i = 1; i < n; i++) { + if (!/^(~|\^|@.*?\(\))$/u.test(x[i])) { + p += /^[0-9*]+$/u.test(x[i]) ? '[' + x[i] + ']' : "['" + x[i] + "']"; + } + } + return p; +}; + +/** + * @param {string} pointer JSON Path + * @returns {string} JSON Pointer + */ +JSONPath.toPointer = function (pointer) { + const x = pointer, + n = x.length; + let p = ''; + for (let i = 1; i < n; i++) { + if (!/^(~|\^|@.*?\(\))$/u.test(x[i])) { + p += '/' + x[i].toString().replace(/~/gu, '~0').replace(/\//gu, '~1'); + } + } + return p; +}; + +/** + * @param {string} expr Expression to convert + * @returns {string[]} + */ +JSONPath.toPathArray = function (expr) { + const { + cache + } = JSONPath; + if (cache[expr]) { + return cache[expr].concat(); + } + const subx = []; + const normalized = expr + // Properties + .replace(/@(?:null|boolean|number|string|integer|undefined|nonFinite|scalar|array|object|function|other)\(\)/gu, ';$&;') + // Parenthetical evaluations (filtering and otherwise), directly + // within brackets or single quotes + .replace(/[['](\??\(.*?\))[\]'](?!.\])/gu, function ($0, $1) { + return '[#' + (subx.push($1) - 1) + ']'; + }) + // Escape periods and tildes within properties + .replace(/\[['"]([^'\]]*)['"]\]/gu, function ($0, prop) { + return "['" + prop.replace(/\./gu, '%@%').replace(/~/gu, '%%@@%%') + "']"; + }) + // Properties operator + .replace(/~/gu, ';~;') + // Split by property boundaries + .replace(/['"]?\.['"]?(?![^[]*\])|\[['"]?/gu, ';') + // Reinsert periods within properties + .replace(/%@%/gu, '.') + // Reinsert tildes within properties + .replace(/%%@@%%/gu, '~') + // Parent + .replace(/(?:;)?(\^+)(?:;)?/gu, function ($0, ups) { + return ';' + ups.split('').join(';') + ';'; + }) + // Descendents + .replace(/;;;|;;/gu, ';..;') + // Remove trailing + .replace(/;$|'?\]|'$/gu, ''); + const exprList = normalized.split(';').map(function (exp) { + const match = exp.match(/#(\d+)/u); + return !match || !match[1] ? exp : subx[match[1]]; + }); + cache[expr] = exprList; + return cache[expr].concat(); +}; +JSONPath.prototype.safeVm = { + Script: SafeScript +}; + +JSONPath.prototype.vm = vm; + +exports.JSONPath = JSONPath; diff --git a/packages/dd-trace/src/payload-tagging/tagging.js b/packages/dd-trace/src/payload-tagging/tagging.js new file mode 100644 index 00000000000..4643b5d7a40 --- /dev/null +++ b/packages/dd-trace/src/payload-tagging/tagging.js @@ -0,0 +1,83 @@ +const { PAYLOAD_TAGGING_MAX_TAGS } = require('../constants') + +const redactedKeys = [ + 'authorization', 'x-authorization', 'password', 'token' +] +const truncated = 'truncated' +const redacted = 'redacted' + +function escapeKey (key) { + return key.replaceAll('.', '\\.') +} + +/** + * Compute normalized payload tags from any given object. + * + * @param {object} object + * @param {import('./mask').Mask} mask + * @param {number} maxDepth + * @param {string} prefix + * @returns + */ +function tagsFromObject (object, opts) { + const { maxDepth, prefix } = opts + + let tagCount = 0 + let abort = false + const result = {} + + function tagRec (prefix, object, depth = 0) { + // Off by one: _dd.payload_tags_trimmed counts as 1 tag + if (abort) { return } + + if (tagCount >= PAYLOAD_TAGGING_MAX_TAGS - 1) { + abort = true + result['_dd.payload_tags_incomplete'] = true + return + } + + if (depth >= maxDepth && typeof object === 'object') { + tagCount += 1 + result[prefix] = truncated + return + } + + if (object === undefined) { + tagCount += 1 + result[prefix] = 'undefined' + return + } + + if (object === null) { + tagCount += 1 + result[prefix] = 'null' + return + } + + if (['number', 'boolean'].includes(typeof object) || Buffer.isBuffer(object)) { + tagCount += 1 + result[prefix] = object.toString().substring(0, 5000) + return + } + + if (typeof object === 'string') { + tagCount += 1 + result[prefix] = object.substring(0, 5000) + } + + if (typeof object === 'object') { + for (const [key, value] of Object.entries(object)) { + if (redactedKeys.includes(key.toLowerCase())) { + tagCount += 1 + result[`${prefix}.${escapeKey(key)}`] = redacted + } else { + tagRec(`${prefix}.${escapeKey(key)}`, value, depth + 1) + } + } + } + } + tagRec(prefix, object) + return result +} + +module.exports = { tagsFromObject } diff --git a/packages/dd-trace/src/plugin_manager.js b/packages/dd-trace/src/plugin_manager.js index b7141a166d3..e9daea9b60b 100644 --- a/packages/dd-trace/src/plugin_manager.js +++ b/packages/dd-trace/src/plugin_manager.js @@ -4,7 +4,6 @@ const { channel } = require('dc-polyfill') const { isFalse } = require('./util') const plugins = require('./plugins') const log = require('./log') -const Nomenclature = require('./service-naming') const loadChannel = channel('dd-trace:instrumentation:load') @@ -102,7 +101,7 @@ module.exports = class PluginManager { // like instrumenter.enable() configure (config = {}) { this._tracerConfig = config - Nomenclature.configure(config) + this._tracer._nomenclature.configure(config) for (const name in pluginClasses) { this.loadPlugin(name) @@ -137,10 +136,21 @@ module.exports = class PluginManager { dbmPropagationMode, dsmEnabled, clientIpEnabled, - memcachedCommandEnabled + memcachedCommandEnabled, + ciVisibilityTestSessionName, + ciVisAgentlessLogSubmissionEnabled } = this._tracerConfig - const sharedConfig = {} + const sharedConfig = { + dbmPropagationMode, + dsmEnabled, + memcachedCommandEnabled, + site, + url, + headers: headerTags || [], + ciVisibilityTestSessionName, + ciVisAgentlessLogSubmissionEnabled + } if (logInjection !== undefined) { sharedConfig.logInjection = logInjection @@ -150,10 +160,6 @@ module.exports = class PluginManager { sharedConfig.queryStringObfuscation = queryStringObfuscation } - sharedConfig.dbmPropagationMode = dbmPropagationMode - sharedConfig.dsmEnabled = dsmEnabled - sharedConfig.memcachedCommandEnabled = memcachedCommandEnabled - if (serviceMapping && serviceMapping[name]) { sharedConfig.service = serviceMapping[name] } @@ -162,10 +168,6 @@ module.exports = class PluginManager { sharedConfig.clientIpEnabled = clientIpEnabled } - sharedConfig.site = site - sharedConfig.url = url - sharedConfig.headers = headerTags || [] - return sharedConfig } } diff --git a/packages/dd-trace/src/plugins/apollo.js b/packages/dd-trace/src/plugins/apollo.js new file mode 100644 index 00000000000..94ab360e921 --- /dev/null +++ b/packages/dd-trace/src/plugins/apollo.js @@ -0,0 +1,52 @@ +const TracingPlugin = require('./tracing') +const { storage } = require('../../../datadog-core') + +class ApolloBasePlugin extends TracingPlugin { + static get id () { return 'apollo.gateway' } + static get type () { return 'web' } + static get kind () { return 'server' } + + bindStart (ctx) { + const store = storage.getStore() + const childOf = store ? store.span : null + + const span = this.startSpan(this.getOperationName(), { + childOf, + service: this.getServiceName(), + type: this.constructor.type, + kind: this.constructor.kind, + meta: {} + }, false) + + ctx.parentStore = store + ctx.currentStore = { ...store, span } + + return ctx.currentStore + } + + end (ctx) { + // Only synchronous operations would have `result` or `error` on `end`. + if (!ctx.hasOwnProperty('result') && !ctx.hasOwnProperty('error')) return + ctx?.currentStore?.span?.finish() + } + + asyncStart (ctx) { + ctx?.currentStore?.span.finish() + return ctx.parentStore + } + + getServiceName () { + return this.serviceName({ + id: `${this.constructor.id}.${this.constructor.operation}`, + pluginConfig: this.config + }) + } + + getOperationName () { + return this.operationName({ + id: `${this.constructor.id}.${this.constructor.operation}` + }) + } +} + +module.exports = ApolloBasePlugin diff --git a/packages/dd-trace/src/plugins/ci_plugin.js b/packages/dd-trace/src/plugins/ci_plugin.js index 0112c4cb4fa..d4c9f32bc68 100644 --- a/packages/dd-trace/src/plugins/ci_plugin.js +++ b/packages/dd-trace/src/plugins/ci_plugin.js @@ -1,5 +1,6 @@ const { getTestEnvironmentMetadata, + getTestSessionName, getCodeOwnersFileEntries, getTestParentSpan, getTestCommonTags, @@ -13,13 +14,26 @@ const { TEST_SESSION_ID, TEST_COMMAND, TEST_MODULE, + TEST_SESSION_NAME, getTestSuiteCommonTags, TEST_STATUS, - TEST_SKIPPED_BY_ITR + TEST_SKIPPED_BY_ITR, + ITR_CORRELATION_ID, + TEST_SOURCE_FILE, + TEST_LEVEL_EVENT_TYPES, + TEST_SUITE } = require('./util/test') const Plugin = require('./plugin') const { COMPONENT } = require('../constants') const log = require('../log') +const { + incrementCountMetric, + distributionMetric, + TELEMETRY_EVENT_CREATED, + TELEMETRY_ITR_SKIPPED +} = require('../ci-visibility/telemetry') +const { CI_PROVIDER_NAME, GIT_REPOSITORY_URL, GIT_COMMIT_SHA, GIT_BRANCH, CI_WORKSPACE_PATH } = require('./util/tags') +const { OS_VERSION, OS_PLATFORM, OS_ARCHITECTURE, RUNTIME_NAME, RUNTIME_VERSION } = require('./util/env') module.exports = class CiPlugin extends Plugin { constructor (...args) { @@ -27,29 +41,31 @@ module.exports = class CiPlugin extends Plugin { this.rootDir = process.cwd() // fallback in case :session:start events are not emitted - this.addSub(`ci:${this.constructor.id}:itr-configuration`, ({ onDone }) => { - if (!this.tracer._exporter || !this.tracer._exporter.getItrConfiguration) { + this.addSub(`ci:${this.constructor.id}:library-configuration`, ({ onDone }) => { + if (!this.tracer._exporter || !this.tracer._exporter.getLibraryConfiguration) { return onDone({ err: new Error('CI Visibility was not initialized correctly') }) } - this.tracer._exporter.getItrConfiguration(this.testConfiguration, (err, itrConfig) => { + this.tracer._exporter.getLibraryConfiguration(this.testConfiguration, (err, libraryConfig) => { if (err) { - log.error(`Intelligent Test Runner configuration could not be fetched. ${err.message}`) + log.error(`Library configuration could not be fetched. ${err.message}`) } else { - this.itrConfig = itrConfig + this.libraryConfig = libraryConfig } - onDone({ err, itrConfig }) + onDone({ err, libraryConfig }) }) }) this.addSub(`ci:${this.constructor.id}:test-suite:skippable`, ({ onDone }) => { - if (!this.tracer._exporter || !this.tracer._exporter.getSkippableSuites) { + if (!this.tracer._exporter?.getSkippableSuites) { return onDone({ err: new Error('CI Visibility was not initialized correctly') }) } - this.tracer._exporter.getSkippableSuites(this.testConfiguration, (err, skippableSuites) => { + this.tracer._exporter.getSkippableSuites(this.testConfiguration, (err, skippableSuites, itrCorrelationId) => { if (err) { log.error(`Skippable suites could not be fetched. ${err.message}`) + } else { + this.itrCorrelationId = itrCorrelationId } - onDone({ err, skippableSuites }) + onDone({ err, skippableSuites, itrCorrelationId }) }) }) @@ -63,6 +79,19 @@ module.exports = class CiPlugin extends Plugin { // only for playwright this.rootDir = rootDir + const testSessionName = getTestSessionName(this.config, this.command, this.testEnvironmentMetadata) + + const metadataTags = {} + for (const testLevel of TEST_LEVEL_EVENT_TYPES) { + metadataTags[testLevel] = { + [TEST_SESSION_NAME]: testSessionName + } + } + // tracer might not be initialized correctly + if (this.tracer._exporter.setMetadataTags) { + this.tracer._exporter.setMetadataTags(metadataTags) + } + this.testSessionSpan = this.tracer.startSpan(`${this.constructor.id}.test_session`, { childOf, tags: { @@ -71,6 +100,9 @@ module.exports = class CiPlugin extends Plugin { ...testSessionSpanMetadata } }) + // TODO: add telemetry tag when we can add `is_agentless_log_submission_enabled` for agentless log submission + this.telemetry.ciVisEvent(TELEMETRY_EVENT_CREATED, 'session') + this.testModuleSpan = this.tracer.startSpan(`${this.constructor.id}.test_module`, { childOf: this.testSessionSpan, tags: { @@ -79,12 +111,24 @@ module.exports = class CiPlugin extends Plugin { ...testModuleSpanMetadata } }) + // only for vitest + // These are added for the worker threads to use + if (this.constructor.id === 'vitest') { + process.env.DD_CIVISIBILITY_TEST_SESSION_ID = this.testSessionSpan.context().toTraceId() + process.env.DD_CIVISIBILITY_TEST_MODULE_ID = this.testModuleSpan.context().toSpanId() + process.env.DD_CIVISIBILITY_TEST_COMMAND = this.command + } + + this.telemetry.ciVisEvent(TELEMETRY_EVENT_CREATED, 'module') }) this.addSub(`ci:${this.constructor.id}:itr:skipped-suites`, ({ skippedSuites, frameworkVersion }) => { const testCommand = this.testSessionSpan.context()._tags[TEST_COMMAND] skippedSuites.forEach((testSuite) => { const testSuiteMetadata = getTestSuiteCommonTags(testCommand, frameworkVersion, testSuite, this.constructor.id) + if (this.itrCorrelationId) { + testSuiteMetadata[ITR_CORRELATION_ID] = this.itrCorrelationId + } this.tracer.startSpan(`${this.constructor.id}.test_suite`, { childOf: this.testModuleSpan, @@ -97,25 +141,66 @@ module.exports = class CiPlugin extends Plugin { } }).finish() }) + this.telemetry.count(TELEMETRY_ITR_SKIPPED, { testLevel: 'suite' }, skippedSuites.length) + }) + + this.addSub(`ci:${this.constructor.id}:known-tests`, ({ onDone }) => { + if (!this.tracer._exporter?.getKnownTests) { + return onDone({ err: new Error('CI Visibility was not initialized correctly') }) + } + this.tracer._exporter.getKnownTests(this.testConfiguration, (err, knownTests) => { + if (err) { + log.error(`Known tests could not be fetched. ${err.message}`) + this.libraryConfig.isEarlyFlakeDetectionEnabled = false + } + onDone({ err, knownTests }) + }) }) } + get telemetry () { + const testFramework = this.constructor.id + return { + ciVisEvent: function (name, testLevel, tags = {}) { + incrementCountMetric(name, { + testLevel, + testFramework, + isUnsupportedCIProvider: !this.ciProviderName, + ...tags + }) + }, + count: function (name, tags, value = 1) { + incrementCountMetric(name, tags, value) + }, + distribution: function (name, tags, measure) { + distributionMetric(name, tags, measure) + } + } + } + configure (config) { super.configure(config) this.testEnvironmentMetadata = getTestEnvironmentMetadata(this.constructor.id, this.config) - this.codeOwnersEntries = getCodeOwnersFileEntries() const { - 'git.repository_url': repositoryUrl, - 'git.commit.sha': sha, - 'os.version': osVersion, - 'os.platform': osPlatform, - 'os.architecture': osArchitecture, - 'runtime.name': runtimeName, - 'runtime.version': runtimeVersion, - 'git.branch': branch + [GIT_REPOSITORY_URL]: repositoryUrl, + [GIT_COMMIT_SHA]: sha, + [OS_VERSION]: osVersion, + [OS_PLATFORM]: osPlatform, + [OS_ARCHITECTURE]: osArchitecture, + [RUNTIME_NAME]: runtimeName, + [RUNTIME_VERSION]: runtimeVersion, + [GIT_BRANCH]: branch, + [CI_PROVIDER_NAME]: ciProviderName, + [CI_WORKSPACE_PATH]: repositoryRoot } = this.testEnvironmentMetadata + this.repositoryRoot = repositoryRoot || process.cwd() + + this.codeOwnersEntries = getCodeOwnersFileEntries(repositoryRoot) + + this.ciProviderName = ciProviderName + this.testConfiguration = { repositoryUrl, sha, @@ -129,6 +214,19 @@ module.exports = class CiPlugin extends Plugin { } } + getCodeOwners (tags) { + const { + [TEST_SOURCE_FILE]: testSourceFile, + [TEST_SUITE]: testSuite + } = tags + // We'll try with the test source file if available (it could be different from the test suite) + let codeOwners = getCodeOwnersForFilename(testSourceFile, this.codeOwnersEntries) + if (!codeOwners) { + codeOwners = getCodeOwnersForFilename(testSuite, this.codeOwnersEntries) + } + return codeOwners + } + startTestSpan (testName, testSuite, testSuiteSpan, extraTags = {}) { const childOf = getTestParentSpan(this.tracer) @@ -143,7 +241,7 @@ module.exports = class CiPlugin extends Plugin { ...extraTags } - const codeOwners = getCodeOwnersForFilename(testSuite, this.codeOwnersEntries) + const codeOwners = this.getCodeOwners(testTags) if (codeOwners) { testTags[TEST_CODE_OWNERS] = codeOwners } @@ -170,6 +268,8 @@ module.exports = class CiPlugin extends Plugin { } } + this.telemetry.ciVisEvent(TELEMETRY_EVENT_CREATED, 'test', { hasCodeOwners: !!codeOwners }) + const testSpan = this.tracer .startSpan(`${this.constructor.id}.test`, { childOf, diff --git a/packages/dd-trace/src/plugins/composite.js b/packages/dd-trace/src/plugins/composite.js index c6137f47f82..a3b422a9f1d 100644 --- a/packages/dd-trace/src/plugins/composite.js +++ b/packages/dd-trace/src/plugins/composite.js @@ -12,11 +12,11 @@ class CompositePlugin extends Plugin { } configure (config) { + super.configure(config) for (const name in this.constructor.plugins) { - const pluginConfig = config[name] === false ? false : { - ...config, - ...config[name] - } + const pluginConfig = config[name] === false + ? false + : { ...config, ...config[name] } this[name].configure(pluginConfig) } diff --git a/packages/dd-trace/src/plugins/database.js b/packages/dd-trace/src/plugins/database.js index a868c594db7..9296ae46d6d 100644 --- a/packages/dd-trace/src/plugins/database.js +++ b/packages/dd-trace/src/plugins/database.js @@ -1,7 +1,7 @@ 'use strict' const StoragePlugin = require('./storage') -const { PEER_SERVICE_KEY } = require('../constants') +const { PEER_SERVICE_KEY, PEER_SERVICE_SOURCE_KEY } = require('../constants') class DatabasePlugin extends StoragePlugin { static get operation () { return 'query' } @@ -20,6 +20,7 @@ class DatabasePlugin extends StoragePlugin { encodedDdpv: '' } } + encodingServiceTags (serviceTag, encodeATag, spanConfig) { if (serviceTag !== spanConfig) { this.serviceTags[serviceTag] = spanConfig @@ -27,16 +28,31 @@ class DatabasePlugin extends StoragePlugin { } } - createDBMPropagationCommentService (serviceName) { + createDBMPropagationCommentService (serviceName, span) { this.encodingServiceTags('dddbs', 'encodedDddbs', serviceName) this.encodingServiceTags('dde', 'encodedDde', this.tracer._env) this.encodingServiceTags('ddps', 'encodedDdps', this.tracer._service) this.encodingServiceTags('ddpv', 'encodedDdpv', this.tracer._version) + if (span.context()._tags['out.host']) { + this.encodingServiceTags('ddh', 'encodedDdh', span._spanContext._tags['out.host']) + } + if (span.context()._tags['db.name']) { + this.encodingServiceTags('dddb', 'encodedDddb', span._spanContext._tags['db.name']) + } - const { encodedDddbs, encodedDde, encodedDdps, encodedDdpv } = this.serviceTags + const { encodedDddb, encodedDddbs, encodedDde, encodedDdh, encodedDdps, encodedDdpv } = this.serviceTags - return `dddbs='${encodedDddbs}',dde='${encodedDde}',` + + let dbmComment = `dddb='${encodedDddb}',dddbs='${encodedDddbs}',dde='${encodedDde}',ddh='${encodedDdh}',` + `ddps='${encodedDdps}',ddpv='${encodedDdpv}'` + + const peerData = this.getPeerService(span.context()._tags) + if (peerData !== undefined && peerData[PEER_SERVICE_SOURCE_KEY] === PEER_SERVICE_KEY) { + this.encodingServiceTags('ddprs', 'encodedDdprs', peerData[PEER_SERVICE_KEY]) + + const { encodedDdprs } = this.serviceTags + dbmComment += `,ddprs='${encodedDdprs}'` + } + return dbmComment } getDbmServiceName (span, tracerService) { @@ -55,7 +71,7 @@ class DatabasePlugin extends StoragePlugin { return query } - const servicePropagation = this.createDBMPropagationCommentService(dbmService) + const servicePropagation = this.createDBMPropagationCommentService(dbmService, span) if (isPreparedStatement || mode === 'service') { return `/*${servicePropagation}*/ ${query}` diff --git a/packages/dd-trace/src/plugins/index.js b/packages/dd-trace/src/plugins/index.js index d2a22cd8b15..80c32401536 100644 --- a/packages/dd-trace/src/plugins/index.js +++ b/packages/dd-trace/src/plugins/index.js @@ -1,7 +1,9 @@ 'use strict' module.exports = { + get '@apollo/gateway' () { return require('../../../datadog-plugin-apollo/src') }, get '@aws-sdk/smithy-client' () { return require('../../../datadog-plugin-aws-sdk/src') }, + get '@azure/functions' () { return require('../../../datadog-plugin-azure-functions/src') }, get '@cucumber/cucumber' () { return require('../../../datadog-plugin-cucumber/src') }, get '@playwright/test' () { return require('../../../datadog-plugin-playwright/src') }, get '@elastic/elasticsearch' () { return require('../../../datadog-plugin-elasticsearch/src') }, @@ -17,60 +19,74 @@ module.exports = { get '@opensearch-project/opensearch' () { return require('../../../datadog-plugin-opensearch/src') }, get '@redis/client' () { return require('../../../datadog-plugin-redis/src') }, get '@smithy/smithy-client' () { return require('../../../datadog-plugin-aws-sdk/src') }, - get 'aerospike' () { return require('../../../datadog-plugin-aerospike/src') }, - get 'amqp10' () { return require('../../../datadog-plugin-amqp10/src') }, - get 'amqplib' () { return require('../../../datadog-plugin-amqplib/src') }, + get '@vitest/runner' () { return require('../../../datadog-plugin-vitest/src') }, + get aerospike () { return require('../../../datadog-plugin-aerospike/src') }, + get amqp10 () { return require('../../../datadog-plugin-amqp10/src') }, + get amqplib () { return require('../../../datadog-plugin-amqplib/src') }, + get avsc () { return require('../../../datadog-plugin-avsc/src') }, get 'aws-sdk' () { return require('../../../datadog-plugin-aws-sdk/src') }, - get 'bunyan' () { return require('../../../datadog-plugin-bunyan/src') }, + get bunyan () { return require('../../../datadog-plugin-bunyan/src') }, get 'cassandra-driver' () { return require('../../../datadog-plugin-cassandra-driver/src') }, - get 'connect' () { return require('../../../datadog-plugin-connect/src') }, - get 'couchbase' () { return require('../../../datadog-plugin-couchbase/src') }, - get 'cypress' () { return require('../../../datadog-plugin-cypress/src') }, - get 'dns' () { return require('../../../datadog-plugin-dns/src') }, - get 'elasticsearch' () { return require('../../../datadog-plugin-elasticsearch/src') }, - get 'express' () { return require('../../../datadog-plugin-express/src') }, - get 'fastify' () { return require('../../../datadog-plugin-fastify/src') }, + get child_process () { return require('../../../datadog-plugin-child_process/src') }, + get connect () { return require('../../../datadog-plugin-connect/src') }, + get couchbase () { return require('../../../datadog-plugin-couchbase/src') }, + get cypress () { return require('../../../datadog-plugin-cypress/src') }, + get dns () { return require('../../../datadog-plugin-dns/src') }, + get elasticsearch () { return require('../../../datadog-plugin-elasticsearch/src') }, + get express () { return require('../../../datadog-plugin-express/src') }, + get fastify () { return require('../../../datadog-plugin-fastify/src') }, get 'find-my-way' () { return require('../../../datadog-plugin-find-my-way/src') }, - get 'graphql' () { return require('../../../datadog-plugin-graphql/src') }, - get 'grpc' () { return require('../../../datadog-plugin-grpc/src') }, - get 'hapi' () { return require('../../../datadog-plugin-hapi/src') }, - get 'http' () { return require('../../../datadog-plugin-http/src') }, - get 'http2' () { return require('../../../datadog-plugin-http2/src') }, - get 'https' () { return require('../../../datadog-plugin-http/src') }, - get 'ioredis' () { return require('../../../datadog-plugin-ioredis/src') }, + get graphql () { return require('../../../datadog-plugin-graphql/src') }, + get grpc () { return require('../../../datadog-plugin-grpc/src') }, + get hapi () { return require('../../../datadog-plugin-hapi/src') }, + get http () { return require('../../../datadog-plugin-http/src') }, + get http2 () { return require('../../../datadog-plugin-http2/src') }, + get https () { return require('../../../datadog-plugin-http/src') }, + get ioredis () { return require('../../../datadog-plugin-ioredis/src') }, get 'jest-circus' () { return require('../../../datadog-plugin-jest/src') }, get 'jest-config' () { return require('../../../datadog-plugin-jest/src') }, get 'jest-environment-node' () { return require('../../../datadog-plugin-jest/src') }, get 'jest-environment-jsdom' () { return require('../../../datadog-plugin-jest/src') }, - get 'jest-jasmine2' () { return require('../../../datadog-plugin-jest/src') }, + get 'jest-runtime' () { return require('../../../datadog-plugin-jest/src') }, get 'jest-worker' () { return require('../../../datadog-plugin-jest/src') }, - get 'koa' () { return require('../../../datadog-plugin-koa/src') }, + get koa () { return require('../../../datadog-plugin-koa/src') }, get 'koa-router' () { return require('../../../datadog-plugin-koa/src') }, - get 'kafkajs' () { return require('../../../datadog-plugin-kafkajs/src') }, - get 'mariadb' () { return require('../../../datadog-plugin-mariadb/src') }, - get 'memcached' () { return require('../../../datadog-plugin-memcached/src') }, + get kafkajs () { return require('../../../datadog-plugin-kafkajs/src') }, + get mariadb () { return require('../../../datadog-plugin-mariadb/src') }, + get memcached () { return require('../../../datadog-plugin-memcached/src') }, get 'microgateway-core' () { return require('../../../datadog-plugin-microgateway-core/src') }, - get 'mocha' () { return require('../../../datadog-plugin-mocha/src') }, + get mocha () { return require('../../../datadog-plugin-mocha/src') }, get 'mocha-each' () { return require('../../../datadog-plugin-mocha/src') }, - get 'moleculer' () { return require('../../../datadog-plugin-moleculer/src') }, - get 'mongodb' () { return require('../../../datadog-plugin-mongodb-core/src') }, + get vitest () { return require('../../../datadog-plugin-vitest/src') }, + get workerpool () { return require('../../../datadog-plugin-mocha/src') }, + get moleculer () { return require('../../../datadog-plugin-moleculer/src') }, + get mongodb () { return require('../../../datadog-plugin-mongodb-core/src') }, get 'mongodb-core' () { return require('../../../datadog-plugin-mongodb-core/src') }, - get 'mysql' () { return require('../../../datadog-plugin-mysql/src') }, - get 'mysql2' () { return require('../../../datadog-plugin-mysql2/src') }, - get 'net' () { return require('../../../datadog-plugin-net/src') }, - get 'next' () { return require('../../../datadog-plugin-next/src') }, - get 'oracledb' () { return require('../../../datadog-plugin-oracledb/src') }, - get 'openai' () { return require('../../../datadog-plugin-openai/src') }, - get 'paperplane' () { return require('../../../datadog-plugin-paperplane/src') }, - get 'pg' () { return require('../../../datadog-plugin-pg/src') }, - get 'pino' () { return require('../../../datadog-plugin-pino/src') }, + get mysql () { return require('../../../datadog-plugin-mysql/src') }, + get mysql2 () { return require('../../../datadog-plugin-mysql2/src') }, + get net () { return require('../../../datadog-plugin-net/src') }, + get next () { return require('../../../datadog-plugin-next/src') }, + get 'node:dns' () { return require('../../../datadog-plugin-dns/src') }, + get 'node:http' () { return require('../../../datadog-plugin-http/src') }, + get 'node:http2' () { return require('../../../datadog-plugin-http2/src') }, + get 'node:https' () { return require('../../../datadog-plugin-http/src') }, + get 'node:net' () { return require('../../../datadog-plugin-net/src') }, + get nyc () { return require('../../../datadog-plugin-nyc/src') }, + get oracledb () { return require('../../../datadog-plugin-oracledb/src') }, + get openai () { return require('../../../datadog-plugin-openai/src') }, + get paperplane () { return require('../../../datadog-plugin-paperplane/src') }, + get pg () { return require('../../../datadog-plugin-pg/src') }, + get pino () { return require('../../../datadog-plugin-pino/src') }, get 'pino-pretty' () { return require('../../../datadog-plugin-pino/src') }, - get 'playwright' () { return require('../../../datadog-plugin-playwright/src') }, - get 'redis' () { return require('../../../datadog-plugin-redis/src') }, - get 'restify' () { return require('../../../datadog-plugin-restify/src') }, - get 'rhea' () { return require('../../../datadog-plugin-rhea/src') }, - get 'router' () { return require('../../../datadog-plugin-router/src') }, - get 'sharedb' () { return require('../../../datadog-plugin-sharedb/src') }, - get 'tedious' () { return require('../../../datadog-plugin-tedious/src') }, - get 'winston' () { return require('../../../datadog-plugin-winston/src') } + get playwright () { return require('../../../datadog-plugin-playwright/src') }, + get protobufjs () { return require('../../../datadog-plugin-protobufjs/src') }, + get redis () { return require('../../../datadog-plugin-redis/src') }, + get restify () { return require('../../../datadog-plugin-restify/src') }, + get rhea () { return require('../../../datadog-plugin-rhea/src') }, + get router () { return require('../../../datadog-plugin-router/src') }, + get 'selenium-webdriver' () { return require('../../../datadog-plugin-selenium/src') }, + get sharedb () { return require('../../../datadog-plugin-sharedb/src') }, + get tedious () { return require('../../../datadog-plugin-tedious/src') }, + get undici () { return require('../../../datadog-plugin-undici/src') }, + get winston () { return require('../../../datadog-plugin-winston/src') } } diff --git a/packages/dd-trace/src/plugins/log_plugin.js b/packages/dd-trace/src/plugins/log_plugin.js index 353008a9e02..b0812ea46d3 100644 --- a/packages/dd-trace/src/plugins/log_plugin.js +++ b/packages/dd-trace/src/plugins/log_plugin.js @@ -54,7 +54,7 @@ module.exports = class LogPlugin extends Plugin { configure (config) { return super.configure({ ...config, - enabled: config.enabled && config.logInjection + enabled: config.enabled && (config.logInjection || config.ciVisAgentlessLogSubmissionEnabled) }) } } diff --git a/packages/dd-trace/src/plugins/plugin.js b/packages/dd-trace/src/plugins/plugin.js index 7bc5562bbcc..78a49b62b14 100644 --- a/packages/dd-trace/src/plugins/plugin.js +++ b/packages/dd-trace/src/plugins/plugin.js @@ -3,6 +3,7 @@ // TODO: move anything related to tracing to TracingPlugin instead const dc = require('dc-polyfill') +const logger = require('../log') const { storage } = require('../../../datadog-core') class Subscription { @@ -72,7 +73,17 @@ module.exports = class Plugin { } addSub (channelName, handler) { - this._subscriptions.push(new Subscription(channelName, handler)) + const plugin = this + const wrappedHandler = function () { + try { + return handler.apply(this, arguments) + } catch (e) { + logger.error('Error in plugin handler:', e) + logger.info('Disabling plugin:', plugin.id) + plugin.configure(false) + } + } + this._subscriptions.push(new Subscription(channelName, wrappedHandler)) } addBind (channelName, transform) { @@ -84,7 +95,7 @@ module.exports = class Plugin { if (!store || !store.span) return - if (!store.span._spanContext._tags['error']) { + if (!store.span._spanContext._tags.error) { store.span.setTag('error', error || 1) } } diff --git a/packages/dd-trace/src/plugins/schema.js b/packages/dd-trace/src/plugins/schema.js new file mode 100644 index 00000000000..675ba6a715f --- /dev/null +++ b/packages/dd-trace/src/plugins/schema.js @@ -0,0 +1,35 @@ +'use strict' + +const Plugin = require('./plugin') + +const SERIALIZATION = 'serialization' +const DESERIALIZATION = 'deserialization' + +class SchemaPlugin extends Plugin { + constructor (...args) { + super(...args) + + this.addSub(`apm:${this.constructor.id}:serialize-start`, this.handleSerializeStart.bind(this)) + this.addSub(`apm:${this.constructor.id}:deserialize-end`, this.handleDeserializeFinish.bind(this)) + } + + handleSerializeStart (args) { + const activeSpan = this.tracer.scope().active() + if (activeSpan && this.config.dsmEnabled) { + this.constructor.schemaExtractor.attachSchemaOnSpan( + args, activeSpan, SERIALIZATION, this.tracer + ) + } + } + + handleDeserializeFinish (args) { + const activeSpan = this.tracer.scope().active() + if (activeSpan && this.config.dsmEnabled) { + this.constructor.schemaExtractor.attachSchemaOnSpan( + args, activeSpan, DESERIALIZATION, this.tracer + ) + } + } +} + +module.exports = SchemaPlugin diff --git a/packages/dd-trace/src/plugins/tracing.js b/packages/dd-trace/src/plugins/tracing.js index 6ef21852174..d2d487a4a6f 100644 --- a/packages/dd-trace/src/plugins/tracing.js +++ b/packages/dd-trace/src/plugins/tracing.js @@ -4,7 +4,6 @@ const Plugin = require('./plugin') const { storage } = require('../../../datadog-core') const analyticsSampler = require('../analytics_sampler') const { COMPONENT } = require('../constants') -const Nomenclature = require('../service-naming') class TracingPlugin extends Plugin { constructor (...args) { @@ -29,7 +28,7 @@ class TracingPlugin extends Plugin { kind = this.constructor.kind } = opts - return Nomenclature.serviceName(type, kind, id, opts) + return this._tracer._nomenclature.serviceName(type, kind, id, opts) } operationName (opts = {}) { @@ -39,7 +38,7 @@ class TracingPlugin extends Plugin { kind = this.constructor.kind } = opts - return Nomenclature.opName(type, kind, id, opts) + return this._tracer._nomenclature.opName(type, kind, id, opts) } configure (config) { @@ -58,8 +57,12 @@ class TracingPlugin extends Plugin { this.activeSpan?.finish() } - error (error) { - this.addError(error) + error (ctxOrError) { + if (ctxOrError?.currentStore) { + ctxOrError.currentStore?.span.setTag('error', ctxOrError?.error) + return + } + this.addError(ctxOrError) } addTraceSubs () { @@ -91,7 +94,7 @@ class TracingPlugin extends Plugin { } addError (error, span = this.activeSpan) { - if (!span._spanContext._tags['error']) { + if (!span._spanContext._tags.error) { // Errors may be wrapped in a context. error = (error && error.error) || error span.setTag('error', error || 1) diff --git a/packages/dd-trace/src/plugins/util/ci.js b/packages/dd-trace/src/plugins/util/ci.js index 35e58c5a94e..86bda260212 100644 --- a/packages/dd-trace/src/plugins/util/ci.js +++ b/packages/dd-trace/src/plugins/util/ci.js @@ -1,3 +1,4 @@ +const { readFileSync } = require('fs') const { GIT_BRANCH, GIT_COMMIT_SHA, @@ -6,6 +7,9 @@ const { GIT_COMMIT_AUTHOR_NAME, GIT_COMMIT_MESSAGE, GIT_COMMIT_AUTHOR_DATE, + GIT_COMMIT_HEAD_SHA, + GIT_PULL_REQUEST_BASE_BRANCH_SHA, + GIT_PULL_REQUEST_BASE_BRANCH, GIT_REPOSITORY_URL, CI_PIPELINE_ID, CI_PIPELINE_NAME, @@ -77,6 +81,13 @@ function resolveTilde (filePath) { return filePath } +function getGitHubEventPayload () { + if (!process.env.GITHUB_EVENT_PATH) { + return + } + return JSON.parse(readFileSync(process.env.GITHUB_EVENT_PATH, 'utf8')) +} + module.exports = { normalizeRef, getCIMetadata () { @@ -241,7 +252,8 @@ module.exports = { GITHUB_REPOSITORY, GITHUB_SERVER_URL, GITHUB_RUN_ATTEMPT, - GITHUB_JOB + GITHUB_JOB, + GITHUB_BASE_REF } = env const repositoryURL = `${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git` @@ -277,6 +289,16 @@ module.exports = { GITHUB_RUN_ATTEMPT }) } + if (GITHUB_BASE_REF) { // `pull_request` or `pull_request_target` event + tags[GIT_PULL_REQUEST_BASE_BRANCH] = GITHUB_BASE_REF + try { + const eventContent = getGitHubEventPayload() + tags[GIT_PULL_REQUEST_BASE_BRANCH_SHA] = eventContent.pull_request.base.sha + tags[GIT_COMMIT_HEAD_SHA] = eventContent.pull_request.head.sha + } catch (e) { + // ignore malformed event content + } + } } if (env.APPVEYOR) { diff --git a/packages/dd-trace/src/plugins/util/env.js b/packages/dd-trace/src/plugins/util/env.js index c53c0956e53..c1721c4bb11 100644 --- a/packages/dd-trace/src/plugins/util/env.js +++ b/packages/dd-trace/src/plugins/util/env.js @@ -5,6 +5,7 @@ const OS_VERSION = 'os.version' const OS_ARCHITECTURE = 'os.architecture' const RUNTIME_NAME = 'runtime.name' const RUNTIME_VERSION = 'runtime.version' +const DD_HOST_CPU_COUNT = '_dd.host.vcpu_count' function getRuntimeAndOSMetadata () { return { @@ -12,7 +13,8 @@ function getRuntimeAndOSMetadata () { [OS_ARCHITECTURE]: process.arch, [OS_PLATFORM]: process.platform, [RUNTIME_NAME]: 'node', - [OS_VERSION]: os.release() + [OS_VERSION]: os.release(), + [DD_HOST_CPU_COUNT]: os.cpus().length } } @@ -22,5 +24,6 @@ module.exports = { OS_VERSION, OS_ARCHITECTURE, RUNTIME_NAME, - RUNTIME_VERSION + RUNTIME_VERSION, + DD_HOST_CPU_COUNT } diff --git a/packages/dd-trace/src/plugins/util/exec.js b/packages/dd-trace/src/plugins/util/exec.js deleted file mode 100644 index a2d091232c6..00000000000 --- a/packages/dd-trace/src/plugins/util/exec.js +++ /dev/null @@ -1,13 +0,0 @@ -const cp = require('child_process') -const log = require('../../log') - -const sanitizedExec = (cmd, flags, options = { stdio: 'pipe' }) => { - try { - return cp.execFileSync(cmd, flags, options).toString().replace(/(\r\n|\n|\r)/gm, '') - } catch (e) { - log.error(e) - return '' - } -} - -module.exports = { sanitizedExec } diff --git a/packages/dd-trace/src/plugins/util/git.js b/packages/dd-trace/src/plugins/util/git.js index 3a640ff249b..06b9521817f 100644 --- a/packages/dd-trace/src/plugins/util/git.js +++ b/packages/dd-trace/src/plugins/util/git.js @@ -1,10 +1,9 @@ -const { execFileSync } = require('child_process') +const cp = require('child_process') const os = require('os') const path = require('path') const fs = require('fs') const log = require('../../log') -const { sanitizedExec } = require('./exec') const { GIT_COMMIT_SHA, GIT_BRANCH, @@ -19,9 +18,55 @@ const { GIT_COMMIT_AUTHOR_NAME, CI_WORKSPACE_PATH } = require('./tags') +const { + incrementCountMetric, + distributionMetric, + TELEMETRY_GIT_COMMAND, + TELEMETRY_GIT_COMMAND_MS, + TELEMETRY_GIT_COMMAND_ERRORS +} = require('../../ci-visibility/telemetry') const { filterSensitiveInfoFromRepository } = require('./url') +const { storage } = require('../../../../datadog-core') + +const GIT_REV_LIST_MAX_BUFFER = 12 * 1024 * 1024 // 12MB -const GIT_REV_LIST_MAX_BUFFER = 8 * 1024 * 1024 // 8MB +function sanitizedExec ( + cmd, + flags, + operationMetric, + durationMetric, + errorMetric +) { + const store = storage.getStore() + storage.enterWith({ noop: true }) + + let startTime + if (operationMetric) { + incrementCountMetric(operationMetric.name, operationMetric.tags) + } + if (durationMetric) { + startTime = Date.now() + } + try { + const result = cp.execFileSync(cmd, flags, { stdio: 'pipe' }).toString().replace(/(\r\n|\n|\r)/gm, '') + if (durationMetric) { + distributionMetric(durationMetric.name, durationMetric.tags, Date.now() - startTime) + } + return result + } catch (err) { + if (errorMetric) { + incrementCountMetric(errorMetric.name, { + ...errorMetric.tags, + errorType: err.code, + exitCode: err.status || err.errno + }) + } + log.error(err) + return '' + } finally { + storage.enterWith(store) + } +} function isDirectory (path) { try { @@ -32,8 +77,26 @@ function isDirectory (path) { } } +function isGitAvailable () { + const isWindows = os.platform() === 'win32' + const command = isWindows ? 'where' : 'which' + try { + cp.execFileSync(command, ['git'], { stdio: 'pipe' }) + return true + } catch (e) { + incrementCountMetric(TELEMETRY_GIT_COMMAND_ERRORS, { command: 'check_git', exitCode: 'missing' }) + return false + } +} + function isShallowRepository () { - return sanitizedExec('git', ['rev-parse', '--is-shallow-repository']) === 'true' + return sanitizedExec( + 'git', + ['rev-parse', '--is-shallow-repository'], + { name: TELEMETRY_GIT_COMMAND, tags: { command: 'check_shallow' } }, + { name: TELEMETRY_GIT_COMMAND_MS, tags: { command: 'check_shallow' } }, + { name: TELEMETRY_GIT_COMMAND_ERRORS, tags: { command: 'check_shallow' } } + ) === 'true' } function getGitVersion () { @@ -72,50 +135,85 @@ function unshallowRepository () { defaultRemoteName ] + incrementCountMetric(TELEMETRY_GIT_COMMAND, { command: 'unshallow' }) + const start = Date.now() try { - execFileSync('git', [ + cp.execFileSync('git', [ ...baseGitOptions, revParseHead ], { stdio: 'pipe' }) - } catch (e) { + } catch (err) { // If the local HEAD is a commit that has not been pushed to the remote, the above command will fail. - log.error(e) + log.error(err) + incrementCountMetric( + TELEMETRY_GIT_COMMAND_ERRORS, + { command: 'unshallow', errorType: err.code, exitCode: err.status || err.errno } + ) const upstreamRemote = sanitizedExec('git', ['rev-parse', '--abbrev-ref', '--symbolic-full-name', '@{upstream}']) try { - execFileSync('git', [ + cp.execFileSync('git', [ ...baseGitOptions, upstreamRemote ], { stdio: 'pipe' }) - } catch (e) { + } catch (err) { // If the CI is working on a detached HEAD or branch tracking hasn’t been set up, the above command will fail. - log.error(e) + log.error(err) + incrementCountMetric( + TELEMETRY_GIT_COMMAND_ERRORS, + { command: 'unshallow', errorType: err.code, exitCode: err.status || err.errno } + ) // We use sanitizedExec here because if this last option fails, we'll give up. - sanitizedExec('git', baseGitOptions) + sanitizedExec( + 'git', + baseGitOptions, + null, + null, + { name: TELEMETRY_GIT_COMMAND_ERRORS, tags: { command: 'unshallow' } } // we log the error in sanitizedExec + ) } } + distributionMetric(TELEMETRY_GIT_COMMAND_MS, { command: 'unshallow' }, Date.now() - start) } function getRepositoryUrl () { - return sanitizedExec('git', ['config', '--get', 'remote.origin.url']) + return sanitizedExec( + 'git', + ['config', '--get', 'remote.origin.url'], + { name: TELEMETRY_GIT_COMMAND, tags: { command: 'get_repository' } }, + { name: TELEMETRY_GIT_COMMAND_MS, tags: { command: 'get_repository' } }, + { name: TELEMETRY_GIT_COMMAND_ERRORS, tags: { command: 'get_repository' } } + ) } function getLatestCommits () { + incrementCountMetric(TELEMETRY_GIT_COMMAND, { command: 'get_local_commits' }) + const startTime = Date.now() try { - return execFileSync('git', ['log', '--format=%H', '-n 1000', '--since="1 month ago"'], { stdio: 'pipe' }) + const result = cp.execFileSync('git', ['log', '--format=%H', '-n 1000', '--since="1 month ago"'], { stdio: 'pipe' }) .toString() .split('\n') .filter(commit => commit) + distributionMetric(TELEMETRY_GIT_COMMAND_MS, { command: 'get_local_commits' }, Date.now() - startTime) + return result } catch (err) { log.error(`Get latest commits failed: ${err.message}`) + incrementCountMetric( + TELEMETRY_GIT_COMMAND_ERRORS, + { command: 'get_local_commits', errorType: err.status } + ) return [] } } -function getCommitsToUpload (commitsToExclude, commitsToInclude) { +function getCommitsRevList (commitsToExclude, commitsToInclude) { + let result = null + const commitsToExcludeString = commitsToExclude.map(commit => `^${commit}`) + incrementCountMetric(TELEMETRY_GIT_COMMAND, { command: 'get_objects' }) + const startTime = Date.now() try { - return execFileSync( + result = cp.execFileSync( 'git', [ 'rev-list', @@ -132,11 +230,17 @@ function getCommitsToUpload (commitsToExclude, commitsToInclude) { .filter(commit => commit) } catch (err) { log.error(`Get commits to upload failed: ${err.message}`) - return [] + incrementCountMetric( + TELEMETRY_GIT_COMMAND_ERRORS, + { command: 'get_objects', errorType: err.code, exitCode: err.status || err.errno } // err.status might be null + ) } + distributionMetric(TELEMETRY_GIT_COMMAND_MS, { command: 'get_objects' }, Date.now() - startTime) + return result } function generatePackFilesForCommits (commitsToUpload) { + let result = [] const tmpFolder = os.tmpdir() if (!isDirectory(tmpFolder)) { @@ -148,10 +252,12 @@ function generatePackFilesForCommits (commitsToUpload) { const temporaryPath = path.join(tmpFolder, randomPrefix) const cwdPath = path.join(process.cwd(), randomPrefix) + incrementCountMetric(TELEMETRY_GIT_COMMAND, { command: 'pack_objects' }) + const startTime = Date.now() // Generates pack files to upload and // returns the ordered list of packfiles' paths function execGitPackObjects (targetPath) { - return execFileSync( + return cp.execFileSync( 'git', [ 'pack-objects', @@ -164,9 +270,13 @@ function generatePackFilesForCommits (commitsToUpload) { } try { - return execGitPackObjects(temporaryPath) + result = execGitPackObjects(temporaryPath) } catch (err) { log.error(err) + incrementCountMetric( + TELEMETRY_GIT_COMMAND_ERRORS, + { command: 'pack_objects', exitCode: err.status || err.errno, errorType: err.code } + ) /** * The generation of pack files in the temporary folder (from `os.tmpdir()`) * sometimes fails in certain CI setups with the error message @@ -180,13 +290,18 @@ function generatePackFilesForCommits (commitsToUpload) { * TODO: fix issue and remove workaround. */ try { - return execGitPackObjects(cwdPath) + result = execGitPackObjects(cwdPath) } catch (err) { log.error(err) + incrementCountMetric( + TELEMETRY_GIT_COMMAND_ERRORS, + { command: 'pack_objects', exitCode: err.status || err.errno, errorType: err.code } + ) } - - return [] } + distributionMetric(TELEMETRY_GIT_COMMAND_MS, { command: 'pack_objects' }, Date.now() - startTime) + + return result } // If there is ciMetadata, it takes precedence. @@ -236,8 +351,9 @@ module.exports = { getLatestCommits, getRepositoryUrl, generatePackFilesForCommits, - getCommitsToUpload, + getCommitsRevList, GIT_REV_LIST_MAX_BUFFER, isShallowRepository, - unshallowRepository + unshallowRepository, + isGitAvailable } diff --git a/packages/dd-trace/src/plugins/util/ip_blocklist.js b/packages/dd-trace/src/plugins/util/ip_blocklist.js deleted file mode 100644 index f346a1571b8..00000000000 --- a/packages/dd-trace/src/plugins/util/ip_blocklist.js +++ /dev/null @@ -1,51 +0,0 @@ -'use strict' - -const semver = require('semver') - -if (semver.satisfies(process.version, '>=14.18.0')) { - const net = require('net') - - module.exports = net.BlockList -} else { - const ipaddr = require('ipaddr.js') - - module.exports = class BlockList { - constructor () { - this.v4Ranges = [] - this.v6Ranges = [] - } - - addSubnet (net, prefix, type) { - this[type === 'ipv4' ? 'v4Ranges' : 'v6Ranges'].push(ipaddr.parseCIDR(`${net}/${prefix}`)) - } - - check (address, type) { - try { - let ip = ipaddr.parse(address) - - type = ip.kind() - - if (type === 'ipv6') { - for (const range of this.v6Ranges) { - if (ip.match(range)) return true - } - - if (ip.isIPv4MappedAddress()) { - ip = ip.toIPv4Address() - type = ip.kind() - } - } - - if (type === 'ipv4') { - for (const range of this.v4Ranges) { - if (ip.match(range)) return true - } - } - - return false - } catch { - return false - } - } - } -} diff --git a/packages/dd-trace/src/plugins/util/ip_extractor.js b/packages/dd-trace/src/plugins/util/ip_extractor.js index 14d87ec64c0..969b02746b5 100644 --- a/packages/dd-trace/src/plugins/util/ip_extractor.js +++ b/packages/dd-trace/src/plugins/util/ip_extractor.js @@ -1,6 +1,6 @@ 'use strict' -const BlockList = require('./ip_blocklist') +const { BlockList } = require('net') const net = require('net') const ipHeaderList = [ diff --git a/packages/dd-trace/src/plugins/util/serverless.js b/packages/dd-trace/src/plugins/util/serverless.js new file mode 100644 index 00000000000..3e969ffdfad --- /dev/null +++ b/packages/dd-trace/src/plugins/util/serverless.js @@ -0,0 +1,7 @@ +const types = require('../../../../../ext/types') +const web = require('./web') + +const serverless = { ...web } +serverless.TYPE = types.SERVERLESS + +module.exports = serverless diff --git a/packages/dd-trace/src/plugins/util/stacktrace.js b/packages/dd-trace/src/plugins/util/stacktrace.js new file mode 100644 index 00000000000..f67ba52c7c2 --- /dev/null +++ b/packages/dd-trace/src/plugins/util/stacktrace.js @@ -0,0 +1,94 @@ +'use strict' + +const { relative, sep, isAbsolute } = require('path') + +const cwd = process.cwd() + +module.exports = { + getCallSites, + getUserLandFrames +} + +// From https://github.com/felixge/node-stack-trace/blob/ba06dcdb50d465cd440d84a563836e293b360427/index.js#L1 +function getCallSites (constructorOpt) { + const oldLimit = Error.stackTraceLimit + Error.stackTraceLimit = Infinity + + const dummy = {} + + const v8Handler = Error.prepareStackTrace + Error.prepareStackTrace = function (_, v8StackTrace) { + return v8StackTrace + } + Error.captureStackTrace(dummy, constructorOpt) + + const v8StackTrace = dummy.stack + Error.prepareStackTrace = v8Handler + Error.stackTraceLimit = oldLimit + + return v8StackTrace +} + +/** + * Get stack trace of user-land frames. + * + * @param {Function} constructorOpt - Function to pass along to Error.captureStackTrace + * @param {number} [limit=Infinity] - The maximum number of frames to return + * @returns {{ file: string, line: number, method: (string|undefined), type: (string|undefined) }[]} - A + */ +function getUserLandFrames (constructorOpt, limit = Infinity) { + const callsites = getCallSites(constructorOpt) + const frames = [] + + for (const callsite of callsites) { + if (callsite.isNative()) { + continue + } + + const filename = callsite.getFileName() + + // If the callsite is native, there will be no associated filename. However, there might be other instances where + // this can happen, so to be sure, we add this additional check + if (filename === null) { + continue + } + + // ESM module paths start with the "file://" protocol (because ESM supports https imports) + // TODO: Node.js also supports `data:` and `node:` imports, should we do something specific for `data:`? + const containsFileProtocol = filename.startsWith('file:') + + // TODO: I'm not sure how stable this check is. Alternatively, we could consider reversing it if we can get + // a comprehensive list of all non-file-based values, eg: + // + // filename === '' || filename.startsWith('node:') + if (containsFileProtocol === false && isAbsolute(filename) === false) { + continue + } + + // TODO: Technically, the algorithm below could be simplified to not use the relative path, but be simply: + // + // if (filename.includes(sep + 'node_modules' + sep)) continue + // + // However, the tests in `packages/dd-trace/test/plugins/util/stacktrace.spec.js` will fail on my machine + // because I have the source code in a parent folder called `node_modules`. So the code below thinks that + // it's not in user-land + const relativePath = relative(cwd, containsFileProtocol ? filename.substring(7) : filename) + if (relativePath.startsWith('node_modules' + sep) || relativePath.includes(sep + 'node_modules' + sep)) { + continue + } + + const method = callsite.getFunctionName() + const type = callsite.getTypeName() + frames.push({ + file: filename, + line: callsite.getLineNumber(), + column: callsite.getColumnNumber(), + method: method ?? undefined, // force to undefined if null so JSON.stringify will omit it + type: type ?? undefined // force to undefined if null so JSON.stringify will omit it + }) + + if (frames.length === limit) break + } + + return frames +} diff --git a/packages/dd-trace/src/plugins/util/tags.js b/packages/dd-trace/src/plugins/util/tags.js index 15a795f4c1d..58709f0ceb7 100644 --- a/packages/dd-trace/src/plugins/util/tags.js +++ b/packages/dd-trace/src/plugins/util/tags.js @@ -9,6 +9,10 @@ const GIT_COMMIT_COMMITTER_NAME = 'git.commit.committer.name' const GIT_COMMIT_AUTHOR_DATE = 'git.commit.author.date' const GIT_COMMIT_AUTHOR_EMAIL = 'git.commit.author.email' const GIT_COMMIT_AUTHOR_NAME = 'git.commit.author.name' +const GIT_COMMIT_HEAD_SHA = 'git.commit.head_sha' + +const GIT_PULL_REQUEST_BASE_BRANCH_SHA = 'git.pull_request.base_branch_sha' +const GIT_PULL_REQUEST_BASE_BRANCH = 'git.pull_request.base_branch' const CI_PIPELINE_ID = 'ci.pipeline.id' const CI_PIPELINE_NAME = 'ci.pipeline.name' @@ -36,6 +40,9 @@ module.exports = { GIT_COMMIT_AUTHOR_DATE, GIT_COMMIT_AUTHOR_EMAIL, GIT_COMMIT_AUTHOR_NAME, + GIT_COMMIT_HEAD_SHA, + GIT_PULL_REQUEST_BASE_BRANCH_SHA, + GIT_PULL_REQUEST_BASE_BRANCH, CI_PIPELINE_ID, CI_PIPELINE_NAME, CI_PIPELINE_NUMBER, diff --git a/packages/dd-trace/src/plugins/util/test.js b/packages/dd-trace/src/plugins/util/test.js index 20e35ae416e..6c0dde70cfb 100644 --- a/packages/dd-trace/src/plugins/util/test.js +++ b/packages/dd-trace/src/plugins/util/test.js @@ -19,7 +19,8 @@ const { GIT_COMMIT_AUTHOR_NAME, GIT_COMMIT_MESSAGE, CI_WORKSPACE_PATH, - CI_PIPELINE_URL + CI_PIPELINE_URL, + CI_JOB_NAME } = require('./tags') const id = require('../../id') @@ -28,6 +29,9 @@ const { SAMPLING_RULE_DECISION } = require('../../constants') const { AUTO_KEEP } = require('../../../../../ext/priority') const { version: ddTraceVersion } = require('../../../../../package.json') +// session tags +const TEST_SESSION_NAME = 'test_session.name' + const TEST_FRAMEWORK = 'test.framework' const TEST_FRAMEWORK_VERSION = 'test.framework_version' const TEST_TYPE = 'test.type' @@ -48,10 +52,21 @@ const TEST_MODULE_ID = 'test_module_id' const TEST_SUITE_ID = 'test_suite_id' const TEST_TOOLCHAIN = 'test.toolchain' const TEST_SKIPPED_BY_ITR = 'test.skipped_by_itr' +// Browser used in browser test. Namespaced by test.configuration because it affects the fingerprint +const TEST_CONFIGURATION_BROWSER_NAME = 'test.configuration.browser_name' +// Early flake detection +const TEST_IS_NEW = 'test.is_new' +const TEST_IS_RETRY = 'test.is_retry' +const TEST_EARLY_FLAKE_ENABLED = 'test.early_flake.enabled' +const TEST_EARLY_FLAKE_ABORT_REASON = 'test.early_flake.abort_reason' const CI_APP_ORIGIN = 'ciapp-test' const JEST_TEST_RUNNER = 'test.jest.test_runner' +const JEST_DISPLAY_NAME = 'test.jest.display_name' + +const CUCUMBER_IS_PARALLEL = 'test.cucumber.is_parallel' +const MOCHA_IS_PARALLEL = 'test.mocha.is_parallel' const TEST_ITR_TESTS_SKIPPED = '_dd.ci.itr.tests_skipped' const TEST_ITR_SKIPPING_ENABLED = 'test.itr.tests_skipping.enabled' @@ -60,18 +75,46 @@ const TEST_ITR_SKIPPING_COUNT = 'test.itr.tests_skipping.count' const TEST_CODE_COVERAGE_ENABLED = 'test.code_coverage.enabled' const TEST_ITR_UNSKIPPABLE = 'test.itr.unskippable' const TEST_ITR_FORCED_RUN = 'test.itr.forced_run' +const ITR_CORRELATION_ID = 'itr_correlation_id' const TEST_CODE_COVERAGE_LINES_PCT = 'test.code_coverage.lines_pct' +// selenium tags +const TEST_BROWSER_DRIVER = 'test.browser.driver' +const TEST_BROWSER_DRIVER_VERSION = 'test.browser.driver_version' +const TEST_BROWSER_NAME = 'test.browser.name' +const TEST_BROWSER_VERSION = 'test.browser.version' + // jest worker variables const JEST_WORKER_TRACE_PAYLOAD_CODE = 60 const JEST_WORKER_COVERAGE_PAYLOAD_CODE = 61 +// cucumber worker variables +const CUCUMBER_WORKER_TRACE_PAYLOAD_CODE = 70 + +// mocha worker variables +const MOCHA_WORKER_TRACE_PAYLOAD_CODE = 80 + +// Early flake detection util strings +const EFD_STRING = "Retried by Datadog's Early Flake Detection" +const EFD_TEST_NAME_REGEX = new RegExp(EFD_STRING + ' \\(#\\d+\\): ', 'g') + +const TEST_LEVEL_EVENT_TYPES = [ + 'test', + 'test_suite_end', + 'test_module_end', + 'test_session_end' +] + module.exports = { TEST_CODE_OWNERS, + TEST_SESSION_NAME, TEST_FRAMEWORK, TEST_FRAMEWORK_VERSION, JEST_TEST_RUNNER, + JEST_DISPLAY_NAME, + CUCUMBER_IS_PARALLEL, + MOCHA_IS_PARALLEL, TEST_TYPE, TEST_NAME, TEST_SUITE, @@ -84,8 +127,15 @@ module.exports = { LIBRARY_VERSION, JEST_WORKER_TRACE_PAYLOAD_CODE, JEST_WORKER_COVERAGE_PAYLOAD_CODE, + CUCUMBER_WORKER_TRACE_PAYLOAD_CODE, + MOCHA_WORKER_TRACE_PAYLOAD_CODE, TEST_SOURCE_START, TEST_SKIPPED_BY_ITR, + TEST_CONFIGURATION_BROWSER_NAME, + TEST_IS_NEW, + TEST_IS_RETRY, + TEST_EARLY_FLAKE_ENABLED, + TEST_EARLY_FLAKE_ABORT_REASON, getTestEnvironmentMetadata, getTestParametersString, finishAllTraceSpans, @@ -111,15 +161,27 @@ module.exports = { TEST_CODE_COVERAGE_LINES_PCT, TEST_ITR_UNSKIPPABLE, TEST_ITR_FORCED_RUN, + ITR_CORRELATION_ID, addIntelligentTestRunnerSpanTags, getCoveredFilenamesFromCoverage, resetCoverage, mergeCoverage, fromCoverageMapToCoverage, getTestLineStart, - getCallSites, removeInvalidMetadata, - parseAnnotations + parseAnnotations, + EFD_STRING, + EFD_TEST_NAME_REGEX, + removeEfdStringFromTestName, + addEfdStringToTestName, + getIsFaultyEarlyFlakeDetection, + TEST_BROWSER_DRIVER, + TEST_BROWSER_DRIVER_VERSION, + TEST_BROWSER_NAME, + TEST_BROWSER_VERSION, + getTestSessionName, + TEST_LEVEL_EVENT_TYPES, + getNumFromKnownTests } // Returns pkg manager and its version, separated by '-', e.g. npm-8.15.0 or yarn-1.22.19 @@ -251,7 +313,6 @@ function getTestCommonTags (name, suite, version, testFramework) { [SAMPLING_PRIORITY]: AUTO_KEEP, [TEST_NAME]: name, [TEST_SUITE]: suite, - [TEST_SOURCE_FILE]: suite, [RESOURCE_NAME]: `${suite}.${name}`, [TEST_FRAMEWORK_VERSION]: version, [LIBRARY_VERSION]: ddTraceVersion @@ -267,7 +328,8 @@ function getTestSuitePath (testSuiteAbsolutePath, sourceRoot) { return sourceRoot } const testSuitePath = testSuiteAbsolutePath === sourceRoot - ? testSuiteAbsolutePath : path.relative(sourceRoot, testSuiteAbsolutePath) + ? testSuiteAbsolutePath + : path.relative(sourceRoot, testSuiteAbsolutePath) return testSuitePath.replace(path.sep, '/') } @@ -279,16 +341,36 @@ const POSSIBLE_CODEOWNERS_LOCATIONS = [ '.gitlab/CODEOWNERS' ] -function getCodeOwnersFileEntries (rootDir = process.cwd()) { - let codeOwnersContent - - POSSIBLE_CODEOWNERS_LOCATIONS.forEach(location => { +function readCodeOwners (rootDir) { + for (const location of POSSIBLE_CODEOWNERS_LOCATIONS) { try { - codeOwnersContent = fs.readFileSync(`${rootDir}/${location}`).toString() + return fs.readFileSync(path.join(rootDir, location)).toString() } catch (e) { // retry with next path } - }) + } + return '' +} + +function getCodeOwnersFileEntries (rootDir) { + let codeOwnersContent + let usedRootDir = rootDir + let isTriedCwd = false + + const processCwd = process.cwd() + + if (!usedRootDir || usedRootDir === processCwd) { + usedRootDir = processCwd + isTriedCwd = true + } + + codeOwnersContent = readCodeOwners(usedRootDir) + + // If we haven't found CODEOWNERS in the provided root dir, we try with process.cwd() + if (!codeOwnersContent && !isTriedCwd) { + codeOwnersContent = readCodeOwners(processCwd) + } + if (!codeOwnersContent) { return null } @@ -475,26 +557,6 @@ function getTestLineStart (err, testSuitePath) { } } -// From https://github.com/felixge/node-stack-trace/blob/ba06dcdb50d465cd440d84a563836e293b360427/index.js#L1 -function getCallSites () { - const oldLimit = Error.stackTraceLimit - Error.stackTraceLimit = Infinity - - const dummy = {} - - const v8Handler = Error.prepareStackTrace - Error.prepareStackTrace = function (_, v8StackTrace) { - return v8StackTrace - } - Error.captureStackTrace(dummy) - - const v8StackTrace = dummy.stack - Error.prepareStackTrace = v8Handler - Error.stackTraceLimit = oldLimit - - return v8StackTrace -} - /** * Gets an object of test tags from an Playwright annotations array. * @param {Object[]} annotations - Annotations from a Playwright test. @@ -521,3 +583,57 @@ function parseAnnotations (annotations) { return tags }, {}) } + +function addEfdStringToTestName (testName, numAttempt) { + return `${EFD_STRING} (#${numAttempt}): ${testName}` +} + +function removeEfdStringFromTestName (testName) { + return testName.replace(EFD_TEST_NAME_REGEX, '') +} + +function getIsFaultyEarlyFlakeDetection (projectSuites, testsBySuiteName, faultyThresholdPercentage) { + let newSuites = 0 + for (const suite of projectSuites) { + if (!testsBySuiteName[suite]) { + newSuites++ + } + } + const newSuitesPercentage = (newSuites / projectSuites.length) * 100 + + // The faulty threshold represents a percentage, but we also want to consider + // smaller projects, where big variations in the % are more likely. + // This is why we also check the absolute number of new suites. + return ( + newSuites > faultyThresholdPercentage && + newSuitesPercentage > faultyThresholdPercentage + ) +} + +function getTestSessionName (config, testCommand, envTags) { + if (config.ciVisibilityTestSessionName) { + return config.ciVisibilityTestSessionName + } + if (envTags[CI_JOB_NAME]) { + return `${envTags[CI_JOB_NAME]}-${testCommand}` + } + return testCommand +} + +// Calculate the number of a tests from the known tests response, which has a shape like: +// { testModule1: { testSuite1: [test1, test2, test3] }, testModule2: { testSuite2: [test4, test5] } } +function getNumFromKnownTests (knownTests) { + if (!knownTests) { + return 0 + } + + let totalNumTests = 0 + + for (const testModule of Object.values(knownTests)) { + for (const testSuite of Object.values(testModule)) { + totalNumTests += testSuite.length + } + } + + return totalNumTests +} diff --git a/packages/dd-trace/src/plugins/util/user-provided-git.js b/packages/dd-trace/src/plugins/util/user-provided-git.js index 4a18a1c58be..b6389a778eb 100644 --- a/packages/dd-trace/src/plugins/util/user-provided-git.js +++ b/packages/dd-trace/src/plugins/util/user-provided-git.js @@ -27,10 +27,11 @@ function removeEmptyValues (tags) { }, {}) } -// The regex is extracted from +// The regex is inspired by // https://github.com/jonschlinkert/is-git-url/blob/396965ffabf2f46656c8af4c47bef1d69f09292e/index.js#L9C15-L9C87 +// The `.git` suffix is optional in this version function validateGitRepositoryUrl (repoUrl) { - return /(?:git|ssh|https?|git@[-\w.]+):(\/\/)?(.*?)(\.git)(\/?|#[-\d\w._]+?)$/.test(repoUrl) + return /(?:git|ssh|https?|git@[-\w.]+):(\/\/)?(.*?)(\/?|#[-\d\w._]+?)$/.test(repoUrl) } function validateGitCommitSha (gitCommitSha) { diff --git a/packages/dd-trace/src/plugins/util/web.js b/packages/dd-trace/src/plugins/util/web.js index 95f5b7a617b..832044b29f8 100644 --- a/packages/dd-trace/src/plugins/util/web.js +++ b/packages/dd-trace/src/plugins/util/web.js @@ -1,6 +1,6 @@ 'use strict' -const uniq = require('lodash.uniq') +const uniq = require('../../../../datadog-core/src/utils/src/uniq') const analyticsSampler = require('../../analytics_sampler') const FORMAT_HTTP_HEADERS = 'http_headers' const log = require('../../log') @@ -36,6 +36,8 @@ const contexts = new WeakMap() const ends = new WeakMap() const web = { + TYPE: WEB, + // Ensure the configuration has the correct structure and defaults. normalizeConfig (config) { const headers = getHeadersToRecord(config) @@ -63,7 +65,7 @@ const web = { if (!span) return span.context()._name = `${name}.request` - span.context()._tags['component'] = name + span.context()._tags.component = name web.setConfig(req, config) }, @@ -103,7 +105,7 @@ const web = { context.res = res this.setConfig(req, config) - addRequestTags(context) + addRequestTags(context, this.TYPE) return span }, @@ -263,7 +265,7 @@ const web = { const context = contexts.get(req) const span = context.span const error = context.error - const hasExistingError = span.context()._tags['error'] || span.context()._tags[ERROR_MESSAGE] + const hasExistingError = span.context()._tags.error || span.context()._tags[ERROR_MESSAGE] if (!hasExistingError && !context.config.validateStatus(statusCode)) { span.setTag(ERROR, error || true) @@ -296,7 +298,7 @@ const web = { if (context.finished && !req.stream) return - addRequestTags(context) + addRequestTags(context, this.TYPE) addResponseTags(context) context.config.hooks.request(context.span, req, res) @@ -405,7 +407,7 @@ function addAllowHeaders (req, res, headers) { } function isOriginAllowed (req, headers) { - const origin = req.headers['origin'] + const origin = req.headers.origin const allowOrigin = headers['access-control-allow-origin'] return origin && (allowOrigin === '*' || allowOrigin === origin) @@ -423,7 +425,7 @@ function reactivate (req, fn) { : fn() } -function addRequestTags (context) { +function addRequestTags (context, spanType) { const { req, span, config } = context const url = extractURL(req) @@ -431,7 +433,7 @@ function addRequestTags (context) { [HTTP_URL]: web.obfuscateQs(config, url), [HTTP_METHOD]: req.method, [SPAN_KIND]: SERVER, - [SPAN_TYPE]: WEB, + [SPAN_TYPE]: spanType, [HTTP_USERAGENT]: req.headers['user-agent'] }) @@ -498,7 +500,7 @@ function extractURL (req) { return `${headers[HTTP2_HEADER_SCHEME]}://${headers[HTTP2_HEADER_AUTHORITY]}${headers[HTTP2_HEADER_PATH]}` } else { const protocol = getProtocol(req) - return `${protocol}://${req.headers['host']}${req.originalUrl || req.url}` + return `${protocol}://${req.headers.host}${req.originalUrl || req.url}` } } diff --git a/packages/dd-trace/src/priority_sampler.js b/packages/dd-trace/src/priority_sampler.js index 40ca1833cca..aae366c2622 100644 --- a/packages/dd-trace/src/priority_sampler.js +++ b/packages/dd-trace/src/priority_sampler.js @@ -2,28 +2,38 @@ const RateLimiter = require('./rate_limiter') const Sampler = require('./sampler') -const ext = require('../../../ext') const { setSamplingRules } = require('./startup-log') +const SamplingRule = require('./sampling_rule') +const { hasOwn } = require('./util') const { SAMPLING_MECHANISM_DEFAULT, SAMPLING_MECHANISM_AGENT, SAMPLING_MECHANISM_RULE, SAMPLING_MECHANISM_MANUAL, + SAMPLING_MECHANISM_REMOTE_USER, + SAMPLING_MECHANISM_REMOTE_DYNAMIC, SAMPLING_RULE_DECISION, SAMPLING_LIMIT_DECISION, SAMPLING_AGENT_DECISION, DECISION_MAKER_KEY } = require('./constants') -const SERVICE_NAME = ext.tags.SERVICE_NAME -const SAMPLING_PRIORITY = ext.tags.SAMPLING_PRIORITY -const MANUAL_KEEP = ext.tags.MANUAL_KEEP -const MANUAL_DROP = ext.tags.MANUAL_DROP -const USER_REJECT = ext.priority.USER_REJECT -const AUTO_REJECT = ext.priority.AUTO_REJECT -const AUTO_KEEP = ext.priority.AUTO_KEEP -const USER_KEEP = ext.priority.USER_KEEP +const { + tags: { + MANUAL_KEEP, + MANUAL_DROP, + SAMPLING_PRIORITY, + SERVICE_NAME + }, + priority: { + AUTO_REJECT, + AUTO_KEEP, + USER_REJECT, + USER_KEEP + } +} = require('../../../ext') + const DEFAULT_KEY = 'service:,env:' const defaultSampler = new Sampler(AUTO_KEEP) @@ -34,9 +44,9 @@ class PrioritySampler { this.update({}) } - configure (env, { sampleRate, rateLimit = 100, rules = [] } = {}) { + configure (env, { sampleRate, provenance = undefined, rateLimit = 100, rules = [] } = {}) { this._env = env - this._rules = this._normalizeRules(rules, sampleRate) + this._rules = this._normalizeRules(rules, sampleRate, rateLimit, provenance) this._limiter = new RateLimiter(rateLimit) setSamplingRules(this._rules) @@ -57,7 +67,7 @@ class PrioritySampler { if (context._sampling.priority !== undefined) return if (!root) return // noop span - const tag = this._getPriorityFromTags(context._tags) + const tag = this._getPriorityFromTags(context._tags, context) if (this.validate(tag)) { context._sampling.priority = tag @@ -104,7 +114,7 @@ class PrioritySampler { _getPriorityFromAuto (span) { const context = this._getContext(span) - const rule = this._findRule(context) + const rule = this._findRule(span) return rule ? this._getPriorityByRule(context, rule) @@ -130,8 +140,12 @@ class PrioritySampler { _getPriorityByRule (context, rule) { context._trace[SAMPLING_RULE_DECISION] = rule.sampleRate context._sampling.mechanism = SAMPLING_MECHANISM_RULE + if (rule.provenance === 'customer') context._sampling.mechanism = SAMPLING_MECHANISM_REMOTE_USER + if (rule.provenance === 'dynamic') context._sampling.mechanism = SAMPLING_MECHANISM_REMOTE_DYNAMIC - return rule.sampler.isSampled(context) && this._isSampledByRateLimit(context) ? USER_KEEP : USER_REJECT + return rule.sample() && this._isSampledByRateLimit(context) + ? USER_KEEP + : USER_REJECT } _isSampledByRateLimit (context) { @@ -172,37 +186,21 @@ class PrioritySampler { } } - _normalizeRules (rules, sampleRate) { + _normalizeRules (rules, sampleRate, rateLimit, provenance) { rules = [].concat(rules || []) return rules - .concat({ sampleRate }) + .concat({ sampleRate, maxPerSecond: rateLimit, provenance }) .map(rule => ({ ...rule, sampleRate: parseFloat(rule.sampleRate) })) .filter(rule => !isNaN(rule.sampleRate)) - .map(rule => ({ ...rule, sampler: new Sampler(rule.sampleRate) })) + .map(SamplingRule.from) } - _findRule (context) { - for (let i = 0, l = this._rules.length; i < l; i++) { - if (this._matchRule(context, this._rules[i])) return this._rules[i] + _findRule (span) { + for (const rule of this._rules) { + if (rule.match(span)) return rule } } - - _matchRule (context, rule) { - const name = context._name - const service = context._tags['service.name'] - - if (rule.name instanceof RegExp && !rule.name.test(name)) return false - if (typeof rule.name === 'string' && rule.name !== name) return false - if (rule.service instanceof RegExp && !rule.service.test(service)) return false - if (typeof rule.service === 'string' && rule.service !== service) return false - - return true - } -} - -function hasOwn (object, prop) { - return Object.prototype.hasOwnProperty.call(object, prop) } module.exports = PrioritySampler diff --git a/packages/dd-trace/src/profiler.js b/packages/dd-trace/src/profiler.js index 349f0438d7c..914cfc3dfdc 100644 --- a/packages/dd-trace/src/profiler.js +++ b/packages/dd-trace/src/profiler.js @@ -8,7 +8,7 @@ process.once('beforeExit', () => { profiler.stop() }) module.exports = { start: config => { - const { service, version, env, url, hostname, port, tags, repositoryUrl, commitSHA } = config + const { service, version, env, url, hostname, port, tags, repositoryUrl, commitSHA, injectionEnabled } = config const { enabled, sourceMap, exporters } = config.profiling const logger = { debug: (message) => log.debug(message), @@ -17,8 +17,17 @@ module.exports = { error: (message) => log.error(message) } + const libraryInjected = injectionEnabled.length > 0 + let activation + if (enabled === 'auto') { + activation = 'auto' + } else if (enabled === 'true') { + activation = 'manual' + } else if (injectionEnabled.includes('profiler')) { + activation = 'injection' + } // else activation = undefined + return profiler.start({ - enabled, service, version, env, @@ -30,7 +39,9 @@ module.exports = { port, tags, repositoryUrl, - commitSHA + commitSHA, + libraryInjected, + activation }) }, diff --git a/packages/dd-trace/src/profiling/config.js b/packages/dd-trace/src/profiling/config.js index a37015e97b7..3c360d65f7a 100644 --- a/packages/dd-trace/src/profiling/config.js +++ b/packages/dd-trace/src/profiling/config.js @@ -14,39 +14,41 @@ const { oomExportStrategies, snapshotKinds } = require('./constants') const { GIT_REPOSITORY_URL, GIT_COMMIT_SHA } = require('../plugins/util/tags') const { tagger } = require('./tagger') const { isFalse, isTrue } = require('../util') +const { getAzureTagsFromMetadata, getAzureAppMetadata } = require('../azure_metadata') class Config { constructor (options = {}) { const { - DD_PROFILING_ENABLED, - DD_PROFILING_PROFILERS, - DD_ENV, - DD_TAGS, - DD_SERVICE, - DD_VERSION, - DD_TRACE_AGENT_URL, DD_AGENT_HOST, - DD_TRACE_AGENT_PORT, + DD_ENV, + DD_PROFILING_CODEHOTSPOTS_ENABLED, + DD_PROFILING_CPU_ENABLED, DD_PROFILING_DEBUG_SOURCE_MAPS, - DD_PROFILING_UPLOAD_TIMEOUT, + DD_PROFILING_ENDPOINT_COLLECTION_ENABLED, + DD_PROFILING_EXPERIMENTAL_CODEHOTSPOTS_ENABLED, + DD_PROFILING_EXPERIMENTAL_CPU_ENABLED, + DD_PROFILING_EXPERIMENTAL_ENDPOINT_COLLECTION_ENABLED, + DD_PROFILING_EXPERIMENTAL_OOM_EXPORT_STRATEGIES, + DD_PROFILING_EXPERIMENTAL_OOM_HEAP_LIMIT_EXTENSION_SIZE, + DD_PROFILING_EXPERIMENTAL_OOM_MAX_HEAP_EXTENSION_COUNT, + DD_PROFILING_EXPERIMENTAL_OOM_MONITORING_ENABLED, + DD_PROFILING_EXPERIMENTAL_TIMELINE_ENABLED, + DD_PROFILING_HEAP_ENABLED, + DD_PROFILING_PPROF_PREFIX, + DD_PROFILING_PROFILERS, DD_PROFILING_SOURCE_MAP, + DD_PROFILING_TIMELINE_ENABLED, DD_PROFILING_UPLOAD_PERIOD, - DD_PROFILING_PPROF_PREFIX, - DD_PROFILING_HEAP_ENABLED, + DD_PROFILING_UPLOAD_TIMEOUT, DD_PROFILING_V8_PROFILER_BUG_WORKAROUND, DD_PROFILING_WALLTIME_ENABLED, - DD_PROFILING_EXPERIMENTAL_OOM_MONITORING_ENABLED, - DD_PROFILING_EXPERIMENTAL_OOM_HEAP_LIMIT_EXTENSION_SIZE, - DD_PROFILING_EXPERIMENTAL_OOM_MAX_HEAP_EXTENSION_COUNT, - DD_PROFILING_EXPERIMENTAL_OOM_EXPORT_STRATEGIES, - DD_PROFILING_EXPERIMENTAL_TIMELINE_ENABLED, - DD_PROFILING_CODEHOTSPOTS_ENABLED, - DD_PROFILING_ENDPOINT_COLLECTION_ENABLED, - DD_PROFILING_EXPERIMENTAL_CODEHOTSPOTS_ENABLED, - DD_PROFILING_EXPERIMENTAL_ENDPOINT_COLLECTION_ENABLED + DD_SERVICE, + DD_TAGS, + DD_TRACE_AGENT_PORT, + DD_TRACE_AGENT_URL, + DD_VERSION } = process.env - const enabled = isTrue(coalesce(options.enabled, DD_PROFILING_ENABLED, true)) const env = coalesce(options.env, DD_ENV) const service = options.service || DD_SERVICE || 'node' const host = os.hostname() @@ -61,7 +63,6 @@ class Config { const pprofPrefix = coalesce(options.pprofPrefix, DD_PROFILING_PPROF_PREFIX, '') - this.enabled = enabled this.service = service this.env = env this.host = host @@ -71,7 +72,8 @@ class Config { this.tags = Object.assign( tagger.parse(DD_TAGS), tagger.parse(options.tags), - tagger.parse({ env, host, service, version, functionname }) + tagger.parse({ env, host, service, version, functionname }), + getAzureTagsFromMetadata(getAzureAppMetadata()) ) // Add source code integration tags if available @@ -91,14 +93,33 @@ class Config { logger.warn(`${deprecatedEnvVarName} is deprecated. Use DD_PROFILING_${shortVarName} instead.`) } } + // Profiler sampling contexts are not available on Windows, so features + // depending on those (code hotspots and endpoint collection) need to default + // to false on Windows. + const samplingContextsAvailable = process.platform !== 'win32' + function checkOptionAllowed (option, description, condition) { + if (option && !condition) { + // injection hardening: all of these can only happen if user explicitly + // sets an environment variable to its non-default value on the platform. + // In practical terms, it'd require someone explicitly turning on OOM + // monitoring, code hotspots, endpoint profiling, or CPU profiling on + // Windows, where it is not supported. + throw new Error(`${description} not supported on ${process.platform}.`) + } + } + function checkOptionWithSamplingContextAllowed (option, description) { + checkOptionAllowed(option, description, samplingContextsAvailable) + } + this.flushInterval = flushInterval this.uploadTimeout = uploadTimeout this.sourceMap = sourceMap this.debugSourceMaps = isTrue(coalesce(options.debugSourceMaps, DD_PROFILING_DEBUG_SOURCE_MAPS, false)) this.endpointCollectionEnabled = isTrue(coalesce(options.endpointCollection, DD_PROFILING_ENDPOINT_COLLECTION_ENABLED, - DD_PROFILING_EXPERIMENTAL_ENDPOINT_COLLECTION_ENABLED, false)) + DD_PROFILING_EXPERIMENTAL_ENDPOINT_COLLECTION_ENABLED, samplingContextsAvailable)) logExperimentalVarDeprecation('ENDPOINT_COLLECTION_ENABLED') + checkOptionWithSamplingContextAllowed(this.endpointCollectionEnabled, 'Endpoint collection') this.pprofPrefix = pprofPrefix this.v8ProfilerBugWorkaroundEnabled = isTrue(coalesce(options.v8ProfilerBugWorkaround, @@ -111,12 +132,19 @@ class Config { port }))) + this.libraryInjected = options.libraryInjected + this.activation = options.activation this.exporters = ensureExporters(options.exporters || [ new AgentExporter(this) ], this) + // OOM monitoring does not work well on Windows, so it is disabled by default. + const oomMonitoringSupported = process.platform !== 'win32' + const oomMonitoringEnabled = isTrue(coalesce(options.oomMonitoring, - DD_PROFILING_EXPERIMENTAL_OOM_MONITORING_ENABLED, true)) + DD_PROFILING_EXPERIMENTAL_OOM_MONITORING_ENABLED, oomMonitoringSupported)) + checkOptionAllowed(oomMonitoringEnabled, 'OOM monitoring', oomMonitoringSupported) + const heapLimitExtensionSize = coalesce(options.oomHeapLimitExtensionSize, Number(DD_PROFILING_EXPERIMENTAL_OOM_HEAP_LIMIT_EXTENSION_SIZE), 0) const maxHeapExtensionCount = coalesce(options.oomMaxHeapExtensionCount, @@ -143,12 +171,22 @@ class Config { }) this.timelineEnabled = isTrue(coalesce(options.timelineEnabled, - DD_PROFILING_EXPERIMENTAL_TIMELINE_ENABLED, false)) + DD_PROFILING_TIMELINE_ENABLED, + DD_PROFILING_EXPERIMENTAL_TIMELINE_ENABLED, samplingContextsAvailable)) + logExperimentalVarDeprecation('TIMELINE_ENABLED') + checkOptionWithSamplingContextAllowed(this.timelineEnabled, 'Timeline view') this.codeHotspotsEnabled = isTrue(coalesce(options.codeHotspotsEnabled, DD_PROFILING_CODEHOTSPOTS_ENABLED, - DD_PROFILING_EXPERIMENTAL_CODEHOTSPOTS_ENABLED, false)) + DD_PROFILING_EXPERIMENTAL_CODEHOTSPOTS_ENABLED, samplingContextsAvailable)) logExperimentalVarDeprecation('CODEHOTSPOTS_ENABLED') + checkOptionWithSamplingContextAllowed(this.codeHotspotsEnabled, 'Code hotspots') + + this.cpuProfilingEnabled = isTrue(coalesce(options.cpuProfilingEnabled, + DD_PROFILING_CPU_ENABLED, + DD_PROFILING_EXPERIMENTAL_CPU_ENABLED, samplingContextsAvailable)) + logExperimentalVarDeprecation('CPU_ENABLED') + checkOptionWithSamplingContextAllowed(this.cpuProfilingEnabled, 'CPU profiling') this.profilers = ensureProfilers(profilers, this) } @@ -208,7 +246,7 @@ function ensureOOMExportStrategies (strategies, options) { } } - return [ ...new Set(strategies) ] + return [...new Set(strategies)] } function getExporter (name, options) { @@ -259,8 +297,9 @@ function ensureProfilers (profilers, options) { } } - // Events profiler is a profiler for timeline events - if (options.timelineEnabled) { + // Events profiler is a profiler that produces timeline events. It is only + // added if timeline is enabled and there's a wall profiler. + if (options.timelineEnabled && profilers.some(p => p instanceof WallProfiler)) { profilers.push(new EventsProfiler(options)) } diff --git a/packages/dd-trace/src/profiling/exporter_cli.js b/packages/dd-trace/src/profiling/exporter_cli.js index 53db9f5f78f..474f4237de5 100644 --- a/packages/dd-trace/src/profiling/exporter_cli.js +++ b/packages/dd-trace/src/profiling/exporter_cli.js @@ -16,10 +16,22 @@ function exporterFromURL (url) { if (url.protocol === 'file:') { return new FileExporter({ pprofPrefix: fileURLToPath(url) }) } else { + const injectionEnabled = (process.env.DD_INJECTION_ENABLED || '').split(',') + const libraryInjected = injectionEnabled.length > 0 + const profilingEnabled = (process.env.DD_PROFILING_ENABLED || '').toLowerCase() + const activation = ['true', '1'].includes(profilingEnabled) + ? 'manual' + : profilingEnabled === 'auto' + ? 'auto' + : injectionEnabled.includes('profiling') + ? 'injection' + : 'unknown' return new AgentExporter({ url, logger, - uploadTimeout: timeoutMs + uploadTimeout: timeoutMs, + libraryInjected, + activation }) } } diff --git a/packages/dd-trace/src/profiling/exporters/agent.js b/packages/dd-trace/src/profiling/exporters/agent.js index 712d03f1406..b34ab3c9d94 100644 --- a/packages/dd-trace/src/profiling/exporters/agent.js +++ b/packages/dd-trace/src/profiling/exporters/agent.js @@ -9,6 +9,9 @@ const docker = require('../../exporters/common/docker') const FormData = require('../../exporters/common/form-data') const { storage } = require('../../../../datadog-core') const version = require('../../../../../package.json').version +const os = require('os') +const { urlToHttpOptions } = require('url') +const perf = require('perf_hooks').performance const containerId = docker.id() @@ -50,7 +53,7 @@ function computeRetries (uploadTimeout) { } class AgentExporter { - constructor ({ url, logger, uploadTimeout } = {}) { + constructor ({ url, logger, uploadTimeout, env, host, service, version, libraryInjected, activation } = {}) { this._url = url this._logger = logger @@ -58,47 +61,93 @@ class AgentExporter { this._backoffTime = backoffTime this._backoffTries = backoffTries + this._env = env + this._host = host + this._service = service + this._appVersion = version + this._libraryInjected = !!libraryInjected + this._activation = activation || 'unknown' } export ({ profiles, start, end, tags }) { - const types = Object.keys(profiles) - - const fields = [ - ['recording-start', start.toISOString()], - ['recording-end', end.toISOString()], - ['language', 'javascript'], - ['runtime', 'nodejs'], - ['runtime_version', process.version], - ['profiler_version', version], - ['format', 'pprof'], - - ['tags[]', 'language:javascript'], - ['tags[]', 'runtime:nodejs'], - ['tags[]', `runtime_version:${process.version}`], - ['tags[]', `profiler_version:${version}`], - ['tags[]', 'format:pprof'], - ...Object.entries(tags).map(([key, value]) => ['tags[]', `${key}:${value}`]) - ] + const fields = [] - this._logger.debug(() => { - const body = fields.map(([key, value]) => ` ${key}: ${value}`).join('\n') - return `Building agent export report: ${'\n' + body}` + function typeToFile (type) { + return `${type}.pprof` + } + + const event = JSON.stringify({ + attachments: Object.keys(profiles).map(typeToFile), + start: start.toISOString(), + end: end.toISOString(), + family: 'node', + version: '4', + tags_profiler: [ + 'language:javascript', + 'runtime:nodejs', + `runtime_arch:${process.arch}`, + `runtime_os:${process.platform}`, + `runtime_version:${process.version}`, + `process_id:${process.pid}`, + `profiler_version:${version}`, + 'format:pprof', + ...Object.entries(tags).map(([key, value]) => `${key}:${value}`) + ].join(','), + info: { + application: { + env: this._env, + service: this._service, + start_time: new Date(perf.nodeTiming.nodeStart + perf.timeOrigin).toISOString(), + version: this._appVersion + }, + platform: { + hostname: this._host, + kernel_name: os.type(), + kernel_release: os.release(), + kernel_version: os.version() + }, + profiler: { + activation: this._activation, + ssi: { + mechanism: this._libraryInjected ? 'injected_agent' : 'none' + }, + version + }, + runtime: { + // Using `nodejs` for consistency with the existing `runtime` tag. + // Note that the event `family` property uses `node`, as that's what's + // proscribed by the Intake API, but that's an internal enum and is + // not customer visible. + engine: 'nodejs', + // strip off leading 'v'. This makes the format consistent with other + // runtimes (e.g. Ruby) but not with the existing `runtime_version` tag. + // We'll keep it like this as we want cross-engine consistency. We + // also aren't changing the format of the existing tag as we don't want + // to break it. + version: process.version.substring(1) + } + } }) - for (let index = 0; index < types.length; index++) { - const type = types[index] - const buffer = profiles[type] + fields.push(['event', event, { + filename: 'event.json', + contentType: 'application/json' + }]) + + this._logger.debug(() => { + return `Building agent export report:\n${event}` + }) + for (const [type, buffer] of Object.entries(profiles)) { this._logger.debug(() => { const bytes = buffer.toString('hex').match(/../g).join(' ') return `Adding ${type} profile to agent export: ` + bytes }) - fields.push([`types[${index}]`, type]) - fields.push([`data[${index}]`, buffer, { - filename: `${type}.pb.gz`, - contentType: 'application/octet-stream', - knownLength: buffer.length + const filename = typeToFile(type) + fields.push([filename, buffer, { + filename, + contentType: 'application/octet-stream' }]) } @@ -120,7 +169,11 @@ class AgentExporter { const options = { method: 'POST', path: '/profiling/v1/input', - headers: form.getHeaders(), + headers: { + 'DD-EVP-ORIGIN': 'dd-trace-js', + 'DD-EVP-ORIGIN-VERSION': version, + ...form.getHeaders() + }, timeout: this._backoffTime * Math.pow(2, attempt) } @@ -131,9 +184,10 @@ class AgentExporter { if (this._url.protocol === 'unix:') { options.socketPath = this._url.pathname } else { - options.protocol = this._url.protocol - options.hostname = this._url.hostname - options.port = this._url.port + const httpOptions = urlToHttpOptions(this._url) + options.protocol = httpOptions.protocol + options.hostname = httpOptions.hostname + options.port = httpOptions.port } this._logger.debug(() => { @@ -145,7 +199,7 @@ class AgentExporter { this._logger.error(`Error from the agent: ${err.message}`) return } else if (err) { - reject(new Error('Profiler agent export back-off period expired')) + reject(err) return } diff --git a/packages/dd-trace/src/profiling/exporters/file.js b/packages/dd-trace/src/profiling/exporters/file.js index aff94ca372c..724eac4656b 100644 --- a/packages/dd-trace/src/profiling/exporters/file.js +++ b/packages/dd-trace/src/profiling/exporters/file.js @@ -2,6 +2,7 @@ const fs = require('fs') const { promisify } = require('util') +const { threadId } = require('worker_threads') const writeFile = promisify(fs.writeFile) function formatDateTime (t) { @@ -19,7 +20,7 @@ class FileExporter { const types = Object.keys(profiles) const dateStr = formatDateTime(end) const tasks = types.map(type => { - return writeFile(`${this._pprofPrefix}${type}_${dateStr}.pprof`, profiles[type]) + return writeFile(`${this._pprofPrefix}${type}_worker_${threadId}_${dateStr}.pprof`, profiles[type]) }) return Promise.all(tasks) diff --git a/packages/dd-trace/src/profiling/loggers/console.js b/packages/dd-trace/src/profiling/loggers/console.js index ddc7f8fb6f2..263f79bd43e 100644 --- a/packages/dd-trace/src/profiling/loggers/console.js +++ b/packages/dd-trace/src/profiling/loggers/console.js @@ -12,7 +12,7 @@ const mapping = { class ConsoleLogger { constructor (options = {}) { - this._level = mapping[options.level] || mapping['error'] + this._level = mapping[options.level] || mapping.error } debug (message) { diff --git a/packages/dd-trace/src/profiling/profiler.js b/packages/dd-trace/src/profiling/profiler.js index 4e5882189a9..3e6c5d7f618 100644 --- a/packages/dd-trace/src/profiling/profiler.js +++ b/packages/dd-trace/src/profiling/profiler.js @@ -3,6 +3,10 @@ const { EventEmitter } = require('events') const { Config } = require('./config') const { snapshotKinds } = require('./constants') +const { threadNamePrefix } = require('./profilers/shared') +const dc = require('dc-polyfill') + +const profileSubmittedChannel = dc.channel('datadog:profiling:profile-submitted') function maybeSourceMap (sourceMap, SourceMapper, debug) { if (!sourceMap) return @@ -11,6 +15,12 @@ function maybeSourceMap (sourceMap, SourceMapper, debug) { ], debug) } +function logError (logger, err) { + if (logger) { + logger.error(err) + } +} + class Profiler extends EventEmitter { constructor () { super() @@ -24,18 +34,19 @@ class Profiler extends EventEmitter { start (options) { return this._start(options).catch((err) => { - if (options.logger) { - options.logger.error(err) - } + logError(options.logger, err) return false }) } + _logError (err) { + logError(this._logger, err) + } + async _start (options) { if (this._enabled) return true const config = this._config = new Config(options) - if (!config.enabled) return false this._logger = config.logger this._enabled = true @@ -49,7 +60,7 @@ class Profiler extends EventEmitter { setLogger(config.logger) mapper = await maybeSourceMap(config.sourceMap, SourceMapper, config.debugSourceMaps) - if (config.SourceMap && config.debugSourceMaps) { + if (config.sourceMap && config.debugSourceMaps) { this._logger.debug(() => { return mapper.infoMap.size === 0 ? 'Found no source maps' @@ -57,23 +68,24 @@ class Profiler extends EventEmitter { }) } } catch (err) { - this._logger.error(err) + this._logError(err) } try { + const start = new Date() for (const profiler of config.profilers) { // TODO: move this out of Profiler when restoring sourcemap support profiler.start({ mapper, nearOOMCallback: this._nearOOMExport.bind(this) }) - this._logger.debug(`Started ${profiler.type} profiler`) + this._logger.debug(`Started ${profiler.type} profiler in ${threadNamePrefix} thread`) } - this._capture(this._timeoutInterval) + this._capture(this._timeoutInterval, start) return true } catch (e) { - this._logger.error(e) + this._logError(e) this._stop() return false } @@ -96,7 +108,7 @@ class Profiler extends EventEmitter { // collect and export current profiles // once collect returns, profilers can be safely stopped - this._collect(snapshotKinds.ON_SHUTDOWN) + this._collect(snapshotKinds.ON_SHUTDOWN, false) this._stop() } @@ -107,18 +119,16 @@ class Profiler extends EventEmitter { for (const profiler of this._config.profilers) { profiler.stop() - this._logger.debug(`Stopped ${profiler.type} profiler`) + this._logger.debug(`Stopped ${profiler.type} profiler in ${threadNamePrefix} thread`) } clearTimeout(this._timer) this._timer = undefined - - return this } - _capture (timeout) { + _capture (timeout, start) { if (!this._enabled) return - this._lastStart = new Date() + this._lastStart = start if (!this._timer || timeout !== this._timeoutInterval) { this._timer = setTimeout(() => this._collect(snapshotKinds.PERIODIC), timeout) this._timer.unref() @@ -127,53 +137,69 @@ class Profiler extends EventEmitter { } } - async _collect (snapshotKind) { + async _collect (snapshotKind, restart = true) { if (!this._enabled) return - const start = this._lastStart - const end = new Date() + const startDate = this._lastStart + const endDate = new Date() const profiles = [] const encodedProfiles = {} try { + if (Object.keys(this._config.profilers).length === 0) { + throw new Error('No profile types configured.') + } + // collect profiles synchronously so that profilers can be safely stopped asynchronously for (const profiler of this._config.profilers) { - const profile = profiler.profile() + const profile = profiler.profile(restart, startDate, endDate) + if (!restart) { + this._logger.debug(`Stopped ${profiler.type} profiler in ${threadNamePrefix} thread`) + } if (!profile) continue profiles.push({ profiler, profile }) } + if (restart) { + this._capture(this._timeoutInterval, endDate) + } + // encode and export asynchronously for (const { profiler, profile } of profiles) { - encodedProfiles[profiler.type] = await profiler.encode(profile) - this._logger.debug(() => { - const profileJson = JSON.stringify(profile, (key, value) => { - return typeof value === 'bigint' ? value.toString() : value + try { + encodedProfiles[profiler.type] = await profiler.encode(profile) + this._logger.debug(() => { + const profileJson = JSON.stringify(profile, (key, value) => { + return typeof value === 'bigint' ? value.toString() : value + }) + return `Collected ${profiler.type} profile: ` + profileJson }) - return `Collected ${profiler.type} profile: ` + profileJson - }) + } catch (err) { + // If encoding one of the profile types fails, we should still try to + // encode and submit the other profile types. + this._logError(err) + } } - this._capture(this._timeoutInterval) - await this._submit(encodedProfiles, start, end, snapshotKind) - this._logger.debug('Submitted profiles') + if (Object.keys(encodedProfiles).length > 0) { + await this._submit(encodedProfiles, startDate, endDate, snapshotKind) + profileSubmittedChannel.publish() + this._logger.debug('Submitted profiles') + } } catch (err) { - this._logger.error(err) + this._logError(err) this._stop() } } _submit (profiles, start, end, snapshotKind) { - if (!Object.keys(profiles).length) { - return Promise.reject(new Error('No profiles to submit')) - } const { tags } = this._config const tasks = [] tags.snapshot = snapshotKind for (const exporter of this._config.exporters) { const task = exporter.export({ profiles, start, end, tags }) - .catch(err => this._logger.error(err)) + .catch(err => this._logError(err)) tasks.push(task) } @@ -195,13 +221,13 @@ class ServerlessProfiler extends Profiler { this._flushAfterIntervals = this._config.flushInterval / 1000 } - async _collect (snapshotKind) { - if (this._profiledIntervals >= this._flushAfterIntervals) { + async _collect (snapshotKind, restart = true) { + if (this._profiledIntervals >= this._flushAfterIntervals || !restart) { this._profiledIntervals = 0 - await super._collect(snapshotKind) + await super._collect(snapshotKind, restart) } else { this._profiledIntervals += 1 - this._capture(this._timeoutInterval) + this._capture(this._timeoutInterval, new Date()) // Don't submit profile until 65 (flushAfterIntervals) intervals have elapsed } } diff --git a/packages/dd-trace/src/profiling/profilers/event_plugins/dns.js b/packages/dd-trace/src/profiling/profilers/event_plugins/dns.js new file mode 100644 index 00000000000..29b1e62775f --- /dev/null +++ b/packages/dd-trace/src/profiling/profilers/event_plugins/dns.js @@ -0,0 +1,13 @@ +const EventPlugin = require('./event') + +class DNSPlugin extends EventPlugin { + static get id () { + return 'dns' + } + + static get entryType () { + return 'dns' + } +} + +module.exports = DNSPlugin diff --git a/packages/dd-trace/src/profiling/profilers/event_plugins/dns_lookup.js b/packages/dd-trace/src/profiling/profilers/event_plugins/dns_lookup.js new file mode 100644 index 00000000000..b72b0eb6205 --- /dev/null +++ b/packages/dd-trace/src/profiling/profilers/event_plugins/dns_lookup.js @@ -0,0 +1,16 @@ +const DNSPlugin = require('./dns') + +class DNSLookupPlugin extends DNSPlugin { + static get operation () { + return 'lookup' + } + + extendEvent (event, startEvent) { + event.name = 'lookup' + event.detail = { hostname: startEvent[0] } + + return event + } +} + +module.exports = DNSLookupPlugin diff --git a/packages/dd-trace/src/profiling/profilers/event_plugins/dns_lookupservice.js b/packages/dd-trace/src/profiling/profilers/event_plugins/dns_lookupservice.js new file mode 100644 index 00000000000..45860eea7aa --- /dev/null +++ b/packages/dd-trace/src/profiling/profilers/event_plugins/dns_lookupservice.js @@ -0,0 +1,16 @@ +const DNSPlugin = require('./dns') + +class DNSLookupServicePlugin extends DNSPlugin { + static get operation () { + return 'lookup_service' + } + + extendEvent (event, startEvent) { + event.name = 'lookupService' + event.detail = { host: startEvent[0], port: startEvent[1] } + + return event + } +} + +module.exports = DNSLookupServicePlugin diff --git a/packages/dd-trace/src/profiling/profilers/event_plugins/dns_resolve.js b/packages/dd-trace/src/profiling/profilers/event_plugins/dns_resolve.js new file mode 100644 index 00000000000..f390e60c375 --- /dev/null +++ b/packages/dd-trace/src/profiling/profilers/event_plugins/dns_resolve.js @@ -0,0 +1,24 @@ +const DNSPlugin = require('./dns') + +const queryNames = new Map() + +class DNSResolvePlugin extends DNSPlugin { + static get operation () { + return 'resolve' + } + + extendEvent (event, startEvent) { + const rrtype = startEvent[1] + let name = queryNames.get(rrtype) + if (!name) { + name = `query${rrtype}` + queryNames.set(rrtype, name) + } + event.name = name + event.detail = { host: startEvent[0] } + + return event + } +} + +module.exports = DNSResolvePlugin diff --git a/packages/dd-trace/src/profiling/profilers/event_plugins/dns_reverse.js b/packages/dd-trace/src/profiling/profilers/event_plugins/dns_reverse.js new file mode 100644 index 00000000000..67ad56c9057 --- /dev/null +++ b/packages/dd-trace/src/profiling/profilers/event_plugins/dns_reverse.js @@ -0,0 +1,16 @@ +const DNSPlugin = require('./dns') + +class DNSReversePlugin extends DNSPlugin { + static get operation () { + return 'reverse' + } + + extendEvent (event, startEvent) { + event.name = 'getHostByAddr' + event.detail = { host: startEvent[0] } + + return event + } +} + +module.exports = DNSReversePlugin diff --git a/packages/dd-trace/src/profiling/profilers/event_plugins/event.js b/packages/dd-trace/src/profiling/profilers/event_plugins/event.js new file mode 100644 index 00000000000..f47a3468f78 --- /dev/null +++ b/packages/dd-trace/src/profiling/profilers/event_plugins/event.js @@ -0,0 +1,48 @@ +const { AsyncLocalStorage } = require('async_hooks') +const TracingPlugin = require('../../../plugins/tracing') +const { performance } = require('perf_hooks') + +// We are leveraging the TracingPlugin class for its functionality to bind +// start/error/finish methods to the appropriate diagnostic channels. +class EventPlugin extends TracingPlugin { + constructor (eventHandler) { + super() + this.eventHandler = eventHandler + this.store = new AsyncLocalStorage() + this.entryType = this.constructor.entryType + } + + start (startEvent) { + this.store.enterWith({ + startEvent, + startTime: performance.now() + }) + } + + error () { + this.store.getStore().error = true + } + + finish () { + const { startEvent, startTime, error } = this.store.getStore() + if (error) { + return // don't emit perf events for failed operations + } + const duration = performance.now() - startTime + + const context = this.activeSpan?.context() + const _ddSpanId = context?.toSpanId() + const _ddRootSpanId = context?._trace.started[0]?.context().toSpanId() || _ddSpanId + + const event = { + entryType: this.entryType, + startTime, + duration, + _ddSpanId, + _ddRootSpanId + } + this.eventHandler(this.extendEvent(event, startEvent)) + } +} + +module.exports = EventPlugin diff --git a/packages/dd-trace/src/profiling/profilers/event_plugins/net.js b/packages/dd-trace/src/profiling/profilers/event_plugins/net.js new file mode 100644 index 00000000000..ffd99bbda70 --- /dev/null +++ b/packages/dd-trace/src/profiling/profilers/event_plugins/net.js @@ -0,0 +1,24 @@ +const EventPlugin = require('./event') + +class NetPlugin extends EventPlugin { + static get id () { + return 'net' + } + + static get operation () { + return 'tcp' + } + + static get entryType () { + return 'net' + } + + extendEvent (event, { options }) { + event.name = 'connect' + event.detail = options + + return event + } +} + +module.exports = NetPlugin diff --git a/packages/dd-trace/src/profiling/profilers/events.js b/packages/dd-trace/src/profiling/profilers/events.js index 4f51e5efdfb..f8f43b06a9a 100644 --- a/packages/dd-trace/src/profiling/profilers/events.js +++ b/packages/dd-trace/src/profiling/profilers/events.js @@ -1,12 +1,8 @@ -const { performance, constants, PerformanceObserver } = require('node:perf_hooks') -const { END_TIMESTAMP, THREAD_NAME, threadNamePrefix } = require('./shared') -const semver = require('semver') +const { performance, constants, PerformanceObserver } = require('perf_hooks') +const { END_TIMESTAMP_LABEL, SPAN_ID_LABEL, LOCAL_ROOT_SPAN_ID_LABEL } = require('./shared') const { Function, Label, Line, Location, Profile, Sample, StringTable, ValueType } = require('pprof-format') const pprof = require('@datadog/pprof/') -// Format of perf_hooks events changed with Node 16, we need to be mindful of it. -const node16 = semver.gte(process.version, '16.0.0') - // perf_hooks uses millis, with fractional part representing nanos. We emit nanos into the pprof file. const MS_TO_NS = 1000000 @@ -15,6 +11,8 @@ const MS_TO_NS = 1000000 const pprofValueType = 'timeline' const pprofValueUnit = 'nanoseconds' +const dateOffset = BigInt(Math.round(performance.timeOrigin * MS_TO_NS)) + function labelFromStr (stringTable, key, valStr) { return new Label({ key, str: stringTable.dedup(valStr) }) } @@ -46,7 +44,7 @@ class GCDecorator { } decorateSample (sampleInput, item) { - const { kind, flags } = node16 ? item.detail : item + const { kind, flags } = item.detail sampleInput.label.push(this.kindLabels[kind]) const reasonLabel = this.getReasonLabel(flags) if (reasonLabel) { @@ -80,7 +78,7 @@ class DNSDecorator { this.operationNameLabelKey = stringTable.dedup('operation') this.hostLabelKey = stringTable.dedup('host') this.addressLabelKey = stringTable.dedup('address') - this.lanes = [] + this.portLabelKey = stringTable.dedup('port') } decorateSample (sampleInput, item) { @@ -97,7 +95,8 @@ class DNSDecorator { addLabel(this.hostLabelKey, detail.hostname) break case 'lookupService': - addLabel(this.addressLabelKey, `${detail.host}:${detail.port}`) + addLabel(this.addressLabelKey, detail.host) + labels.push(new Label({ key: this.portLabelKey, num: detail.port })) break case 'getHostByAddr': addLabel(this.addressLabelKey, detail.host) @@ -107,152 +106,237 @@ class DNSDecorator { addLabel(this.hostLabelKey, detail.host) } } - labels.push(this.getLaneLabelFor(item)) } +} - // Maintains "lanes" (or virtual threads) to avoid overlaps in events. The - // decorator starts out with no lanes, and dynamically adds them as needed. - // Every event is put in the first lane where it doesn't overlap with the last - // event in that lane. If there's no lane without overlaps, a new lane is - // created. - getLaneLabelFor (item) { - const startTime = item.startTime - const endTime = startTime + item.duration - - // Biases towards populating earlier lanes, but at least it's simple - for (const lane of this.lanes) { - if (lane.endTime <= startTime) { - lane.endTime = endTime - return lane.label - } +class NetDecorator { + constructor (stringTable) { + this.stringTable = stringTable + this.operationNameLabelKey = stringTable.dedup('operation') + this.hostLabelKey = stringTable.dedup('host') + this.portLabelKey = stringTable.dedup('port') + } + + decorateSample (sampleInput, item) { + const labels = sampleInput.label + const stringTable = this.stringTable + function addLabel (labelNameKey, labelValue) { + labels.push(labelFromStr(stringTable, labelNameKey, labelValue)) + } + const op = item.name + addLabel(this.operationNameLabelKey, op) + if (op === 'connect') { + const detail = item.detail + addLabel(this.hostLabelKey, detail.host) + labels.push(new Label({ key: this.portLabelKey, num: detail.port })) } - const label = labelFromStrStr( - this.stringTable, - THREAD_NAME, - `${threadNamePrefix} DNS-${this.lanes.length}` - ) - this.lanes.push({ endTime, label }) - return label } } // Keys correspond to PerformanceEntry.entryType, values are constructor // functions for type-specific decorators. const decoratorTypes = { - gc: GCDecorator + gc: GCDecorator, + dns: DNSDecorator, + net: NetDecorator } -// Needs at least node 16 for DNS -if (node16) { - decoratorTypes.dns = DNSDecorator + +// Translates performance entries into pprof samples. +class EventSerializer { + constructor () { + this.stringTable = new StringTable() + this.samples = [] + this.locations = [] + this.functions = [] + this.decorators = {} + + // A synthetic single-frame location to serve as the location for timeline + // samples. We need these as the profiling backend (mimicking official pprof + // tool's behavior) ignores these. + const fn = new Function({ id: this.functions.length + 1, name: this.stringTable.dedup('') }) + this.functions.push(fn) + const line = new Line({ functionId: fn.id }) + const location = new Location({ id: this.locations.length + 1, line: [line] }) + this.locations.push(location) + this.locationId = [location.id] + + this.timestampLabelKey = this.stringTable.dedup(END_TIMESTAMP_LABEL) + this.spanIdKey = this.stringTable.dedup(SPAN_ID_LABEL) + this.rootSpanIdKey = this.stringTable.dedup(LOCAL_ROOT_SPAN_ID_LABEL) + } + + addEvent (item) { + const { entryType, startTime, duration, _ddSpanId, _ddRootSpanId } = item + let decorator = this.decorators[entryType] + if (!decorator) { + const DecoratorCtor = decoratorTypes[entryType] + if (DecoratorCtor) { + decorator = new DecoratorCtor(this.stringTable) + decorator.eventTypeLabel = labelFromStrStr(this.stringTable, 'event', entryType) + this.decorators[entryType] = decorator + } else { + // Shouldn't happen but it's better to not rely on observer only getting + // requested event types. + return + } + } + const endTime = startTime + duration + const label = [ + decorator.eventTypeLabel, + new Label({ key: this.timestampLabelKey, num: dateOffset + BigInt(Math.round(endTime * MS_TO_NS)) }) + ] + if (_ddSpanId) { + label.push(labelFromStr(this.stringTable, this.spanIdKey, _ddSpanId)) + } + if (_ddRootSpanId) { + label.push(labelFromStr(this.stringTable, this.rootSpanIdKey, _ddRootSpanId)) + } + + const sampleInput = { + value: [Math.round(duration * MS_TO_NS)], + locationId: this.locationId, + label + } + decorator.decorateSample(sampleInput, item) + this.samples.push(new Sample(sampleInput)) + } + + createProfile (startDate, endDate) { + const timeValueType = new ValueType({ + type: this.stringTable.dedup(pprofValueType), + unit: this.stringTable.dedup(pprofValueUnit) + }) + + return new Profile({ + sampleType: [timeValueType], + timeNanos: endDate.getTime() * MS_TO_NS, + periodType: timeValueType, + period: 1, + durationNanos: (endDate.getTime() - startDate.getTime()) * MS_TO_NS, + sample: this.samples, + location: this.locations, + function: this.functions, + stringTable: this.stringTable + }) + } } /** - * This class generates pprof files with timeline events sourced from Node.js - * performance measurement APIs. + * Class that sources timeline events through Node.js performance measurement APIs. */ -class EventsProfiler { - constructor (options = {}) { - this.type = 'events' - this._flushIntervalNanos = (options.flushInterval || 60000) * 1e6 // 60 sec - this._observer = undefined - this.entries = [] +class NodeApiEventSource { + constructor (eventHandler, entryTypes) { + this.eventHandler = eventHandler + this.observer = undefined + this.entryTypes = entryTypes || Object.keys(decoratorTypes) } start () { + // if already started, do nothing + if (this.observer) return + function add (items) { - this.entries.push(...items.getEntries()) - } - if (!this._observer) { - this._observer = new PerformanceObserver(add.bind(this)) + for (const item of items.getEntries()) { + this.eventHandler(item) + } } - this._observer.observe({ entryTypes: Object.keys(decoratorTypes) }) + + this.observer = new PerformanceObserver(add.bind(this)) + this.observer.observe({ entryTypes: this.entryTypes }) } stop () { - if (this._observer) { - this._observer.disconnect() + if (this.observer) { + this.observer.disconnect() + this.observer = undefined } } +} - profile () { - if (this.entries.length === 0) { - // No events in the period; don't produce a profile - return null - } +class DatadogInstrumentationEventSource { + constructor (eventHandler) { + this.plugins = ['dns_lookup', 'dns_lookupservice', 'dns_resolve', 'dns_reverse', 'net'].map(m => { + const Plugin = require(`./event_plugins/${m}`) + return new Plugin(eventHandler) + }) - const stringTable = new StringTable() - const locations = [] - const functions = [] + this.started = false + } - // A synthetic single-frame location to serve as the location for timeline - // samples. We need these as the profiling backend (mimicking official pprof - // tool's behavior) ignores these. - const locationId = (() => { - const fn = new Function({ id: functions.length + 1, name: stringTable.dedup('') }) - functions.push(fn) - const line = new Line({ functionId: fn.id }) - const location = new Location({ id: locations.length + 1, line: [line] }) - locations.push(location) - return [location.id] - })() - - const decorators = {} - for (const [eventType, DecoratorCtor] of Object.entries(decoratorTypes)) { - const decorator = new DecoratorCtor(stringTable) - decorator.eventTypeLabel = labelFromStrStr(stringTable, 'event', eventType) - decorators[eventType] = decorator + start () { + if (!this.started) { + this.plugins.forEach(p => p.configure({ enabled: true })) + this.started = true } - const timestampLabelKey = stringTable.dedup(END_TIMESTAMP) + } - let durationFrom = Number.POSITIVE_INFINITY - let durationTo = 0 - const dateOffset = BigInt(Math.round(performance.timeOrigin * MS_TO_NS)) + stop () { + if (this.started) { + this.plugins.forEach(p => p.configure({ enabled: false })) + this.started = false + } + } +} - const samples = this.entries.map((item) => { - const decorator = decorators[item.entryType] - if (!decorator) { - // Shouldn't happen but it's better to not rely on observer only getting - // requested event types. - return null - } - const { startTime, duration } = item - const endTime = startTime + duration - if (durationFrom > startTime) durationFrom = startTime - if (durationTo < endTime) durationTo = endTime - const sampleInput = { - value: [Math.round(duration * MS_TO_NS)], - locationId, - label: [ - decorator.eventTypeLabel, - new Label({ key: timestampLabelKey, num: dateOffset + BigInt(Math.round(endTime * MS_TO_NS)) }) - ] - } - decorator.decorateSample(sampleInput, item) - return new Sample(sampleInput) - }).filter(v => v) +class CompositeEventSource { + constructor (sources) { + this.sources = sources + } - this.entries = [] + start () { + this.sources.forEach(s => s.start()) + } - const timeValueType = new ValueType({ - type: stringTable.dedup(pprofValueType), - unit: stringTable.dedup(pprofValueUnit) - }) + stop () { + this.sources.forEach(s => s.stop()) + } +} - return new Profile({ - sampleType: [timeValueType], - timeNanos: dateOffset + BigInt(Math.round(durationFrom * MS_TO_NS)), - periodType: timeValueType, - period: this._flushIntervalNanos, - durationNanos: Math.max(0, Math.round((durationTo - durationFrom) * MS_TO_NS)), - sample: samples, - location: locations, - function: functions, - stringTable: stringTable - }) +/** + * This class generates pprof files with timeline events. It combines an event + * source with an event serializer. + */ +class EventsProfiler { + constructor (options = {}) { + this.type = 'events' + this.eventSerializer = new EventSerializer() + + const eventHandler = event => { + this.eventSerializer.addEvent(event) + } + + if (options.codeHotspotsEnabled) { + // Use Datadog instrumentation to collect events with span IDs. Still use + // Node API for GC events. + this.eventSource = new CompositeEventSource([ + new DatadogInstrumentationEventSource(eventHandler), + new NodeApiEventSource(eventHandler, ['gc']) + ]) + } else { + // Use Node API instrumentation to collect events without span IDs + this.eventSource = new NodeApiEventSource(eventHandler) + } + } + + start () { + this.eventSource.start() + } + + stop () { + this.eventSource.stop() + } + + profile (restart, startDate, endDate) { + if (!restart) { + this.stop() + } + const thatEventSerializer = this.eventSerializer + this.eventSerializer = new EventSerializer() + return () => thatEventSerializer.createProfile(startDate, endDate) } encode (profile) { - return pprof.encode(profile) + return pprof.encode(profile()) } } diff --git a/packages/dd-trace/src/profiling/profilers/shared.js b/packages/dd-trace/src/profiling/profilers/shared.js index 49acc6ced61..8f1e15c75c2 100644 --- a/packages/dd-trace/src/profiling/profilers/shared.js +++ b/packages/dd-trace/src/profiling/profilers/shared.js @@ -1,9 +1,50 @@ 'use strict' -const { isMainThread, threadId } = require('node:worker_threads') +const { isMainThread, threadId } = require('worker_threads') + +const END_TIMESTAMP_LABEL = 'end_timestamp_ns' +const THREAD_NAME_LABEL = 'thread name' +const OS_THREAD_ID_LABEL = 'os thread id' +const THREAD_ID_LABEL = 'thread id' +const SPAN_ID_LABEL = 'span id' +const LOCAL_ROOT_SPAN_ID_LABEL = 'local root span id' + +const threadNamePrefix = isMainThread ? 'Main' : `Worker #${threadId}` +const eventLoopThreadName = `${threadNamePrefix} Event Loop` + +function getThreadLabels () { + const pprof = require('@datadog/pprof') + const nativeThreadId = pprof.getNativeThreadId() + return { + [THREAD_NAME_LABEL]: eventLoopThreadName, + [THREAD_ID_LABEL]: `${threadId}`, + [OS_THREAD_ID_LABEL]: `${nativeThreadId}` + } +} + +function cacheThreadLabels () { + let labels + return () => { + if (!labels) { + labels = getThreadLabels() + } + return labels + } +} + +function getNonJSThreadsLabels () { + return { [THREAD_NAME_LABEL]: 'Non-JS threads', [THREAD_ID_LABEL]: 'NA', [OS_THREAD_ID_LABEL]: 'NA' } +} module.exports = { - END_TIMESTAMP: 'end_timestamp_ns', - THREAD_NAME: 'thread name', - threadNamePrefix: isMainThread ? 'Main' : `Worker #${threadId}` + END_TIMESTAMP_LABEL, + THREAD_NAME_LABEL, + THREAD_ID_LABEL, + OS_THREAD_ID_LABEL, + SPAN_ID_LABEL, + LOCAL_ROOT_SPAN_ID_LABEL, + threadNamePrefix, + eventLoopThreadName, + getNonJSThreadsLabels, + getThreadLabels: cacheThreadLabels() } diff --git a/packages/dd-trace/src/profiling/profilers/space.js b/packages/dd-trace/src/profiling/profilers/space.js index 767136603ba..7a250fa105f 100644 --- a/packages/dd-trace/src/profiling/profilers/space.js +++ b/packages/dd-trace/src/profiling/profilers/space.js @@ -1,6 +1,7 @@ 'use strict' const { oomExportStrategies } = require('../constants') +const { getThreadLabels } = require('./shared') function strategiesToCallbackMode (strategies, callbackMode) { return strategies.includes(oomExportStrategies.ASYNC_CALLBACK) ? callbackMode.Async : 0 @@ -13,9 +14,12 @@ class NativeSpaceProfiler { this._stackDepth = options.stackDepth || 64 this._pprof = undefined this._oomMonitoring = options.oomMonitoring || {} + this._started = false } start ({ mapper, nearOOMCallback } = {}) { + if (this._started) return + this._mapper = mapper this._pprof = require('@datadog/pprof') this._pprof.heap.start(this._samplingInterval, this._stackDepth) @@ -30,10 +34,16 @@ class NativeSpaceProfiler { strategiesToCallbackMode(strategies, this._pprof.heap.CallbackMode) ) } + + this._started = true } - profile () { - return this._pprof.heap.profile(undefined, this._mapper) + profile (restart) { + const profile = this._pprof.heap.profile(undefined, this._mapper, getThreadLabels) + if (!restart) { + this.stop() + } + return profile } encode (profile) { @@ -41,7 +51,13 @@ class NativeSpaceProfiler { } stop () { + if (!this._started) return this._pprof.heap.stop() + this._started = false + } + + isStarted () { + return this._started } } diff --git a/packages/dd-trace/src/profiling/profilers/wall.js b/packages/dd-trace/src/profiling/profilers/wall.js index 991a44efd0a..3d7041cfecf 100644 --- a/packages/dd-trace/src/profiling/profilers/wall.js +++ b/packages/dd-trace/src/profiling/profilers/wall.js @@ -7,15 +7,20 @@ const { HTTP_METHOD, HTTP_ROUTE, RESOURCE_NAME, SPAN_TYPE } = require('../../../ const { WEB } = require('../../../../../ext/types') const runtimeMetrics = require('../../runtime_metrics') const telemetryMetrics = require('../../telemetry/metrics') -const { END_TIMESTAMP, THREAD_NAME, threadNamePrefix } = require('./shared') +const { + END_TIMESTAMP_LABEL, + SPAN_ID_LABEL, + LOCAL_ROOT_SPAN_ID_LABEL, + getNonJSThreadsLabels, + getThreadLabels +} = require('./shared') const beforeCh = dc.channel('dd-trace:storage:before') const enterCh = dc.channel('dd-trace:storage:enter') const spanFinishCh = dc.channel('dd-trace:span:finish') const profilerTelemetryMetrics = telemetryMetrics.manager.namespace('profilers') -const threadName = `${threadNamePrefix} Event Loop` -const MemoizedWebTags = Symbol('NativeWallProfiler.MemoizedWebTags') +const ProfilingContext = Symbol('NativeWallProfiler.ProfilingContext') let kSampleCount @@ -39,36 +44,42 @@ function endpointNameFromTags (tags) { ].filter(v => v).join(' ') } -function getWebTags (startedSpans, i, span) { - // Are web tags for this span already memoized? - const memoizedWebTags = span[MemoizedWebTags] - if (memoizedWebTags !== undefined) { - return memoizedWebTags - } - // No, we'll have to memoize a new value - function memoize (tags) { - span[MemoizedWebTags] = tags - return tags - } - // Is this span itself a web span? - const context = span.context() - const tags = context._tags - if (isWebServerSpan(tags)) { - return memoize(tags) - } - // It isn't. Get parent's web tags (memoize them too recursively.) - // There might be several webspans, for example with next.js, http plugin creates the first span - // and then next.js plugin creates a child span, and this child span has the correct endpoint - // information. That's why we always use the tags of the closest ancestor web span. - const parentId = context._parentId - while (--i >= 0) { - const ispan = startedSpans[i] - if (ispan.context()._spanId === parentId) { - return memoize(getWebTags(startedSpans, i, ispan)) +let channelsActivated = false +function ensureChannelsActivated () { + if (channelsActivated) return + + const { AsyncLocalStorage, createHook } = require('async_hooks') + const shimmer = require('../../../../datadog-shimmer') + + createHook({ before: () => beforeCh.publish() }).enable() + + let inRun = false + shimmer.wrap(AsyncLocalStorage.prototype, 'enterWith', function (original) { + return function (...args) { + const retVal = original.apply(this, args) + if (!inRun) enterCh.publish() + return retVal } - } - // Local root span with no web span - return memoize(null) + }) + + shimmer.wrap(AsyncLocalStorage.prototype, 'run', function (original) { + return function (store, callback, ...args) { + const wrappedCb = shimmer.wrapFunction(callback, cb => function (...args) { + inRun = false + enterCh.publish() + const retVal = cb.apply(this, args) + inRun = true + return retVal + }) + inRun = true + const retVal = original.call(this, store, wrappedCb, ...args) + enterCh.publish() + inRun = false + return retVal + } + }) + + channelsActivated = true } class NativeWallProfiler { @@ -79,13 +90,15 @@ class NativeWallProfiler { this._codeHotspotsEnabled = !!options.codeHotspotsEnabled this._endpointCollectionEnabled = !!options.endpointCollectionEnabled this._timelineEnabled = !!options.timelineEnabled + this._cpuProfilingEnabled = !!options.cpuProfilingEnabled // We need to capture span data into the sample context for either code hotspots // or endpoint collection. this._captureSpanData = this._codeHotspotsEnabled || this._endpointCollectionEnabled // We need to run the pprof wall profiler with sample contexts if we're either // capturing span data or timeline is enabled (so we need sample timestamps, and for now - // timestamps require the sample contexts feature in the pprof wall profiler.) - this._withContexts = this._captureSpanData || this._timelineEnabled + // timestamps require the sample contexts feature in the pprof wall profiler), or + // cpu profiling is enabled. + this._withContexts = this._captureSpanData || this._timelineEnabled || this._cpuProfilingEnabled this._v8ProfilerBugWorkaroundEnabled = !!options.v8ProfilerBugWorkaroundEnabled this._mapper = undefined this._pprof = undefined @@ -96,12 +109,9 @@ class NativeWallProfiler { this._enter = this._enter.bind(this) this._spanFinished = this._spanFinished.bind(this) } - this._generateLabels = this._generateLabels.bind(this) - } else { - // Explicitly assigning, to express the intent that this is meant to be - // undefined when passed to pprof.time.stop() when not using sample contexts. - this._generateLabels = undefined } + this._generateLabels = this._generateLabels.bind(this) + this._logger = options.logger this._started = false } @@ -117,6 +127,8 @@ class NativeWallProfiler { start ({ mapper } = {}) { if (this._started) return + ensureChannelsActivated() + this._mapper = mapper this._pprof = require('@datadog/pprof') kSampleCount = this._pprof.time.constants.kSampleCount @@ -135,18 +147,15 @@ class NativeWallProfiler { sourceMapper: this._mapper, withContexts: this._withContexts, lineNumbers: false, - workaroundV8Bug: this._v8ProfilerBugWorkaroundEnabled + workaroundV8Bug: this._v8ProfilerBugWorkaroundEnabled, + collectCpuTime: this._cpuProfilingEnabled }) if (this._withContexts) { - this._currentContext = {} - this._pprof.time.setContext(this._currentContext) + this._setNewContext() if (this._captureSpanData) { this._profilerState = this._pprof.time.getState() - this._lastSpan = undefined - this._lastStartedSpans = undefined - this._lastWebTags = undefined this._lastSampleCount = 0 beforeCh.subscribe(this._enter) @@ -164,51 +173,78 @@ class NativeWallProfiler { const sampleCount = this._profilerState[kSampleCount] if (sampleCount !== this._lastSampleCount) { this._lastSampleCount = sampleCount - const context = this._currentContext - this._currentContext = {} - this._pprof.time.setContext(this._currentContext) + const context = this._currentContext.ref + this._setNewContext() this._updateContext(context) } const span = getActiveSpan() - if (span) { + this._currentContext.ref = span ? this._getProfilingContext(span) : {} + } + + _getProfilingContext (span) { + let profilingContext = span[ProfilingContext] + if (profilingContext === undefined) { const context = span.context() - this._lastSpan = span const startedSpans = getStartedSpans(context) - this._lastStartedSpans = startedSpans + + let spanId + let rootSpanId + if (this._codeHotspotsEnabled) { + spanId = context._spanId + rootSpanId = startedSpans.length ? startedSpans[0].context()._spanId : context._spanId + } + + let webTags if (this._endpointCollectionEnabled) { - this._lastWebTags = getWebTags(startedSpans, startedSpans.length, span) + const tags = context._tags + if (isWebServerSpan(tags)) { + webTags = tags + } else { + // Get parent's context's web tags + const parentId = context._parentId + for (let i = startedSpans.length; --i >= 0;) { + const ispan = startedSpans[i] + if (ispan.context()._spanId === parentId) { + webTags = this._getProfilingContext(ispan).webTags + break + } + } + } } - } else { - this._lastStartedSpans = undefined - this._lastSpan = undefined - this._lastWebTags = undefined + + profilingContext = { spanId, rootSpanId, webTags } + span[ProfilingContext] = profilingContext } + return profilingContext + } + + _setNewContext () { + this._pprof.time.setContext( + this._currentContext = { + ref: {} + } + ) } _updateContext (context) { - if (!this._lastSpan) { - return + if (typeof context.spanId === 'object') { + context.spanId = context.spanId.toString(10) } - if (this._codeHotspotsEnabled) { - context.spanId = this._lastSpan.context().toSpanId() - const rootSpan = this._lastStartedSpans[0] - if (rootSpan) { - context.rootSpanId = rootSpan.context().toSpanId() - } + if (typeof context.rootSpanId === 'object') { + context.rootSpanId = context.rootSpanId.toString(10) } - if (this._lastWebTags) { - context.webTags = this._lastWebTags + if (context.webTags !== undefined && context.endpoint === undefined) { // endpoint may not be determined yet, but keep it as fallback // if tags are not available anymore during serialization - context.endpoint = endpointNameFromTags(this._lastWebTags) + context.endpoint = endpointNameFromTags(context.webTags) } } _spanFinished (span) { - if (span[MemoizedWebTags]) { - span[MemoizedWebTags] = undefined + if (span[ProfilingContext] !== undefined) { + span[ProfilingContext] = undefined } } @@ -224,35 +260,61 @@ class NativeWallProfiler { _stop (restart) { if (!this._started) return + if (this._captureSpanData) { // update last sample context if needed this._enter() this._lastSampleCount = 0 } const profile = this._pprof.time.stop(restart, this._generateLabels) + if (restart) { const v8BugDetected = this._pprof.time.v8ProfilerStuckEventLoopDetected() if (v8BugDetected !== 0) { this._reportV8bug(v8BugDetected === 1) } + } else { + if (this._captureSpanData) { + beforeCh.unsubscribe(this._enter) + enterCh.unsubscribe(this._enter) + spanFinishCh.unsubscribe(this._spanFinished) + this._profilerState = undefined + } + this._started = false } + return profile } - _generateLabels ({ context: { spanId, rootSpanId, webTags, endpoint }, timestamp }) { - const labels = this._timelineEnabled ? { - [THREAD_NAME]: threadName, + _generateLabels ({ node, context }) { + // check for special node that represents CPU time all non-JS threads. + // In that case only return a special thread name label since we cannot associate any timestamp/span/endpoint to it. + if (node.name === this._pprof.time.constants.NON_JS_THREADS_FUNCTION_NAME) { + return getNonJSThreadsLabels() + } + + if (context == null) { + // generateLabels is also called for samples without context. + // In that case just return thread labels. + return getThreadLabels() + } + + const labels = { ...getThreadLabels() } + + const { context: { ref: { spanId, rootSpanId, webTags, endpoint } }, timestamp } = context + + if (this._timelineEnabled) { // Incoming timestamps are in microseconds, we emit nanos. - [END_TIMESTAMP]: timestamp * 1000n - } : {} + labels[END_TIMESTAMP_LABEL] = timestamp * 1000n + } - if (spanId) { - labels['span id'] = spanId + if (spanId !== undefined) { + labels[SPAN_ID_LABEL] = spanId } - if (rootSpanId) { - labels['local root span id'] = rootSpanId + if (rootSpanId !== undefined) { + labels[LOCAL_ROOT_SPAN_ID_LABEL] = rootSpanId } - if (webTags && Object.keys(webTags).length !== 0) { + if (webTags !== undefined && Object.keys(webTags).length !== 0) { labels['trace endpoint'] = endpointNameFromTags(webTags) } else if (endpoint) { // fallback to endpoint computed when sample was taken @@ -262,8 +324,8 @@ class NativeWallProfiler { return labels } - profile () { - return this._stop(true) + profile (restart) { + return this._stop(restart) } encode (profile) { @@ -271,21 +333,11 @@ class NativeWallProfiler { } stop () { - if (!this._started) return - - const profile = this._stop(false) - if (this._captureSpanData) { - beforeCh.unsubscribe(this._enter) - enterCh.unsubscribe(this._enter) - spanFinishCh.unsubscribe(this._spanFinished) - this._profilerState = undefined - this._lastSpan = undefined - this._lastStartedSpans = undefined - this._lastWebTags = undefined - } + this._stop(false) + } - this._started = false - return profile + isStarted () { + return this._started } } diff --git a/packages/dd-trace/src/profiling/ssi-heuristics.js b/packages/dd-trace/src/profiling/ssi-heuristics.js new file mode 100644 index 00000000000..4790ae2b9b5 --- /dev/null +++ b/packages/dd-trace/src/profiling/ssi-heuristics.js @@ -0,0 +1,194 @@ +'use strict' + +const telemetryMetrics = require('../telemetry/metrics') +const profilersNamespace = telemetryMetrics.manager.namespace('profilers') +const dc = require('dc-polyfill') +const log = require('../log') + +// If the process lives for at least 30 seconds, it's considered long-lived +const DEFAULT_LONG_LIVED_THRESHOLD = 30000 + +/** + * This class embodies the SSI profiler-triggering heuristics and also emits telemetry metrics about + * the profiler behavior under SSI. It emits the following metrics: + * - `number_of_profiles`: The number of profiles that were submitted + * - `number_of_runtime_id`: The number of runtime IDs in the app (always 1 for Node.js, emitted + * once when the tags won't change for the remaineder of of the app's lifetime.) + * It will also add tags describing the state of heuristics triggers, the enablement choice, and + * whether actual profiles were sent (as opposed to mock profiles). There is a mock profiler that is + * activated when the profiler is not enabled, and it will emit mock profile submission events at + * the same cadence the profiler would, providing insight into how many profiles would've been + * emitted if SSI enabled profiling. Note that heuristics (and thus telemetry) is per tracer + * instance, and each worker thread will have its own instance. + */ +class SSIHeuristics { + constructor (config) { + const injectionIncludesProfiler = config.injectionEnabled.includes('profiler') + this._heuristicsActive = injectionIncludesProfiler || config.profiling.enabled === 'auto' + this._emitsTelemetry = config.injectionEnabled.length > 0 && config.profiling.enabled !== 'false' + + if (this._emitsTelemetry) { + if (config.profiling.enabled === 'true') { + this.enablementChoice = 'manually_enabled' + } else if (injectionIncludesProfiler) { + this.enablementChoice = 'ssi_enabled' + } else if (config.profiling.enabled === 'auto') { + this.enablementChoice = 'auto_enabled' + } else { + this.enablementChoice = 'ssi_not_enabled' + } + } + + const longLivedThreshold = config.profiling.longLivedThreshold || DEFAULT_LONG_LIVED_THRESHOLD + if (typeof longLivedThreshold !== 'number' || longLivedThreshold <= 0) { + this.longLivedThreshold = DEFAULT_LONG_LIVED_THRESHOLD + log.warn( + `Invalid SSIHeuristics.longLivedThreshold value: ${config.profiling.longLivedThreshold}. ` + + `Using default value: ${DEFAULT_LONG_LIVED_THRESHOLD}` + ) + } else { + this.longLivedThreshold = longLivedThreshold + } + + this.hasSentProfiles = false + this.noSpan = true + this.shortLived = true + } + + get emitsTelemetry () { + return this._emitsTelemetry + } + + get heuristicsActive () { + return this._heuristicsActive + } + + start () { + if (this.heuristicsActive || this.emitsTelemetry) { + // Used to determine short-livedness of the process. We could use the process start time as the + // reference point, but the tracer initialization point is more relevant, as we couldn't be + // collecting profiles earlier anyway. The difference is not particularly significant if the + // tracer is initialized early in the process lifetime. + setTimeout(() => { + this.shortLived = false + this._maybeTriggered() + }, this.longLivedThreshold).unref() + + this._onSpanCreated = this._onSpanCreated.bind(this) + dc.subscribe('dd-trace:span:start', this._onSpanCreated) + + if (this.emitsTelemetry) { + this._onProfileSubmitted = this._onProfileSubmitted.bind(this) + this._onMockProfileSubmitted = this._onMockProfileSubmitted.bind(this) + + dc.subscribe('datadog:profiling:profile-submitted', this._onProfileSubmitted) + dc.subscribe('datadog:profiling:mock-profile-submitted', this._onMockProfileSubmitted) + } + + this._onAppClosing = this._onAppClosing.bind(this) + dc.subscribe('datadog:telemetry:app-closing', this._onAppClosing) + } + } + + onTriggered (callback) { + switch (typeof callback) { + case 'undefined': + case 'function': + this.triggeredCallback = callback + process.nextTick(() => { + this._maybeTriggered() + }) + break + default: + // injection hardening: only usage is internal, one call site with + // a function and another with undefined, so we can throw here. + throw new TypeError('callback must be a function or undefined') + } + } + + _maybeTriggered () { + if (!this.shortLived && !this.noSpan) { + if (typeof this.triggeredCallback === 'function') { + this.triggeredCallback.call(null) + } + } + } + + _onSpanCreated () { + this.noSpan = false + this._maybeTriggered() + dc.unsubscribe('dd-trace:span:start', this._onSpanCreated) + } + + _onProfileSubmitted () { + this.hasSentProfiles = true + this._incProfileCount() + } + + _onMockProfileSubmitted () { + this._incProfileCount() + } + + _incProfileCount () { + this._ensureProfileMetrics() + this._profileCount.inc() + } + + _ensureProfileMetrics () { + const decision = [] + if (this.noSpan) { + decision.push('no_span') + } + if (this.shortLived) { + decision.push('short_lived') + } + if (decision.length === 0) { + decision.push('triggered') + } + + const tags = [ + 'installation:ssi', + `enablement_choice:${this.enablementChoice}`, + `has_sent_profiles:${this.hasSentProfiles}`, + `heuristic_hypothetical_decision:${decision.join('_')}` + ] + + this._profileCount = profilersNamespace.count('ssi_heuristic.number_of_profiles', tags) + this._runtimeIdCount = profilersNamespace.count('ssi_heuristic.number_of_runtime_id', tags) + + if ( + !this._emittedRuntimeId && + decision[0] === 'triggered' && + // When heuristics are active, hasSentProfiles can transition from false to true when the + // profiler gets started and the first profile is submitted, so we have to wait for it. + (!this.heuristicsActive || this.hasSentProfiles) + ) { + // Tags won't change anymore, so we can emit the runtime ID metric now. + this._emittedRuntimeId = true + this._runtimeIdCount.inc() + } + } + + _onAppClosing () { + if (this.emitsTelemetry) { + this._ensureProfileMetrics() + // Last ditch effort to emit a runtime ID count metric + if (!this._emittedRuntimeId) { + this._emittedRuntimeId = true + this._runtimeIdCount.inc() + } + // So we have the metrics in the final state + this._profileCount.inc(0) + + dc.unsubscribe('datadog:profiling:profile-submitted', this._onProfileSubmitted) + dc.unsubscribe('datadog:profiling:mock-profile-submitted', this._onMockProfileSubmitted) + } + + dc.unsubscribe('datadog:telemetry:app-closing', this._onAppClosing) + if (this.noSpan) { + dc.unsubscribe('dd-trace:span:start', this._onSpanCreated) + } + } +} + +module.exports = { SSIHeuristics } diff --git a/packages/dd-trace/src/profiling/ssi-telemetry-mock-profiler.js b/packages/dd-trace/src/profiling/ssi-telemetry-mock-profiler.js new file mode 100644 index 00000000000..564046e383b --- /dev/null +++ b/packages/dd-trace/src/profiling/ssi-telemetry-mock-profiler.js @@ -0,0 +1,28 @@ +'use strict' + +const dc = require('dc-polyfill') +const coalesce = require('koalas') +const profileSubmittedChannel = dc.channel('datadog:profiling:mock-profile-submitted') +const { DD_PROFILING_UPLOAD_PERIOD } = process.env + +let timerId + +module.exports = { + start: config => { + // Copied from packages/dd-trace/src/profiler.js + const flushInterval = coalesce(config.interval, Number(DD_PROFILING_UPLOAD_PERIOD) * 1000, 65 * 1000) + + timerId = setTimeout(() => { + profileSubmittedChannel.publish() + timerId.refresh() + }, flushInterval) + timerId.unref() + }, + + stop: () => { + if (timerId !== undefined) { + clearTimeout(timerId) + timerId = undefined + } + } +} diff --git a/packages/dd-trace/src/proxy.js b/packages/dd-trace/src/proxy.js index 2919ad9483b..b8916b205d4 100644 --- a/packages/dd-trace/src/proxy.js +++ b/packages/dd-trace/src/proxy.js @@ -5,19 +5,49 @@ const Config = require('./config') const runtimeMetrics = require('./runtime_metrics') const log = require('./log') const { setStartupLogPluginManager } = require('./startup-log') +const DynamicInstrumentation = require('./debugger') const telemetry = require('./telemetry') +const nomenclature = require('./service-naming') const PluginManager = require('./plugin_manager') const remoteConfig = require('./appsec/remote_config') const AppsecSdk = require('./appsec/sdk') const dogstatsd = require('./dogstatsd') +const NoopDogStatsDClient = require('./noop/dogstatsd') +const spanleak = require('./spanleak') +const { SSIHeuristics } = require('./profiling/ssi-heuristics') +const appsecStandalone = require('./appsec/standalone') + +class LazyModule { + constructor (provider) { + this.provider = provider + } + + enable (...args) { + this.module = this.provider() + this.module.enable(...args) + } + + disable () { + this.module?.disable() + } +} class Tracer extends NoopProxy { constructor () { super() this._initialized = false + this._nomenclature = nomenclature this._pluginManager = new PluginManager(this) - this.dogstatsd = new dogstatsd.NoopDogStatsDClient() + this.dogstatsd = new NoopDogStatsDClient() + this._tracingInitialized = false + this._flare = new LazyModule(() => require('./flare')) + + // these requires must work with esm bundler + this._modules = { + appsec: new LazyModule(() => require('./appsec')), + iast: new LazyModule(() => require('./appsec/iast')) + } } init (options) { @@ -27,6 +57,7 @@ class Tracer extends NoopProxy { try { const config = new Config(options) // TODO: support dynamic code config + telemetry.start(config, this._pluginManager) if (config.dogstatsd) { // Custom Metrics @@ -35,71 +66,114 @@ class Tracer extends NoopProxy { setInterval(() => { this.dogstatsd.flush() }, 10 * 1000).unref() + + process.once('beforeExit', () => { + this.dogstatsd.flush() + }) + } + + if (config.spanLeakDebug > 0) { + if (config.spanLeakDebug === spanleak.MODES.LOG) { + spanleak.enableLogging() + } else if (config.spanLeakDebug === spanleak.MODES.GC_AND_LOG) { + spanleak.enableGarbageCollection() + } + spanleak.startScrubber() } if (config.remoteConfig.enabled && !config.isCiVisibility) { - const rc = remoteConfig.enable(config) + const rc = remoteConfig.enable(config, this._modules.appsec) - rc.on('APM_TRACING', (action, conf) => { + rc.setProductHandler('APM_TRACING', (action, conf) => { if (action === 'unapply') { config.configure({}, true) } else { config.configure(conf.lib_config, true) } + this._enableOrDisableTracing(config) + }) + + rc.setProductHandler('AGENT_CONFIG', (action, conf) => { + if (!conf?.name?.startsWith('flare-log-level.')) return - if (config.tracing) { - this._tracer.configure(config) - this._pluginManager.configure(config) + if (action === 'unapply') { + this._flare.disable() + } else if (conf.config?.log_level) { + this._flare.enable(config) + this._flare.module.prepare(conf.config.log_level) } }) - } - if (config.isGCPFunction || config.isAzureFunctionConsumptionPlan) { - require('./serverless').maybeStartServerlessMiniAgent(config) - } + rc.setProductHandler('AGENT_TASK', (action, conf) => { + if (action === 'unapply' || !conf) return + if (conf.task_type !== 'tracer_flare' || !conf.args) return - if (config.profiling.enabled) { - // do not stop tracer initialization if the profiler fails to be imported - try { - const profiler = require('./profiler') - this._profilerStarted = profiler.start(config) - } catch (e) { - log.error(e) + this._flare.enable(config) + this._flare.module.send(conf.args) + }) + + if (config.dynamicInstrumentationEnabled) { + DynamicInstrumentation.start(config, rc) } } - if (!this._profilerStarted) { - this._profilerStarted = Promise.resolve(false) - } - if (config.runtimeMetrics) { - runtimeMetrics.start(config) + if (config.isGCPFunction || config.isAzureFunction) { + require('./serverless').maybeStartServerlessMiniAgent(config) } - if (config.tracing) { - // TODO: This should probably not require tracing to be enabled. - telemetry.start(config, this._pluginManager) - - // dirty require for now so zero appsec code is executed unless explicitly enabled - if (config.appsec.enabled) { - require('./appsec').enable(config) + if (config.profiling.enabled !== 'false') { + const ssiHeuristics = new SSIHeuristics(config) + ssiHeuristics.start() + let mockProfiler = null + if (config.profiling.enabled === 'true') { + this._profilerStarted = this._startProfiler(config) + } else if (ssiHeuristics.emitsTelemetry) { + // Start a mock profiler that emits mock profile-submitted events for the telemetry. + // It will be stopped if the real profiler is started by the heuristics. + mockProfiler = require('./profiling/ssi-telemetry-mock-profiler') + mockProfiler.start(config) } - this._tracer = new DatadogTracer(config) - this.appsec = new AppsecSdk(this._tracer, config) + if (ssiHeuristics.heuristicsActive) { + ssiHeuristics.onTriggered(() => { + if (mockProfiler) { + mockProfiler.stop() + } + this._startProfiler(config) + ssiHeuristics.onTriggered() // deregister this callback + }) + } - if (config.iast.enabled) { - require('./appsec/iast').enable(config, this._tracer) + if (!this._profilerStarted) { + this._profilerStarted = Promise.resolve(false) } + } - this._pluginManager.configure(config) - setStartupLogPluginManager(this._pluginManager) + if (config.runtimeMetrics) { + runtimeMetrics.start(config) + } + this._enableOrDisableTracing(config) + + if (config.tracing) { if (config.isManualApiEnabled) { const TestApiManualPlugin = require('./ci-visibility/test-api-manual/test-api-manual-plugin') this._testApiManualPlugin = new TestApiManualPlugin(this) this._testApiManualPlugin.configure({ ...config, enabled: true }) } } + if (config.ciVisAgentlessLogSubmissionEnabled) { + if (process.env.DD_API_KEY) { + const LogSubmissionPlugin = require('./ci-visibility/log-submission/log-submission-plugin') + const automaticLogPlugin = new LogSubmissionPlugin(this) + automaticLogPlugin.configure({ ...config, enabled: true }) + } else { + log.warn( + 'DD_AGENTLESS_LOG_SUBMISSION_ENABLED is set, ' + + 'but DD_API_KEY is undefined, so no automatic log submission will be performed.' + ) + } + } } catch (e) { log.error(e) } @@ -107,8 +181,46 @@ class Tracer extends NoopProxy { return this } + _startProfiler (config) { + // do not stop tracer initialization if the profiler fails to be imported + try { + return require('./profiler').start(config) + } catch (e) { + log.error(e) + } + } + + _enableOrDisableTracing (config) { + if (config.tracing !== false) { + if (config.appsec.enabled) { + this._modules.appsec.enable(config) + } + if (!this._tracingInitialized) { + const prioritySampler = appsecStandalone.configure(config) + this._tracer = new DatadogTracer(config, prioritySampler) + this.dataStreamsCheckpointer = this._tracer.dataStreamsCheckpointer + this.appsec = new AppsecSdk(this._tracer, config) + this._tracingInitialized = true + } + if (config.iast.enabled) { + this._modules.iast.enable(config, this._tracer) + } + } else if (this._tracingInitialized) { + this._modules.appsec.disable() + this._modules.iast.disable() + } + + if (this._tracingInitialized) { + this._tracer.configure(config) + this._pluginManager.configure(config) + DynamicInstrumentation.configure(config) + setStartupLogPluginManager(this._pluginManager) + } + } + profilerStarted () { if (!this._profilerStarted) { + // injection hardening: this is only ever invoked from tests. throw new Error('profilerStarted() must be called after init()') } return this._profilerStarted diff --git a/packages/dd-trace/src/rate_limiter.js b/packages/dd-trace/src/rate_limiter.js index 99b7ceb57ec..8417d777896 100644 --- a/packages/dd-trace/src/rate_limiter.js +++ b/packages/dd-trace/src/rate_limiter.js @@ -3,9 +3,9 @@ const limiter = require('limiter') class RateLimiter { - constructor (rateLimit) { + constructor (rateLimit, interval = 'second') { this._rateLimit = parseInt(rateLimit) - this._limiter = new limiter.RateLimiter(this._rateLimit, 'second') + this._limiter = new limiter.RateLimiter(this._rateLimit, interval) this._tokensRequested = 0 this._prevIntervalTokens = 0 this._prevTokensRequested = 0 diff --git a/packages/dd-trace/src/ritm.js b/packages/dd-trace/src/ritm.js index 509e9ad732e..882e1509cdf 100644 --- a/packages/dd-trace/src/ritm.js +++ b/packages/dd-trace/src/ritm.js @@ -50,8 +50,20 @@ function Hook (modules, options, onrequire) { if (patchedRequire) return + const _origRequire = Module.prototype.require patchedRequire = Module.prototype.require = function (request) { - const filename = Module._resolveFilename(request, this) + /* + If resolving the filename for a `require(...)` fails, defer to the wrapped + require implementation rather than failing right away. This allows a + possibly monkey patched `require` to work. + */ + let filename + try { + filename = Module._resolveFilename(request, this) + } catch (resolveErr) { + return _origRequire.apply(this, arguments) + } + const core = filename.indexOf(path.sep) === -1 let name, basedir, hooks // return known patched modules immediately diff --git a/packages/dd-trace/src/runtime_metrics.js b/packages/dd-trace/src/runtime_metrics.js index 37d075c2b25..b2711879a05 100644 --- a/packages/dd-trace/src/runtime_metrics.js +++ b/packages/dd-trace/src/runtime_metrics.js @@ -238,12 +238,15 @@ function captureHistograms () { * * performance.eventLoopUtilization available in Node.js >= v14.10, >= v12.19, >= v16 */ -const captureELU = ('eventLoopUtilization' in performance) ? () => { - // if elu is undefined (first run) the measurement is from start of process - elu = performance.eventLoopUtilization(elu) +let captureELU = () => {} +if ('eventLoopUtilization' in performance) { + captureELU = () => { + // if elu is undefined (first run) the measurement is from start of process + elu = performance.eventLoopUtilization(elu) - client.gauge('runtime.node.event_loop.utilization', elu.utilization) -} : () => {} + client.gauge('runtime.node.event_loop.utilization', elu.utilization) + } +} function captureCommonMetrics () { captureMemoryUsage() diff --git a/packages/dd-trace/src/sampling_rule.js b/packages/dd-trace/src/sampling_rule.js new file mode 100644 index 00000000000..d37e814153e --- /dev/null +++ b/packages/dd-trace/src/sampling_rule.js @@ -0,0 +1,131 @@ +'use strict' + +const { globMatch } = require('../src/util') +const RateLimiter = require('./rate_limiter') +const Sampler = require('./sampler') + +class AlwaysMatcher { + match () { + return true + } +} + +class GlobMatcher { + constructor (pattern, locator) { + this.pattern = pattern + this.locator = locator + } + + match (span) { + const subject = this.locator(span) + if (!subject) return false + return globMatch(this.pattern, subject) + } +} + +class RegExpMatcher { + constructor (pattern, locator) { + this.pattern = pattern + this.locator = locator + } + + match (span) { + const subject = this.locator(span) + if (!subject) return false + return this.pattern.test(subject) + } +} + +function matcher (pattern, locator) { + if (pattern instanceof RegExp) { + return new RegExpMatcher(pattern, locator) + } + + if (typeof pattern === 'string' && pattern !== '*') { + return new GlobMatcher(pattern, locator) + } + + return new AlwaysMatcher() +} + +function makeTagLocator (tag) { + return (span) => span.context()._tags[tag] +} + +function nameLocator (span) { + return span.context()._name +} + +function serviceLocator (span) { + const { _tags: tags } = span.context() + return tags.service || + tags['service.name'] || + span.tracer()._service +} + +class SamplingRule { + constructor ({ name, service, resource, tags, sampleRate = 1.0, provenance = undefined, maxPerSecond } = {}) { + this.matchers = [] + + if (name) { + this.matchers.push(matcher(name, nameLocator)) + } + if (service) { + this.matchers.push(matcher(service, serviceLocator)) + } + if (resource) { + this.matchers.push(matcher(resource, makeTagLocator('resource.name'))) + } + for (const [key, value] of Object.entries(tags || {})) { + this.matchers.push(matcher(value, makeTagLocator(key))) + } + + this._sampler = new Sampler(sampleRate) + this._limiter = undefined + this.provenance = provenance + + if (Number.isFinite(maxPerSecond)) { + this._limiter = new RateLimiter(maxPerSecond) + } + } + + static from (config) { + return new SamplingRule(config) + } + + get sampleRate () { + return this._sampler.rate() + } + + get effectiveRate () { + return this._limiter && this._limiter.effectiveRate() + } + + get maxPerSecond () { + return this._limiter && this._limiter._rateLimit + } + + match (span) { + for (const matcher of this.matchers) { + if (!matcher.match(span)) { + return false + } + } + + return true + } + + sample () { + if (!this._sampler.isSampled()) { + return false + } + + if (this._limiter) { + return this._limiter.isAllowed() + } + + return true + } +} + +module.exports = SamplingRule diff --git a/packages/dd-trace/src/serverless.js b/packages/dd-trace/src/serverless.js index 1180f0d1060..d352cae899e 100644 --- a/packages/dd-trace/src/serverless.js +++ b/packages/dd-trace/src/serverless.js @@ -4,7 +4,7 @@ const log = require('./log') function maybeStartServerlessMiniAgent (config) { if (process.platform !== 'win32' && process.platform !== 'linux') { - log.error(`Serverless Mini Agent is only supported on Windows and Linux.`) + log.error('Serverless Mini Agent is only supported on Windows and Linux.') return } @@ -34,7 +34,8 @@ function getRustBinaryPath (config) { const rustBinaryPathRoot = config.isGCPFunction ? '/workspace' : '/home/site/wwwroot' const rustBinaryPathOsFolder = process.platform === 'win32' - ? 'datadog-serverless-agent-windows-amd64' : 'datadog-serverless-agent-linux-amd64' + ? 'datadog-serverless-agent-windows-amd64' + : 'datadog-serverless-agent-linux-amd64' const rustBinaryExtension = process.platform === 'win32' ? '.exe' : '' @@ -52,18 +53,16 @@ function getIsGCPFunction () { return isDeprecatedGCPFunction || isNewerGCPFunction } -function getIsAzureFunctionConsumptionPlan () { +function getIsAzureFunction () { const isAzureFunction = process.env.FUNCTIONS_EXTENSION_VERSION !== undefined && process.env.FUNCTIONS_WORKER_RUNTIME !== undefined - const azureWebsiteSKU = process.env.WEBSITE_SKU - const isConsumptionPlan = azureWebsiteSKU === undefined || azureWebsiteSKU === 'Dynamic' - return isAzureFunction && isConsumptionPlan + return isAzureFunction } module.exports = { maybeStartServerlessMiniAgent, getIsGCPFunction, - getIsAzureFunctionConsumptionPlan, + getIsAzureFunction, getRustBinaryPath } diff --git a/packages/dd-trace/src/service-naming/schemas/v0/index.js b/packages/dd-trace/src/service-naming/schemas/v0/index.js index c2751a64bf0..1b0b746035d 100644 --- a/packages/dd-trace/src/service-naming/schemas/v0/index.js +++ b/packages/dd-trace/src/service-naming/schemas/v0/index.js @@ -3,5 +3,6 @@ const messaging = require('./messaging') const storage = require('./storage') const graphql = require('./graphql') const web = require('./web') +const serverless = require('./serverless') -module.exports = new SchemaDefinition({ messaging, storage, web, graphql }) +module.exports = new SchemaDefinition({ messaging, storage, web, graphql, serverless }) diff --git a/packages/dd-trace/src/service-naming/schemas/v0/serverless.js b/packages/dd-trace/src/service-naming/schemas/v0/serverless.js new file mode 100644 index 00000000000..fcccdcb465a --- /dev/null +++ b/packages/dd-trace/src/service-naming/schemas/v0/serverless.js @@ -0,0 +1,12 @@ +const { identityService } = require('../util') + +const serverless = { + server: { + 'azure-functions': { + opName: () => 'azure-functions.invoke', + serviceName: identityService + } + } +} + +module.exports = serverless diff --git a/packages/dd-trace/src/service-naming/schemas/v0/web.js b/packages/dd-trace/src/service-naming/schemas/v0/web.js index 0b86b3592fd..0c2228a563b 100644 --- a/packages/dd-trace/src/service-naming/schemas/v0/web.js +++ b/packages/dd-trace/src/service-naming/schemas/v0/web.js @@ -30,9 +30,37 @@ const web = { lambda: { opName: () => 'aws.request', serviceName: awsServiceV0 + }, + undici: { + opName: () => 'undici.request', + serviceName: httpPluginClientService } }, server: { + 'apollo.gateway.request': { + opName: () => 'apollo.gateway.request', + serviceName: ({ pluginConfig, tracerService }) => pluginConfig.service || tracerService + }, + 'apollo.gateway.plan': { + opName: () => 'apollo.gateway.plan', + serviceName: ({ pluginConfig, tracerService }) => pluginConfig.service || tracerService + }, + 'apollo.gateway.validate': { + opName: () => 'apollo.gateway.validate', + serviceName: ({ pluginConfig, tracerService }) => pluginConfig.service || tracerService + }, + 'apollo.gateway.execute': { + opName: () => 'apollo.gateway.execute', + serviceName: ({ pluginConfig, tracerService }) => pluginConfig.service || tracerService + }, + 'apollo.gateway.fetch': { + opName: () => 'apollo.gateway.fetch', + serviceName: ({ pluginConfig, tracerService }) => pluginConfig.service || tracerService + }, + 'apollo.gateway.postprocessing': { + opName: () => 'apollo.gateway.postprocessing', + serviceName: ({ pluginConfig, tracerService }) => pluginConfig.service || tracerService + }, grpc: { opName: () => DD_MAJOR <= 2 ? 'grpc.request' : 'grpc.server', serviceName: identityService diff --git a/packages/dd-trace/src/service-naming/schemas/v1/index.js b/packages/dd-trace/src/service-naming/schemas/v1/index.js index c2751a64bf0..1b0b746035d 100644 --- a/packages/dd-trace/src/service-naming/schemas/v1/index.js +++ b/packages/dd-trace/src/service-naming/schemas/v1/index.js @@ -3,5 +3,6 @@ const messaging = require('./messaging') const storage = require('./storage') const graphql = require('./graphql') const web = require('./web') +const serverless = require('./serverless') -module.exports = new SchemaDefinition({ messaging, storage, web, graphql }) +module.exports = new SchemaDefinition({ messaging, storage, web, graphql, serverless }) diff --git a/packages/dd-trace/src/service-naming/schemas/v1/serverless.js b/packages/dd-trace/src/service-naming/schemas/v1/serverless.js new file mode 100644 index 00000000000..fcccdcb465a --- /dev/null +++ b/packages/dd-trace/src/service-naming/schemas/v1/serverless.js @@ -0,0 +1,12 @@ +const { identityService } = require('../util') + +const serverless = { + server: { + 'azure-functions': { + opName: () => 'azure-functions.invoke', + serviceName: identityService + } + } +} + +module.exports = serverless diff --git a/packages/dd-trace/src/service-naming/schemas/v1/storage.js b/packages/dd-trace/src/service-naming/schemas/v1/storage.js index 3b1de3c63a0..96389e63652 100644 --- a/packages/dd-trace/src/service-naming/schemas/v1/storage.js +++ b/packages/dd-trace/src/service-naming/schemas/v1/storage.js @@ -1,4 +1,3 @@ - function configWithFallback ({ tracerService, pluginConfig }) { return pluginConfig.service || tracerService } diff --git a/packages/dd-trace/src/service-naming/schemas/v1/web.js b/packages/dd-trace/src/service-naming/schemas/v1/web.js index fd2a740bc46..333ccae51c3 100644 --- a/packages/dd-trace/src/service-naming/schemas/v1/web.js +++ b/packages/dd-trace/src/service-naming/schemas/v1/web.js @@ -29,9 +29,37 @@ const web = { lambda: { opName: () => 'aws.lambda.invoke', serviceName: identityService + }, + undici: { + opName: () => 'undici.request', + serviceName: httpPluginClientService } }, server: { + 'apollo.gateway.request': { + opName: () => 'apollo.gateway.request', + serviceName: ({ pluginConfig, tracerService }) => pluginConfig.service || tracerService + }, + 'apollo.gateway.plan': { + opName: () => 'apollo.gateway.plan', + serviceName: ({ pluginConfig, tracerService }) => pluginConfig.service || tracerService + }, + 'apollo.gateway.validate': { + opName: () => 'apollo.gateway.validate', + serviceName: ({ pluginConfig, tracerService }) => pluginConfig.service || tracerService + }, + 'apollo.gateway.execute': { + opName: () => 'apollo.gateway.execute', + serviceName: ({ pluginConfig, tracerService }) => pluginConfig.service || tracerService + }, + 'apollo.gateway.fetch': { + opName: () => 'apollo.gateway.fetch', + serviceName: ({ pluginConfig, tracerService }) => pluginConfig.service || tracerService + }, + 'apollo.gateway.postprocessing': { + opName: () => 'apollo.gateway.postprocessing', + serviceName: ({ pluginConfig, tracerService }) => pluginConfig.service || tracerService + }, grpc: { opName: () => 'grpc.server.request', serviceName: identityService diff --git a/packages/dd-trace/src/span_processor.js b/packages/dd-trace/src/span_processor.js index aea348b11fb..6dc19407d56 100644 --- a/packages/dd-trace/src/span_processor.js +++ b/packages/dd-trace/src/span_processor.js @@ -27,10 +27,14 @@ class SpanProcessor { const active = [] const formatted = [] const trace = spanContext._trace - const { flushMinSpans } = this._config + const { flushMinSpans, tracing } = this._config const { started, finished } = trace if (trace.record === false) return + if (tracing === false) { + this._erase(trace, active) + return + } if (started.length === finished.length || finished.length >= flushMinSpans) { this._prioritySampler.sample(spanContext) this._spanSampler.sample(spanContext) @@ -54,11 +58,11 @@ class SpanProcessor { } if (this._killAll) { - started.map(startedSpan => { + for (const startedSpan of started) { if (!startedSpan._finished) { startedSpan.finish() } - }) + } } } diff --git a/packages/dd-trace/src/span_sampler.js b/packages/dd-trace/src/span_sampler.js index d43fba63cc8..0a8900b5db5 100644 --- a/packages/dd-trace/src/span_sampler.js +++ b/packages/dd-trace/src/span_sampler.js @@ -1,67 +1,16 @@ 'use strict' -const { globMatch } = require('../src/util') -const { USER_KEEP, AUTO_KEEP } = require('../../../ext').priority -const RateLimiter = require('./rate_limiter') -const Sampler = require('./sampler') - -class SpanSamplingRule { - constructor ({ service, name, sampleRate = 1.0, maxPerSecond } = {}) { - this.service = service - this.name = name - - this._sampler = new Sampler(sampleRate) - this._limiter = undefined - - if (Number.isFinite(maxPerSecond)) { - this._limiter = new RateLimiter(maxPerSecond) - } - } - - get sampleRate () { - return this._sampler.rate() - } - - get maxPerSecond () { - return this._limiter && this._limiter._rateLimit - } - - static from (config) { - return new SpanSamplingRule(config) - } - - match (service, name) { - if (this.service && !globMatch(this.service, service)) { - return false - } - - if (this.name && !globMatch(this.name, name)) { - return false - } - - return true - } - - sample () { - if (!this._sampler.isSampled()) { - return false - } - if (this._limiter) { - return this._limiter.isAllowed() - } - - return true - } -} +const { USER_KEEP, AUTO_KEEP } = require('../../../ext').priority +const SamplingRule = require('./sampling_rule') class SpanSampler { constructor ({ spanSamplingRules = [] } = {}) { - this._rules = spanSamplingRules.map(SpanSamplingRule.from) + this._rules = spanSamplingRules.map(SamplingRule.from) } - findRule (service, name) { + findRule (context) { for (const rule of this._rules) { - if (rule.match(service, name)) { + if (rule.match(context)) { return rule } } @@ -73,14 +22,7 @@ class SpanSampler { const { started } = spanContext._trace for (const span of started) { - const context = span.context() - const tags = context._tags || {} - const name = context._name - const service = tags.service || - tags['service.name'] || - span.tracer()._service - - const rule = this.findRule(service, name) + const rule = this.findRule(span) if (rule && rule.sample()) { span.context()._spanSampling = { sampleRate: rule.sampleRate, diff --git a/packages/dd-trace/src/span_stats.js b/packages/dd-trace/src/span_stats.js index 9dd0139e34a..790166d058a 100644 --- a/packages/dd-trace/src/span_stats.js +++ b/packages/dd-trace/src/span_stats.js @@ -126,7 +126,9 @@ class SpanStatsProcessor { port, url, env, - tags + tags, + appsec, + version } = {}) { this.exporter = new SpanStatsExporter({ hostname, @@ -138,12 +140,13 @@ class SpanStatsProcessor { this.bucketSizeNs = interval * 1e9 this.buckets = new TimeBuckets() this.hostname = os.hostname() - this.enabled = enabled + this.enabled = enabled && !appsec?.standalone?.enabled this.env = env this.tags = tags || {} this.sequence = 0 + this.version = version - if (enabled) { + if (this.enabled) { this.timer = setInterval(this.onInterval.bind(this), interval * 1e3) this.timer.unref() } @@ -156,7 +159,7 @@ class SpanStatsProcessor { this.exporter.export({ Hostname: this.hostname, Env: this.env, - Version: version, + Version: this.version || version, Stats: serialized, Lang: 'javascript', TracerVersion: pkg.version, @@ -181,7 +184,7 @@ class SpanStatsProcessor { const { bucketSizeNs } = this const serializedBuckets = [] - for (const [ timeNs, bucket ] of this.buckets.entries()) { + for (const [timeNs, bucket] of this.buckets.entries()) { const bucketAggStats = [] for (const stats of bucket.values()) { diff --git a/packages/dd-trace/src/spanleak.js b/packages/dd-trace/src/spanleak.js new file mode 100644 index 00000000000..bfded4d8d3e --- /dev/null +++ b/packages/dd-trace/src/spanleak.js @@ -0,0 +1,98 @@ +'use strict' + +/* eslint-disable no-console */ + +const SortedSet = require('tlhunter-sorted-set') + +const INTERVAL = 1000 // look for expired spans every 1s +const LIFETIME = 60 * 1000 // all spans have a max lifetime of 1m + +const MODES = { + DISABLED: 0, + // METRICS_ONLY + LOG: 1, + GC_AND_LOG: 2 + // GC +} + +module.exports.MODES = MODES + +const spans = new SortedSet() + +// TODO: should these also be delivered as runtime metrics? + +// const registry = new FinalizationRegistry(name => { +// spans.del(span) // there is no span +// }) + +let interval +let mode = MODES.DISABLED + +module.exports.disable = function () { + mode = MODES.DISABLED +} + +module.exports.enableLogging = function () { + mode = MODES.LOG +} + +module.exports.enableGarbageCollection = function () { + mode = MODES.GC_AND_LOG +} + +module.exports.startScrubber = function () { + if (!isEnabled()) return + + interval = setInterval(() => { + const now = Date.now() + const expired = spans.rangeByScore(0, now) + + if (!expired.length) return + + const gc = isGarbageCollecting() + + const expirationsByType = Object.create(null) // { [spanType]: count } + + for (const wrapped of expired) { + spans.del(wrapped) + const span = wrapped.deref() + + if (!span) continue // span has already been garbage collected + + // TODO: Should we also do things like record the route to help users debug leaks? + if (!expirationsByType[span._name]) expirationsByType[span._name] = 0 + expirationsByType[span._name]++ + + if (!gc) continue // everything after this point is related to manual GC + + // TODO: what else can we do to alleviate memory usage + span.context()._tags = Object.create(null) + } + + console.log('expired spans:' + + Object.keys(expirationsByType).reduce((a, c) => `${a} ${c}: ${expirationsByType[c]}`, '')) + }, INTERVAL) +} + +module.exports.stopScrubber = function () { + clearInterval(interval) +} + +module.exports.addSpan = function (span) { + if (!isEnabled()) return + + const now = Date.now() + const expiration = now + LIFETIME + // eslint-disable-next-line no-undef + const wrapped = new WeakRef(span) + spans.add(wrapped, expiration) + // registry.register(span, span._name) +} + +function isEnabled () { + return mode > MODES.DISABLED +} + +function isGarbageCollecting () { + return mode >= MODES.GC_AND_LOG +} diff --git a/packages/dd-trace/src/startup-log.js b/packages/dd-trace/src/startup-log.js index 2cce76ed848..12086ae1168 100644 --- a/packages/dd-trace/src/startup-log.js +++ b/packages/dd-trace/src/startup-log.js @@ -6,6 +6,7 @@ const os = require('os') const { inspect } = require('util') const tracerVersion = require('../../../package.json').version +const errors = {} let config let pluginManager let samplingRules = [] @@ -36,6 +37,23 @@ function startupLog ({ agentError } = {}) { return } + const out = tracerInfo({ agentError }) + + if (agentError) { + out.agent_error = agentError.message + } + + info('DATADOG TRACER CONFIGURATION - ' + out) + if (agentError) { + warn('DATADOG TRACER DIAGNOSTIC - Agent Error: ' + agentError.message) + errors.agentError = { + code: agentError.code ? agentError.code : '', + message: `Agent Error:${agentError.message}` + } + } +} + +function tracerInfo () { const url = config.url || `http://${config.hostname || 'localhost'}:${config.port}` const out = { @@ -58,9 +76,6 @@ function startupLog ({ agentError } = {}) { out.enabled = config.enabled out.service = config.service out.agent_url = url - if (agentError) { - out.agent_error = agentError.message - } out.debug = !!config.debug out.sample_rate = config.sampler.sampleRate out.sampling_rules = samplingRules @@ -86,14 +101,7 @@ function startupLog ({ agentError } = {}) { // out.service_mapping // out.service_mapping_error - info('DATADOG TRACER CONFIGURATION - ' + out) - if (agentError) { - warn('DATADOG TRACER DIAGNOSTIC - Agent Error: ' + agentError.message) - } - - config = undefined - pluginManager = undefined - samplingRules = undefined + return out } function setStartupLogConfig (aConfig) { @@ -112,5 +120,7 @@ module.exports = { startupLog, setStartupLogConfig, setStartupLogPluginManager, - setSamplingRules + setSamplingRules, + tracerInfo, + errors } diff --git a/packages/dd-trace/src/tagger.js b/packages/dd-trace/src/tagger.js index fb922c7bd9c..41c8616a086 100644 --- a/packages/dd-trace/src/tagger.js +++ b/packages/dd-trace/src/tagger.js @@ -1,27 +1,46 @@ 'use strict' +const constants = require('./constants') const log = require('./log') +const ERROR_MESSAGE = constants.ERROR_MESSAGE +const ERROR_STACK = constants.ERROR_STACK +const ERROR_TYPE = constants.ERROR_TYPE -function add (carrier, keyValuePairs) { +const otelTagMap = { + 'deployment.environment': 'env', + 'service.name': 'service', + 'service.version': 'version' +} + +function add (carrier, keyValuePairs, parseOtelTags = false) { if (!carrier || !keyValuePairs) return if (Array.isArray(keyValuePairs)) { return keyValuePairs.forEach(tags => add(carrier, tags)) } - try { if (typeof keyValuePairs === 'string') { const segments = keyValuePairs.split(',') for (const segment of segments) { - const separatorIndex = segment.indexOf(':') + const separatorIndex = parseOtelTags ? segment.indexOf('=') : segment.indexOf(':') if (separatorIndex === -1) continue - const key = segment.slice(0, separatorIndex) + let key = segment.slice(0, separatorIndex) const value = segment.slice(separatorIndex + 1) + if (parseOtelTags && key in otelTagMap) { + key = otelTagMap[key] + } + carrier[key.trim()] = value.trim() } } else { + // HACK: to ensure otel.recordException does not influence trace.error + if (ERROR_MESSAGE in keyValuePairs || ERROR_STACK in keyValuePairs || ERROR_TYPE in keyValuePairs) { + if (!('doNotSetTraceError' in keyValuePairs)) { + carrier.setTraceError = true + } + } Object.assign(carrier, keyValuePairs) } } catch (e) { diff --git a/packages/dd-trace/src/telemetry/dependencies.js b/packages/dd-trace/src/telemetry/dependencies.js index 6d502a748f3..992dde7d2ec 100644 --- a/packages/dd-trace/src/telemetry/dependencies.js +++ b/packages/dd-trace/src/telemetry/dependencies.js @@ -6,28 +6,65 @@ const requirePackageJson = require('../require-package-json') const { sendData } = require('./send-data') const dc = require('dc-polyfill') const { fileURLToPath } = require('url') +const { isTrue } = require('../../src/util') const savedDependenciesToSend = new Set() const detectedDependencyKeys = new Set() const detectedDependencyVersions = new Set() -const FILE_URI_START = `file://` +const FILE_URI_START = 'file://' const moduleLoadStartChannel = dc.channel('dd-trace:moduleLoadStart') -let immediate, config, application, host +let immediate, config, application, host, initialLoad let isFirstModule = true +let getRetryData +let updateRetryData +function createBatchPayload (payload) { + const batchPayload = payload.map(item => { + return { + request_type: item.reqType, + payload: item.payload + } + }) + + return batchPayload +} function waitAndSend (config, application, host) { if (!immediate) { immediate = setImmediate(() => { immediate = null if (savedDependenciesToSend.size > 0) { - const dependencies = Array.from(savedDependenciesToSend.values()).splice(0, 1000).map(pair => { - savedDependenciesToSend.delete(pair) - const [name, version] = pair.split(' ') - return { name, version } - }) - sendData(config, application, host, 'app-dependencies-loaded', { dependencies }) + const dependencies = Array.from(savedDependenciesToSend.values()) + // if a depencdency is from the initial load, *always* send the event + // Otherwise, only send if dependencyCollection is enabled + .filter(dep => { + const initialLoadModule = isTrue(dep.split(' ')[2]) + const sendModule = initialLoadModule || (config.telemetry?.dependencyCollection) + + if (!sendModule) savedDependenciesToSend.delete(dep) // we'll never send it + return sendModule + }) + .splice(0, 2000) // v2 documentation specifies up to 2000 dependencies can be sent at once + .map(pair => { + savedDependenciesToSend.delete(pair) + const [name, version] = pair.split(' ') + return { name, version } + }) + let currPayload + const retryData = getRetryData() + if (retryData) { + currPayload = { reqType: 'app-dependencies-loaded', payload: { dependencies } } + } else { + if (!dependencies.length) return // no retry data and no dependencies, nothing to send + currPayload = { dependencies } + } + + const payload = retryData ? createBatchPayload([currPayload, retryData]) : currPayload + const reqType = retryData ? 'message-batch' : 'app-dependencies-loaded' + + sendData(config, application, host, reqType, payload, updateRetryData) + if (savedDependenciesToSend.size > 0) { waitAndSend(config, application, host) } @@ -76,7 +113,7 @@ function onModuleLoad (data) { const dependencyAndVersion = `${name} ${version}` if (!detectedDependencyVersions.has(dependencyAndVersion)) { - savedDependenciesToSend.add(dependencyAndVersion) + savedDependenciesToSend.add(`${dependencyAndVersion} ${initialLoad}`) detectedDependencyVersions.add(dependencyAndVersion) waitAndSend(config, application, host) @@ -89,11 +126,19 @@ function onModuleLoad (data) { } } } -function start (_config, _application, _host) { +function start (_config = {}, _application, _host, getRetryDataFunction, updateRetryDatafunction) { config = _config application = _application host = _host + initialLoad = true + getRetryData = getRetryDataFunction + updateRetryData = updateRetryDatafunction moduleLoadStartChannel.subscribe(onModuleLoad) + + // try and capture intially loaded modules in the first tick + // since, ideally, the tracer (and this module) should be loaded first, + // this should capture any first-tick dependencies + queueMicrotask(() => { initialLoad = false }) } function isDependency (filename, request) { diff --git a/packages/dd-trace/src/telemetry/index.js b/packages/dd-trace/src/telemetry/index.js index 7b3ee094787..5df7d6fcae3 100644 --- a/packages/dd-trace/src/telemetry/index.js +++ b/packages/dd-trace/src/telemetry/index.js @@ -1,27 +1,71 @@ 'use strict' - const tracerVersion = require('../../../../package.json').version const dc = require('dc-polyfill') const os = require('os') const dependencies = require('./dependencies') const { sendData } = require('./send-data') - +const { errors } = require('../startup-log') const { manager: metricsManager } = require('./metrics') -const logs = require('./logs') +const telemetryLogger = require('./logs') +const logger = require('../log') const telemetryStartChannel = dc.channel('datadog:telemetry:start') const telemetryStopChannel = dc.channel('datadog:telemetry:stop') +const telemetryAppClosingChannel = dc.channel('datadog:telemetry:app-closing') let config let pluginManager let application let host -let interval let heartbeatTimeout let heartbeatInterval +let extendedInterval +let integrations +let configWithOrigin = [] +let retryData = null +const extendedHeartbeatPayload = {} + const sentIntegrations = new Set() +function getRetryData () { + return retryData +} + +function updateRetryData (error, retryObj) { + if (error) { + if (retryObj.reqType === 'message-batch') { + const payload = retryObj.payload[0].payload + const reqType = retryObj.payload[0].request_type + retryData = { payload, reqType } + + // Since this payload failed twice it now gets save in to the extended heartbeat + const failedPayload = retryObj.payload[1].payload + const failedReqType = retryObj.payload[1].request_type + + // save away the dependencies and integration request for extended heartbeat. + if (failedReqType === 'app-integrations-change') { + if (extendedHeartbeatPayload.integrations) { + extendedHeartbeatPayload.integrations.push(failedPayload) + } else { + extendedHeartbeatPayload.integrations = [failedPayload] + } + } + if (failedReqType === 'app-dependencies-loaded') { + if (extendedHeartbeatPayload.dependencies) { + extendedHeartbeatPayload.dependencies.push(failedPayload) + } else { + extendedHeartbeatPayload.dependencies = [failedPayload] + } + } + } else { + retryData = retryObj + } + } else { + retryData = null + } +} + function getIntegrations () { const newIntegrations = [] for (const pluginName in pluginManager._pluginsByName) { @@ -38,43 +82,66 @@ function getIntegrations () { return newIntegrations } -function flatten (input, result = [], prefix = [], traversedObjects = null) { - traversedObjects = traversedObjects || new WeakSet() - if (traversedObjects.has(input)) { - return +function getProducts (config) { + const products = { + appsec: { + enabled: config.appsec.enabled + }, + profiler: { + version: tracerVersion, + enabled: profilingEnabledToBoolean(config.profiling.enabled) + } } - traversedObjects.add(input) - for (const [key, value] of Object.entries(input)) { - if (typeof value === 'object' && value !== null) { - flatten(value, result, [...prefix, key], traversedObjects) - } else { - result.push({ name: [...prefix, key].join('.'), value }) + if (errors.profilingError) { + products.profiler.error = errors.profilingError + errors.profilingError = {} + } + return products +} + +function getInstallSignature (config) { + const { installSignature: sig } = config + if (sig && (sig.id || sig.time || sig.type)) { + return { + install_id: sig.id, + install_time: sig.time, + install_type: sig.type } } - return result } -function appStarted () { - return { - integrations: getIntegrations(), - dependencies: [], - configuration: flatten(formatConfig(config)), - additional_payload: [] +function appStarted (config) { + const app = { + products: getProducts(config), + configuration: configWithOrigin } + const installSignature = getInstallSignature(config) + if (installSignature) { + app.install_signature = installSignature + } + // TODO: add app.error with correct error codes + // if (errors.agentError) { + // app.error = errors.agentError + // errors.agentError = {} + // } + return app } -function formatConfig (config) { - // format peerServiceMapping from an object to a string map in order for - // telemetry intake to accept the configuration - config.peerServiceMapping = config.peerServiceMapping - ? Object.entries(config.peerServiceMapping).map(([key, value]) => `${key}:${value}`).join(',') - : '' - return config +function appClosing () { + if (!config?.telemetry?.enabled) { + return + } + // Give chance to listeners to update metrics before shutting down. + telemetryAppClosingChannel.publish() + const { reqType, payload } = createPayload('app-closing') + sendData(config, application, host, reqType, payload) + // We flush before shutting down. + metricsManager.send(config, application, host) } function onBeforeExit () { process.removeListener('beforeExit', onBeforeExit) - sendData(config, application, host, 'app-closing') + appClosing() } function createAppObject (config) { @@ -121,16 +188,58 @@ function getTelemetryData () { return { config, application, host, heartbeatInterval } } +function createBatchPayload (payload) { + const batchPayload = payload.map(item => { + return { + request_type: item.reqType, + payload: item.payload + } + }) + + return batchPayload +} + +function createPayload (currReqType, currPayload = {}) { + if (getRetryData()) { + const payload = { reqType: currReqType, payload: currPayload } + const batchPayload = createBatchPayload([payload, retryData]) + return { reqType: 'message-batch', payload: batchPayload } + } + + return { reqType: currReqType, payload: currPayload } +} + function heartbeat (config, application, host) { heartbeatTimeout = setTimeout(() => { - sendData(config, application, host, 'app-heartbeat') + metricsManager.send(config, application, host) + telemetryLogger.send(config, application, host) + + const { reqType, payload } = createPayload('app-heartbeat') + sendData(config, application, host, reqType, payload, updateRetryData) heartbeat(config, application, host) }, heartbeatInterval).unref() return heartbeatTimeout } +function extendedHeartbeat (config) { + extendedInterval = setInterval(() => { + const appPayload = appStarted(config) + const payload = { + ...appPayload, + ...extendedHeartbeatPayload + } + sendData(config, application, host, 'app-extended-heartbeat', payload) + Object.keys(extendedHeartbeatPayload).forEach(key => delete extendedHeartbeatPayload[key]) + }, 1000 * 60 * 60 * 24).unref() + return extendedInterval +} + function start (aConfig, thePluginManager) { if (!aConfig.telemetry.enabled) { + if (aConfig.sca?.enabled) { + logger.warn('DD_APPSEC_SCA_ENABLED requires enabling telemetry to work.') + } + return } config = aConfig @@ -138,19 +247,23 @@ function start (aConfig, thePluginManager) { application = createAppObject(config) host = createHostObject() heartbeatInterval = config.telemetry.heartbeatInterval + integrations = getIntegrations() - dependencies.start(config, application, host) - logs.start(config) + dependencies.start(config, application, host, getRetryData, updateRetryData) + telemetryLogger.start(config) + + sendData(config, application, host, 'app-started', appStarted(config)) + + if (integrations.length > 0) { + sendData(config, application, host, 'app-integrations-change', + { integrations }, updateRetryData) + } - sendData(config, application, host, 'app-started', appStarted()) heartbeat(config, application, host) - interval = setInterval(() => { - metricsManager.send(config, application, host) - logs.send(config, application, host) - }, heartbeatInterval) - interval.unref() - process.on('beforeExit', onBeforeExit) + extendedHeartbeat(config) + + process.on('beforeExit', onBeforeExit) telemetryStartChannel.publish(getTelemetryData()) } @@ -158,7 +271,7 @@ function stop () { if (!config) { return } - clearInterval(interval) + clearInterval(extendedInterval) clearTimeout(heartbeatTimeout) process.removeListener('beforeExit', onBeforeExit) @@ -175,39 +288,103 @@ function updateIntegrations () { if (integrations.length === 0) { return } - sendData(config, application, host, 'app-integrations-change', { integrations }) + + const { reqType, payload } = createPayload('app-integrations-change', { integrations }) + + sendData(config, application, host, reqType, payload, updateRetryData) +} + +function formatMapForTelemetry (map) { + // format from an object to a string map in order for + // telemetry intake to accept the configuration + return map + ? Object.entries(map).map(([key, value]) => `${key}:${value}`).join(',') + : '' } function updateConfig (changes, config) { if (!config.telemetry.enabled) return if (changes.length === 0) return - // Hack to make system tests happy until we ship telemetry v2 - if (process.env.DD_INTERNAL_TELEMETRY_V2_ENABLED !== '1') return - const application = createAppObject(config) const host = createHostObject() - const names = { + const nameMapping = { sampleRate: 'DD_TRACE_SAMPLE_RATE', logInjection: 'DD_LOG_INJECTION', - headerTags: 'DD_TRACE_HEADER_TAGS' + headerTags: 'DD_TRACE_HEADER_TAGS', + tags: 'DD_TAGS', + 'sampler.rules': 'DD_TRACE_SAMPLING_RULES', + traceEnabled: 'DD_TRACE_ENABLED', + url: 'DD_TRACE_AGENT_URL', + 'sampler.rateLimit': 'DD_TRACE_RATE_LIMIT', + queryStringObfuscation: 'DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP', + version: 'DD_VERSION', + env: 'DD_ENV', + service: 'DD_SERVICE', + clientIpHeader: 'DD_TRACE_CLIENT_IP_HEADER', + 'grpc.client.error.statuses': 'DD_GRPC_CLIENT_ERROR_STATUSES', + 'grpc.server.error.statuses': 'DD_GRPC_SERVER_ERROR_STATUSES' } - const configuration = changes.map(change => ({ - name: names[change.name], - value: Array.isArray(change.value) ? change.value.join(',') : change.value, - origin: change.origin - })) + const namesNeedFormatting = new Set(['DD_TAGS', 'peerServiceMapping', 'serviceMapping']) - sendData(config, application, host, 'app-client-configuration-change', { - configuration - }) + const configuration = [] + const names = [] // list of config names whose values have been changed + + for (const change of changes) { + const name = nameMapping[change.name] || change.name + + names.push(name) + const { origin, value } = change + const entry = { name, value, origin } + + if (namesNeedFormatting.has(entry.name)) { + entry.value = formatMapForTelemetry(entry.value) + } else if (entry.name === 'url') { + if (entry.value) { + entry.value = entry.value.toString() + } + } else if (entry.name === 'DD_TRACE_SAMPLING_RULES') { + entry.value = JSON.stringify(entry.value) + } else if (Array.isArray(entry.value)) { + entry.value = value.join(',') + } + configuration.push(entry) + } + + function isNotModified (entry) { + return !names.includes(entry.name) + } + + if (!configWithOrigin.length) { + configWithOrigin = configuration + } else { + // update configWithOrigin to contain up-to-date full list of config values for app-extended-heartbeat + configWithOrigin = configWithOrigin.filter(isNotModified) + configWithOrigin = configWithOrigin.concat(configuration) + const { reqType, payload } = createPayload('app-client-configuration-change', { configuration }) + sendData(config, application, host, reqType, payload, updateRetryData) + } +} + +function profilingEnabledToBoolean (profilingEnabled) { + if (typeof profilingEnabled === 'boolean') { + return profilingEnabled + } + if (['auto', 'true'].includes(profilingEnabled)) { + return true + } + if (profilingEnabled === 'false') { + return false + } + return undefined } module.exports = { start, stop, updateIntegrations, - updateConfig + updateConfig, + appClosing } diff --git a/packages/dd-trace/src/telemetry/init-telemetry.js b/packages/dd-trace/src/telemetry/init-telemetry.js new file mode 100644 index 00000000000..a126ecc6238 --- /dev/null +++ b/packages/dd-trace/src/telemetry/init-telemetry.js @@ -0,0 +1,75 @@ +'use strict' + +const fs = require('fs') +const { spawn } = require('child_process') +const tracerVersion = require('../../../../package.json').version +const log = require('../log') + +module.exports = sendTelemetry + +if (!process.env.DD_INJECTION_ENABLED) { + module.exports = () => {} +} + +if (!process.env.DD_TELEMETRY_FORWARDER_PATH) { + module.exports = () => {} +} + +if (!fs.existsSync(process.env.DD_TELEMETRY_FORWARDER_PATH)) { + module.exports = () => {} +} + +const metadata = { + language_name: 'nodejs', + language_version: process.versions.node, + runtime_name: 'nodejs', + runtime_version: process.versions.node, + tracer_version: tracerVersion, + pid: process.pid +} + +const seen = [] +function hasSeen (point) { + if (point.name === 'abort') { + // This one can only be sent once, regardless of tags + return seen.includes('abort') + } + if (point.name === 'abort.integration') { + // For now, this is the only other one we want to dedupe + const compiledPoint = point.name + point.tags.join('') + return seen.includes(compiledPoint) + } + return false +} + +function sendTelemetry (name, tags = []) { + let points = name + if (typeof name === 'string') { + points = [{ name, tags }] + } + if (['1', 'true', 'True'].includes(process.env.DD_INJECT_FORCE)) { + points = points.filter(p => ['error', 'complete'].includes(p.name)) + } + points = points.filter(p => !hasSeen(p)) + points.forEach(p => { + p.name = `library_entrypoint.${p.name}` + }) + if (points.length === 0) { + return + } + const proc = spawn(process.env.DD_TELEMETRY_FORWARDER_PATH, ['library_entrypoint'], { + stdio: 'pipe' + }) + proc.on('error', () => { + log.error('Failed to spawn telemetry forwarder') + }) + proc.on('exit', (code) => { + if (code !== 0) { + log.error(`Telemetry forwarder exited with code ${code}`) + } + }) + proc.stdin.on('error', () => { + log.error('Failed to write telemetry data to telemetry forwarder') + }) + proc.stdin.end(JSON.stringify({ metadata, points })) +} diff --git a/packages/dd-trace/src/telemetry/logs/index.js b/packages/dd-trace/src/telemetry/logs/index.js index 4584061613e..54e7c51fa97 100644 --- a/packages/dd-trace/src/telemetry/logs/index.js +++ b/packages/dd-trace/src/telemetry/logs/index.js @@ -5,6 +5,7 @@ const logCollector = require('./log-collector') const { sendData } = require('../send-data') const telemetryLog = dc.channel('datadog:telemetry:log') +const errorLog = dc.channel('datadog:log:error') let enabled = false @@ -33,12 +34,29 @@ function onLog (log) { } } +function onErrorLog (msg) { + if (msg instanceof Error) { + onLog({ + level: 'ERROR', + message: msg.message, + stack_trace: msg.stack + }) + } else if (typeof msg === 'string') { + onLog({ + level: 'ERROR', + message: msg + }) + } +} + function start (config) { if (!config.telemetry.logCollection || enabled) return enabled = true telemetryLog.subscribe(onLog) + + errorLog.subscribe(onErrorLog) } function stop () { @@ -47,6 +65,8 @@ function stop () { if (telemetryLog.hasSubscribers) { telemetryLog.unsubscribe(onLog) } + + errorLog.unsubscribe(onErrorLog) } function send (config, application, host) { @@ -54,7 +74,7 @@ function send (config, application, host) { const logs = logCollector.drain() if (logs) { - sendData(config, application, host, 'logs', logs) + sendData(config, application, host, 'logs', { logs }) } } diff --git a/packages/dd-trace/src/telemetry/logs/log-collector.js b/packages/dd-trace/src/telemetry/logs/log-collector.js index 740f5453797..182842fc4c4 100644 --- a/packages/dd-trace/src/telemetry/logs/log-collector.js +++ b/packages/dd-trace/src/telemetry/logs/log-collector.js @@ -1,6 +1,7 @@ 'use strict' const log = require('../../log') +const { calculateDDBasePath } = require('../../util') const logs = new Map() @@ -29,6 +30,37 @@ function isValid (logEntry) { return logEntry?.level && logEntry.message } +const ddBasePath = calculateDDBasePath(__dirname) +const EOL = '\n' +const STACK_FRAME_LINE_REGEX = /^\s*at\s/gm + +function sanitize (logEntry) { + const stack = logEntry.stack_trace + if (!stack) return logEntry + + let stackLines = stack.split(EOL) + + const firstIndex = stackLines.findIndex(l => l.match(STACK_FRAME_LINE_REGEX)) + + const isDDCode = firstIndex > -1 && stackLines[firstIndex].includes(ddBasePath) + stackLines = stackLines + .filter((line, index) => (isDDCode && index < firstIndex) || line.includes(ddBasePath)) + .map(line => line.replace(ddBasePath, '')) + + logEntry.stack_trace = stackLines.join(EOL) + if (logEntry.stack_trace === '') { + // If entire stack was removed, we'd just have a message saying "omitted" + // in which case we'd rather not log it at all. + return null + } + + if (!isDDCode) { + logEntry.message = 'omitted' + } + + return logEntry +} + const logCollector = { add (logEntry) { try { @@ -37,9 +69,13 @@ const logCollector = { // NOTE: should errors have higher priority? and discard log entries with lower priority? if (logs.size >= maxEntries) { overflowedCount++ - return + return false } + logEntry = sanitize(logEntry) + if (!logEntry) { + return false + } const hash = createHash(logEntry) if (!logs.has(hash)) { logs.set(hash, logEntry) @@ -51,6 +87,11 @@ const logCollector = { return false }, + // Used for testing + hasEntry (logEntry) { + return logs.has(createHash(logEntry)) + }, + drain () { if (logs.size === 0) return diff --git a/packages/dd-trace/src/telemetry/metrics.js b/packages/dd-trace/src/telemetry/metrics.js index ae6994e39f3..34740aa7f2d 100644 --- a/packages/dd-trace/src/telemetry/metrics.js +++ b/packages/dd-trace/src/telemetry/metrics.js @@ -75,8 +75,8 @@ class CountMetric extends Metric { return this.track(value) } - dec (value = -1) { - return this.track(value) + dec (value = 1) { + return this.track(-value) } track (value = 1) { diff --git a/packages/dd-trace/src/telemetry/send-data.js b/packages/dd-trace/src/telemetry/send-data.js index f460dfdac3a..813fa427812 100644 --- a/packages/dd-trace/src/telemetry/send-data.js +++ b/packages/dd-trace/src/telemetry/send-data.js @@ -1,9 +1,13 @@ const request = require('../exporters/common/request') +const log = require('../log') +const { isTrue } = require('../util') + +let agentTelemetry = true function getHeaders (config, application, reqType) { const headers = { 'content-type': 'application/json', - 'dd-telemetry-api-version': 'v1', + 'dd-telemetry-api-version': 'v2', 'dd-telemetry-request-type': reqType, 'dd-client-library-language': application.language_name, 'dd-client-library-version': application.tracer_version @@ -12,9 +16,19 @@ function getHeaders (config, application, reqType) { if (debug) { headers['dd-telemetry-debug-enabled'] = 'true' } + if (config.apiKey) { + headers['dd-api-key'] = config.apiKey + } return headers } +function getAgentlessTelemetryEndpoint (site) { + if (site === 'datad0g.com') { // staging + return 'https://all-http-intake.logs.datad0g.com' + } + return `https://instrumentation-telemetry-intake.${site}` +} + let seqId = 0 function getPayload (payload) { @@ -28,23 +42,39 @@ function getPayload (payload) { } } -function sendData (config, application, host, reqType, payload = {}) { +function sendData (config, application, host, reqType, payload = {}, cb = () => {}) { const { hostname, port, - url + isCiVisibility } = config + let url = config.url + + const isCiVisibilityAgentlessMode = isCiVisibility && isTrue(process.env.DD_CIVISIBILITY_AGENTLESS_ENABLED) + + if (isCiVisibilityAgentlessMode) { + try { + url = url || new URL(getAgentlessTelemetryEndpoint(config.site)) + } catch (err) { + log.error(err) + // No point to do the request if the URL is invalid + return cb(err, { payload, reqType }) + } + } + const options = { url, hostname, port, method: 'POST', - path: '/telemetry/proxy/api/v2/apmtelemetry', + path: isCiVisibilityAgentlessMode ? '/api/v2/apmtelemetry' : '/telemetry/proxy/api/v2/apmtelemetry', headers: getHeaders(config, application, reqType) } + const data = JSON.stringify({ - api_version: 'v1', + api_version: 'v2', + naming_schema_version: config.spanAttributeSchema ? config.spanAttributeSchema : '', request_type: reqType, tracer_time: Math.floor(Date.now() / 1000), runtime_id: config.tags['runtime-id'], @@ -54,8 +84,35 @@ function sendData (config, application, host, reqType, payload = {}) { host }) - request(data, options, () => { - // ignore errors + request(data, options, (error) => { + if (error && process.env.DD_API_KEY && config.site) { + if (agentTelemetry) { + log.warn('Agent telemetry failed, started agentless telemetry') + agentTelemetry = false + } + // figure out which data center to send to + const backendUrl = getAgentlessTelemetryEndpoint(config.site) + const backendHeader = { ...options.headers, 'DD-API-KEY': process.env.DD_API_KEY } + const backendOptions = { + ...options, + url: backendUrl, + headers: backendHeader, + path: '/api/v2/apmtelemetry' + } + if (backendUrl) { + request(data, backendOptions, (error) => { log.error(error) }) + } else { + log.error('Invalid Telemetry URL') + } + } + + if (!error && !agentTelemetry) { + agentTelemetry = true + log.info('Started agent telemetry') + } + + // call the callback function so that we can track the error and payload + cb(error, { payload, reqType }) }) } diff --git a/packages/dd-trace/src/tracer.js b/packages/dd-trace/src/tracer.js index afa7da037b2..64b6b1be52d 100644 --- a/packages/dd-trace/src/tracer.js +++ b/packages/dd-trace/src/tracer.js @@ -8,9 +8,12 @@ const { isError } = require('./util') const { setStartupLogConfig } = require('./startup-log') const { ERROR_MESSAGE, ERROR_TYPE, ERROR_STACK } = require('../../dd-trace/src/constants') const { DataStreamsProcessor } = require('./datastreams/processor') -const { decodePathwayContext } = require('./datastreams/pathway') +const { DsmPathwayCodec } = require('./datastreams/pathway') const { DD_MAJOR } = require('../../../version') const DataStreamsContext = require('./data_streams_context') +const { DataStreamsCheckpointer } = require('./data_streams') +const { flushStartupLogs } = require('../../datadog-instrumentations/src/check_require_cache') +const log = require('./log/writer') const SPAN_TYPE = tags.SPAN_TYPE const RESOURCE_NAME = tags.RESOURCE_NAME @@ -18,11 +21,13 @@ const SERVICE_NAME = tags.SERVICE_NAME const MEASURED = tags.MEASURED class DatadogTracer extends Tracer { - constructor (config) { - super(config) + constructor (config, prioritySampler) { + super(config, prioritySampler) this._dataStreamsProcessor = new DataStreamsProcessor(config) + this.dataStreamsCheckpointer = new DataStreamsCheckpointer(this) this._scope = new Scope() setStartupLogConfig(config) + flushStartupLogs(log) } configure ({ env, sampler }) { @@ -39,13 +44,17 @@ class DatadogTracer extends Tracer { return ctx } - decodeDataStreamsContext (data) { - const ctx = decodePathwayContext(data) + decodeDataStreamsContext (carrier) { + const ctx = DsmPathwayCodec.decode(carrier) // we erase the previous context everytime we decode a new one DataStreamsContext.setDataStreamsContext(ctx) return ctx } + setOffset (offsetData) { + return this._dataStreamsProcessor.setOffset(offsetData) + } + trace (name, options, fn) { options = Object.assign({ childOf: this.scope().active() @@ -131,6 +140,7 @@ class DatadogTracer extends Tracer { setUrl (url) { this._exporter.setUrl(url) + this._dataStreamsProcessor.setUrl(url) } scope () { diff --git a/packages/dd-trace/src/util.js b/packages/dd-trace/src/util.js index 04634b62359..04048c9b187 100644 --- a/packages/dd-trace/src/util.js +++ b/packages/dd-trace/src/util.js @@ -32,13 +32,6 @@ function globMatch (pattern, subject) { if (px < pattern.length) { const c = pattern[px] switch (c) { - default: // ordinary character - if (sx < subject.length && subject[sx] === c) { - px++ - sx++ - continue - } - break case '?': if (sx < subject.length) { px++ @@ -51,6 +44,13 @@ function globMatch (pattern, subject) { nextSx = sx + 1 px++ continue + default: // ordinary character + if (sx < subject.length && subject[sx] === c) { + px++ + sx++ + continue + } + break } } if (nextSx > 0 && nextSx <= subject.length) { @@ -69,10 +69,15 @@ function calculateDDBasePath (dirname) { return dirSteps.slice(0, packagesIndex + 1).join(path.sep) + path.sep } +function hasOwn (object, prop) { + return Object.prototype.hasOwnProperty.call(object, prop) +} + module.exports = { isTrue, isFalse, isError, globMatch, - calculateDDBasePath + calculateDDBasePath, + hasOwn } diff --git a/packages/dd-trace/test/.eslintrc.json b/packages/dd-trace/test/.eslintrc.json index ed8a9ff7a87..3a9e197c393 100644 --- a/packages/dd-trace/test/.eslintrc.json +++ b/packages/dd-trace/test/.eslintrc.json @@ -2,8 +2,12 @@ "extends": [ "../../../.eslintrc.json" ], + "parserOptions": { + "ecmaVersion": 2022 + }, "env": { - "mocha": true + "mocha": true, + "es2022": true }, "globals": { "expect": true, diff --git a/packages/dd-trace/test/appsec/activation.spec.js b/packages/dd-trace/test/appsec/activation.spec.js new file mode 100644 index 00000000000..7ebf2ee599f --- /dev/null +++ b/packages/dd-trace/test/appsec/activation.spec.js @@ -0,0 +1,41 @@ +'use strict' + +const Activation = require('../../src/appsec/activation') + +describe('Appsec Activation', () => { + let config + + beforeEach(() => { + config = { + appsec: {} + } + }) + + it('should return ONECLICK with undefined value', () => { + config.appsec.enabled = undefined + const activation = Activation.fromConfig(config) + + expect(activation).to.equal(Activation.ONECLICK) + }) + + it('should return ENABLED with true value', () => { + config.appsec.enabled = true + const activation = Activation.fromConfig(config) + + expect(activation).to.equal(Activation.ENABLED) + }) + + it('should return DISABLED with false value', () => { + config.appsec.enabled = false + const activation = Activation.fromConfig(config) + + expect(activation).to.equal(Activation.DISABLED) + }) + + it('should return DISABLED with invalid value', () => { + config.appsec.enabled = 'invalid' + const activation = Activation.fromConfig(config) + + expect(activation).to.equal(Activation.DISABLED) + }) +}) diff --git a/packages/dd-trace/test/appsec/api_security_rules.json b/packages/dd-trace/test/appsec/api_security_rules.json index fad50fcd358..8202fc82fd0 100644 --- a/packages/dd-trace/test/appsec/api_security_rules.json +++ b/packages/dd-trace/test/appsec/api_security_rules.json @@ -94,7 +94,7 @@ { "inputs": [ { - "address": "http.response.body" + "address": "server.response.body" } ], "output": "_dd.appsec.s.res.body" diff --git a/packages/dd-trace/test/appsec/api_security_sampler.spec.js b/packages/dd-trace/test/appsec/api_security_sampler.spec.js new file mode 100644 index 00000000000..5a69af05a5c --- /dev/null +++ b/packages/dd-trace/test/appsec/api_security_sampler.spec.js @@ -0,0 +1,71 @@ +'use strict' + +const apiSecuritySampler = require('../../src/appsec/api_security_sampler') + +describe('Api Security Sampler', () => { + let config + + beforeEach(() => { + config = { + apiSecurity: { + enabled: true, + requestSampling: 1 + } + } + + sinon.stub(Math, 'random').returns(0.3) + }) + + afterEach(sinon.restore) + + describe('sampleRequest', () => { + it('should sample request if enabled and sampling 1', () => { + apiSecuritySampler.configure(config) + + expect(apiSecuritySampler.sampleRequest({})).to.true + }) + + it('should not sample request if enabled and sampling 0', () => { + config.apiSecurity.requestSampling = 0 + apiSecuritySampler.configure(config) + + expect(apiSecuritySampler.sampleRequest({})).to.false + }) + + it('should sample request if enabled and sampling greater than random', () => { + config.apiSecurity.requestSampling = 0.5 + + apiSecuritySampler.configure(config) + + expect(apiSecuritySampler.sampleRequest({})).to.true + }) + + it('should not sample request if enabled and sampling less than random', () => { + config.apiSecurity.requestSampling = 0.1 + + apiSecuritySampler.configure(config) + + expect(apiSecuritySampler.sampleRequest()).to.false + }) + + it('should not sample request if incorrect config value', () => { + config.apiSecurity.requestSampling = NaN + + apiSecuritySampler.configure(config) + + expect(apiSecuritySampler.sampleRequest()).to.false + }) + + it('should sample request according to the config', () => { + config.apiSecurity.requestSampling = 1 + + apiSecuritySampler.configure(config) + + expect(apiSecuritySampler.sampleRequest({})).to.true + + apiSecuritySampler.setRequestSampling(0) + + expect(apiSecuritySampler.sampleRequest()).to.false + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/attacker-fingerprinting-rules.json b/packages/dd-trace/test/appsec/attacker-fingerprinting-rules.json new file mode 100644 index 00000000000..722f9153ce4 --- /dev/null +++ b/packages/dd-trace/test/appsec/attacker-fingerprinting-rules.json @@ -0,0 +1,204 @@ +{ + "version": "2.2", + "metadata": { + "rules_version": "1.5.0" + }, + "rules": [ + { + "id": "tst-000-001-", + "name": "rule to test fingerprint", + "tags": { + "type": "attack_tool", + "category": "attack_attempt", + "confidence": "1" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.query" + } + ], + "list": [ + "testattack" + ] + }, + "operator": "phrase_match" + } + ], + "transformers": [] + } + ], + "processors": [ + { + "id": "http-endpoint-fingerprint", + "generator": "http_endpoint_fingerprint", + "conditions": [ + { + "operator": "exists", + "parameters": { + "inputs": [ + { + "address": "waf.context.event" + }, + { + "address": "server.business_logic.users.login.failure" + }, + { + "address": "server.business_logic.users.login.success" + } + ] + } + } + ], + "parameters": { + "mappings": [ + { + "method": [ + { + "address": "server.request.method" + } + ], + "uri_raw": [ + { + "address": "server.request.uri.raw" + } + ], + "body": [ + { + "address": "server.request.body" + } + ], + "query": [ + { + "address": "server.request.query" + } + ], + "output": "_dd.appsec.fp.http.endpoint" + } + ] + }, + "evaluate": false, + "output": true + }, + { + "id": "http-header-fingerprint", + "generator": "http_header_fingerprint", + "conditions": [ + { + "operator": "exists", + "parameters": { + "inputs": [ + { + "address": "waf.context.event" + }, + { + "address": "server.business_logic.users.login.failure" + }, + { + "address": "server.business_logic.users.login.success" + } + ] + } + } + ], + "parameters": { + "mappings": [ + { + "headers": [ + { + "address": "server.request.headers.no_cookies" + } + ], + "output": "_dd.appsec.fp.http.header" + } + ] + }, + "evaluate": false, + "output": true + }, + { + "id": "http-network-fingerprint", + "generator": "http_network_fingerprint", + "conditions": [ + { + "operator": "exists", + "parameters": { + "inputs": [ + { + "address": "waf.context.event" + }, + { + "address": "server.business_logic.users.login.failure" + }, + { + "address": "server.business_logic.users.login.success" + } + ] + } + } + ], + "parameters": { + "mappings": [ + { + "headers": [ + { + "address": "server.request.headers.no_cookies" + } + ], + "output": "_dd.appsec.fp.http.network" + } + ] + }, + "evaluate": false, + "output": true + }, + { + "id": "session-fingerprint", + "generator": "session_fingerprint", + "conditions": [ + { + "operator": "exists", + "parameters": { + "inputs": [ + { + "address": "waf.context.event" + }, + { + "address": "server.business_logic.users.login.failure" + }, + { + "address": "server.business_logic.users.login.success" + } + ] + } + } + ], + "parameters": { + "mappings": [ + { + "cookies": [ + { + "address": "server.request.cookies" + } + ], + "session_id": [ + { + "address": "usr.session_id" + } + ], + "user_id": [ + { + "address": "usr.id" + } + ], + "output": "_dd.appsec.fp.session" + } + ] + }, + "evaluate": false, + "output": true + } + ] +} diff --git a/packages/dd-trace/test/appsec/attacker-fingerprinting.express.plugin.spec.js b/packages/dd-trace/test/appsec/attacker-fingerprinting.express.plugin.spec.js new file mode 100644 index 00000000000..bc7c918965c --- /dev/null +++ b/packages/dd-trace/test/appsec/attacker-fingerprinting.express.plugin.spec.js @@ -0,0 +1,79 @@ +'use strict' + +const axios = require('axios') +const { assert } = require('chai') +const path = require('path') + +const agent = require('../plugins/agent') +const appsec = require('../../src/appsec') +const Config = require('../../src/config') + +describe('Attacker fingerprinting', () => { + let port, server + + before(() => { + return agent.load(['express', 'http'], { client: false }) + }) + + before((done) => { + const express = require('../../../../versions/express').get() + const bodyParser = require('../../../../versions/body-parser').get() + + const app = express() + app.use(bodyParser.json()) + + app.post('/', (req, res) => { + res.end('DONE') + }) + + server = app.listen(port, () => { + port = server.address().port + done() + }) + }) + + after(() => { + server.close() + return agent.close({ ritmReset: false }) + }) + + beforeEach(() => { + appsec.enable(new Config( + { + appsec: { + enabled: true, + rules: path.join(__dirname, 'attacker-fingerprinting-rules.json') + } + } + )) + }) + + afterEach(() => { + appsec.disable() + }) + + it('should report http fingerprints', async () => { + await axios.post( + `http://localhost:${port}/?key=testattack`, + { + bodyParam: 'bodyValue' + }, + { + headers: { + headerName: 'headerValue', + 'x-real-ip': '255.255.255.255' + } + } + ) + + await agent.use((traces) => { + const span = traces[0][0] + assert.property(span.meta, '_dd.appsec.fp.http.header') + assert.equal(span.meta['_dd.appsec.fp.http.header'], 'hdr-0110000110-6431a3e6-5-55682ec1') + assert.property(span.meta, '_dd.appsec.fp.http.network') + assert.equal(span.meta['_dd.appsec.fp.http.network'], 'net-1-0100000000') + assert.property(span.meta, '_dd.appsec.fp.http.endpoint') + assert.equal(span.meta['_dd.appsec.fp.http.endpoint'], 'http-post-8a5edab2-2c70e12b-be31090f') + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/attacker-fingerprinting.passport-http.plugin.spec.js b/packages/dd-trace/test/appsec/attacker-fingerprinting.passport-http.plugin.spec.js new file mode 100644 index 00000000000..58b54e2c704 --- /dev/null +++ b/packages/dd-trace/test/appsec/attacker-fingerprinting.passport-http.plugin.spec.js @@ -0,0 +1,107 @@ +'use strict' + +const Axios = require('axios') +const { assert } = require('chai') + +const agent = require('../plugins/agent') +const appsec = require('../../src/appsec') +const Config = require('../../src/config') + +function assertFingerprintInTraces (traces) { + const span = traces[0][0] + assert.property(span.meta, '_dd.appsec.fp.http.header') + assert.equal(span.meta['_dd.appsec.fp.http.header'], 'hdr-0110000110-6431a3e6-5-e58aa9dd') + assert.property(span.meta, '_dd.appsec.fp.http.network') + assert.equal(span.meta['_dd.appsec.fp.http.network'], 'net-0-0000000000') + assert.property(span.meta, '_dd.appsec.fp.http.endpoint') + assert.equal(span.meta['_dd.appsec.fp.http.endpoint'], 'http-post-7e93fba0--') +} + +withVersions('passport-http', 'passport-http', version => { + describe('Attacker fingerprinting', () => { + let port, server, axios + + before(() => { + return agent.load(['express', 'http'], { client: false }) + }) + + before(() => { + appsec.enable(new Config({ + appsec: true + })) + }) + + before((done) => { + const express = require('../../../../versions/express').get() + const bodyParser = require('../../../../versions/body-parser').get() + const passport = require('../../../../versions/passport').get() + const { BasicStrategy } = require(`../../../../versions/passport-http@${version}`).get() + + const app = express() + app.use(bodyParser.json()) + app.use(passport.initialize()) + + passport.use(new BasicStrategy( + function verify (username, password, done) { + if (username === 'success') { + done(null, { + id: 1234, + username + }) + } else { + done(null, false) + } + } + )) + + app.post('/login', passport.authenticate('basic', { session: false }), function (req, res) { + res.end() + }) + + server = app.listen(port, () => { + port = server.address().port + axios = Axios.create({ + baseURL: `http://localhost:${port}` + }) + done() + }) + }) + + after(() => { + server.close() + return agent.close({ ritmReset: false }) + }) + + after(() => { + appsec.disable() + }) + + it('should report http fingerprints on login fail', async () => { + try { + await axios.post( + `http://localhost:${port}/login`, {}, { + auth: { + username: 'fail', + password: '1234' + } + } + ) + } catch (e) {} + + await agent.use(assertFingerprintInTraces) + }) + + it('should report http fingerprints on login successful', async () => { + await axios.post( + `http://localhost:${port}/login`, {}, { + auth: { + username: 'success', + password: '1234' + } + } + ) + + await agent.use(assertFingerprintInTraces) + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/attacker-fingerprinting.passport-local.plugin.spec.js b/packages/dd-trace/test/appsec/attacker-fingerprinting.passport-local.plugin.spec.js new file mode 100644 index 00000000000..b51aa57de9c --- /dev/null +++ b/packages/dd-trace/test/appsec/attacker-fingerprinting.passport-local.plugin.spec.js @@ -0,0 +1,105 @@ +'use strict' + +const Axios = require('axios') +const { assert } = require('chai') + +const agent = require('../plugins/agent') +const appsec = require('../../src/appsec') +const Config = require('../../src/config') + +function assertFingerprintInTraces (traces) { + const span = traces[0][0] + assert.property(span.meta, '_dd.appsec.fp.http.header') + assert.equal(span.meta['_dd.appsec.fp.http.header'], 'hdr-0110000110-6431a3e6-4-c348f529') + assert.property(span.meta, '_dd.appsec.fp.http.network') + assert.equal(span.meta['_dd.appsec.fp.http.network'], 'net-0-0000000000') + assert.property(span.meta, '_dd.appsec.fp.http.endpoint') + assert.equal(span.meta['_dd.appsec.fp.http.endpoint'], 'http-post-7e93fba0--f29f6224') +} + +withVersions('passport-local', 'passport-local', version => { + describe('Attacker fingerprinting', () => { + let port, server, axios + + before(() => { + return agent.load(['express', 'http'], { client: false }) + }) + + before(() => { + appsec.enable(new Config({ + appsec: true + })) + }) + + before((done) => { + const express = require('../../../../versions/express').get() + const bodyParser = require('../../../../versions/body-parser').get() + const passport = require('../../../../versions/passport').get() + const LocalStrategy = require(`../../../../versions/passport-local@${version}`).get() + + const app = express() + app.use(bodyParser.json()) + app.use(passport.initialize()) + + passport.use(new LocalStrategy( + function verify (username, password, done) { + if (username === 'success') { + done(null, { + id: 1234, + username + }) + } else { + done(null, false) + } + } + )) + + app.post('/login', passport.authenticate('local', { session: false }), function (req, res) { + res.end() + }) + + server = app.listen(port, () => { + port = server.address().port + axios = Axios.create({ + baseURL: `http://localhost:${port}` + }) + done() + }) + }) + + after(() => { + server.close() + return agent.close({ ritmReset: false }) + }) + + after(() => { + appsec.disable() + }) + + it('should report http fingerprints on login fail', async () => { + try { + await axios.post( + `http://localhost:${port}/login`, + { + username: 'fail', + password: '1234' + } + ) + } catch (e) {} + + await agent.use(assertFingerprintInTraces) + }) + + it('should report http fingerprints on login successful', async () => { + await axios.post( + `http://localhost:${port}/login`, + { + username: 'success', + password: '1234' + } + ) + + await agent.use(assertFingerprintInTraces) + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/attacker-fingerprinting.spec.js b/packages/dd-trace/test/appsec/attacker-fingerprinting.spec.js new file mode 100644 index 00000000000..013c9cbd3ed --- /dev/null +++ b/packages/dd-trace/test/appsec/attacker-fingerprinting.spec.js @@ -0,0 +1,83 @@ +'use strict' + +const axios = require('axios') +const { assert } = require('chai') +const agent = require('../plugins/agent') +const tracer = require('../../../../index') +const appsec = require('../../src/appsec') +const Config = require('../../src/config') + +describe('Attacker fingerprinting', () => { + describe('SDK', () => { + let http + let controller + let appListener + let port + + function listener (req, res) { + if (controller) { + controller(req, res) + } + } + + before(() => { + appsec.enable(new Config({ + enabled: true + })) + }) + + before(async () => { + await agent.load('http') + http = require('http') + }) + + before(done => { + const server = new http.Server(listener) + appListener = server + .listen(port, 'localhost', () => { + port = appListener.address().port + done() + }) + }) + + after(() => { + appListener.close() + appsec.disable() + return agent.close({ ritmReset: false }) + }) + + it('should provide fingerprinting on successful user login track', (done) => { + controller = (req, res) => { + tracer.appsec.trackUserLoginSuccessEvent({ + id: 'test_user_id' + }, { metakey: 'metaValue' }) + res.end() + } + + agent.use(traces => { + assert.property(traces[0][0].meta, '_dd.appsec.fp.http.header') + assert.equal(traces[0][0].meta['_dd.appsec.fp.http.header'], 'hdr-0110000010-6431a3e6-3-98425651') + assert.property(traces[0][0].meta, '_dd.appsec.fp.http.network') + assert.equal(traces[0][0].meta['_dd.appsec.fp.http.network'], 'net-0-0000000000') + }).then(done).catch(done) + + axios.get(`http://localhost:${port}/`) + }) + + it('should provide fingerprinting on failed user login track', (done) => { + controller = (req, res) => { + tracer.appsec.trackUserLoginFailureEvent('test_user_id', true, { metakey: 'metaValue' }) + res.end() + } + + agent.use(traces => { + assert.property(traces[0][0].meta, '_dd.appsec.fp.http.header') + assert.equal(traces[0][0].meta['_dd.appsec.fp.http.header'], 'hdr-0110000010-6431a3e6-3-98425651') + assert.property(traces[0][0].meta, '_dd.appsec.fp.http.network') + assert.equal(traces[0][0].meta['_dd.appsec.fp.http.network'], 'net-0-0000000000') + }).then(done).catch(done) + + axios.get(`http://localhost:${port}/`) + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/blocking-actions-rules.json b/packages/dd-trace/test/appsec/blocking-actions-rules.json index 76ec63a10d5..f4f443e148b 100644 --- a/packages/dd-trace/test/appsec/blocking-actions-rules.json +++ b/packages/dd-trace/test/appsec/blocking-actions-rules.json @@ -33,7 +33,11 @@ "actions": [ { "id": "block", - "otherParam": "other" + "otherParam": "other", + "parameters": { + "location": "/error", + "status_code": 302 + } }, { "id": "otherId", diff --git a/packages/dd-trace/test/appsec/blocking.spec.js b/packages/dd-trace/test/appsec/blocking.spec.js index ec6c87757a7..04a3c496b46 100644 --- a/packages/dd-trace/test/appsec/blocking.spec.js +++ b/packages/dd-trace/test/appsec/blocking.spec.js @@ -1,7 +1,5 @@ 'use strict' -const { AbortController } = require('node-abort-controller') - describe('blocking', () => { const defaultBlockedTemplate = { html: 'block test', @@ -16,7 +14,7 @@ describe('blocking', () => { } let log - let block, setTemplates, updateBlockingConfiguration + let block, setTemplates let req, res, rootSpan beforeEach(() => { @@ -31,7 +29,6 @@ describe('blocking', () => { block = blocking.block setTemplates = blocking.setTemplates - updateBlockingConfiguration = blocking.updateBlockingConfiguration req = { headers: {} @@ -40,7 +37,9 @@ describe('blocking', () => { res = { setHeader: sinon.stub(), writeHead: sinon.stub(), - end: sinon.stub() + end: sinon.stub(), + getHeaderNames: sinon.stub().returns([]), + removeHeader: sinon.stub() } res.writeHead.returns(res) @@ -70,9 +69,10 @@ describe('blocking', () => { block(req, res, rootSpan) expect(rootSpan.addTags).to.have.been.calledOnceWithExactly({ 'appsec.blocked': 'true' }) - expect(res.setHeader).to.have.been.calledTwice - expect(res.setHeader.firstCall).to.have.been.calledWithExactly('Content-Type', 'text/html; charset=utf-8') - expect(res.setHeader.secondCall).to.have.been.calledWithExactly('Content-Length', 12) + expect(res.writeHead).to.have.been.calledOnceWithExactly(403, { + 'Content-Type': 'text/html; charset=utf-8', + 'Content-Length': 12 + }) expect(res.end).to.have.been.calledOnceWithExactly('htmlBodyéé') }) @@ -81,9 +81,10 @@ describe('blocking', () => { block(req, res, rootSpan) expect(rootSpan.addTags).to.have.been.calledOnceWithExactly({ 'appsec.blocked': 'true' }) - expect(res.setHeader).to.have.been.calledTwice - expect(res.setHeader.firstCall).to.have.been.calledWithExactly('Content-Type', 'application/json') - expect(res.setHeader.secondCall).to.have.been.calledWithExactly('Content-Length', 8) + expect(res.writeHead).to.have.been.calledOnceWithExactly(403, { + 'Content-Type': 'application/json', + 'Content-Length': 8 + }) expect(res.end).to.have.been.calledOnceWithExactly('jsonBody') }) @@ -91,9 +92,10 @@ describe('blocking', () => { block(req, res, rootSpan) expect(rootSpan.addTags).to.have.been.calledOnceWithExactly({ 'appsec.blocked': 'true' }) - expect(res.setHeader).to.have.been.calledTwice - expect(res.setHeader.firstCall).to.have.been.calledWithExactly('Content-Type', 'application/json') - expect(res.setHeader.secondCall).to.have.been.calledWithExactly('Content-Length', 8) + expect(res.writeHead).to.have.been.calledOnceWithExactly(403, { + 'Content-Type': 'application/json', + 'Content-Length': 8 + }) expect(res.end).to.have.been.calledOnceWithExactly('jsonBody') }) @@ -102,12 +104,29 @@ describe('blocking', () => { block(req, res, rootSpan, abortController) expect(rootSpan.addTags).to.have.been.calledOnceWithExactly({ 'appsec.blocked': 'true' }) - expect(res.setHeader).to.have.been.calledTwice - expect(res.setHeader.firstCall).to.have.been.calledWithExactly('Content-Type', 'application/json') - expect(res.setHeader.secondCall).to.have.been.calledWithExactly('Content-Length', 8) + expect(res.writeHead).to.have.been.calledOnceWithExactly(403, { + 'Content-Type': 'application/json', + 'Content-Length': 8 + }) expect(res.end).to.have.been.calledOnceWithExactly('jsonBody') expect(abortController.signal.aborted).to.be.true }) + + it('should remove all headers before sending blocking response', () => { + res.getHeaderNames.returns(['header1', 'header2']) + + block(req, res, rootSpan) + + expect(rootSpan.addTags).to.have.been.calledOnceWithExactly({ 'appsec.blocked': 'true' }) + expect(res.removeHeader).to.have.been.calledTwice + expect(res.removeHeader.firstCall).to.have.been.calledWithExactly('header1') + expect(res.removeHeader.secondCall).to.have.been.calledWithExactly('header2') + expect(res.writeHead).to.have.been.calledOnceWithExactly(403, { + 'Content-Type': 'application/json', + 'Content-Length': 8 + }) + expect(res.end).to.have.been.calledOnceWithExactly('jsonBody') + }) }) describe('block with default templates', () => { @@ -145,131 +164,141 @@ describe('blocking', () => { } it('should block with default html template and custom status', () => { - updateBlockingConfiguration({ - id: 'block', - type: 'block_request', - parameters: { - status_code: 401, - type: 'auto' - } - }) + const actionParameters = { + status_code: 401, + type: 'auto' + } req.headers.accept = 'text/html' setTemplates(config) - block(req, res, rootSpan) + block(req, res, rootSpan, null, actionParameters) + expect(res.writeHead).to.have.been.calledOnceWith(401) expect(res.end).to.have.been.calledOnceWithExactly(defaultBlockedTemplate.html) - expect(res.statusCode).to.be.equal(401) }) it('should block with default json template and custom status ' + 'when type is forced to json and accept is html', () => { - updateBlockingConfiguration({ - id: 'block', - type: 'block_request', - parameters: { - status_code: 401, - type: 'json' - } - }) + const actionParameters = { + status_code: 401, + type: 'json' + } req.headers.accept = 'text/html' setTemplates(config) - block(req, res, rootSpan) + block(req, res, rootSpan, null, actionParameters) + expect(res.writeHead).to.have.been.calledOnceWith(401) expect(res.end).to.have.been.calledOnceWithExactly(defaultBlockedTemplate.json) - expect(res.statusCode).to.be.equal(401) }) it('should block with default html template and custom status ' + 'when type is forced to html and accept is html', () => { - updateBlockingConfiguration({ - id: 'block', - type: 'block_request', - parameters: { - status_code: 401, - type: 'html' - } - }) + const actionParameters = { + status_code: 401, + type: 'html' + } req.headers.accept = 'text/html' setTemplates(config) - block(req, res, rootSpan) + block(req, res, rootSpan, null, actionParameters) + expect(res.writeHead).to.have.been.calledOnceWith(401) expect(res.end).to.have.been.calledOnceWithExactly(defaultBlockedTemplate.html) - expect(res.statusCode).to.be.equal(401) }) it('should block with default json template and custom status', () => { - updateBlockingConfiguration({ - id: 'block', - type: 'block_request', - parameters: { - status_code: 401, - type: 'auto' - } - }) + const actionParameters = { + status_code: 401, + type: 'auto' + } setTemplates(config) - block(req, res, rootSpan) + block(req, res, rootSpan, null, actionParameters) + expect(res.writeHead).to.have.been.calledOnceWith(401) expect(res.end).to.have.been.calledOnceWithExactly(defaultBlockedTemplate.json) - expect(res.statusCode).to.be.equal(401) }) it('should block with default json template and custom status ' + 'when type is forced to json and accept is not defined', () => { - updateBlockingConfiguration({ - id: 'block', - type: 'block_request', - parameters: { - status_code: 401, - type: 'json' - } - }) + const actionParameters = { + status_code: 401, + type: 'json' + } setTemplates(config) - block(req, res, rootSpan) + block(req, res, rootSpan, null, actionParameters) + expect(res.writeHead).to.have.been.calledOnceWith(401) expect(res.end).to.have.been.calledOnceWithExactly(defaultBlockedTemplate.json) - expect(res.statusCode).to.be.equal(401) }) it('should block with default html template and custom status ' + 'when type is forced to html and accept is not defined', () => { - updateBlockingConfiguration({ - id: 'block', - type: 'block_request', - parameters: { - status_code: 401, - type: 'html' - } - }) + const actionParameters = { + status_code: 401, + type: 'html' + } setTemplates(config) - block(req, res, rootSpan) + block(req, res, rootSpan, null, actionParameters) + expect(res.writeHead).to.have.been.calledOnceWith(401) expect(res.end).to.have.been.calledOnceWithExactly(defaultBlockedTemplate.html) - expect(res.statusCode).to.be.equal(401) }) it('should block with custom redirect', () => { - updateBlockingConfiguration({ - id: 'block', - type: 'redirect_request', - parameters: { - status_code: 301, - location: '/you-have-been-blocked' - } - }) + const actionParameters = { + status_code: 301, + location: '/you-have-been-blocked' + } setTemplates(config) - block(req, res, rootSpan) + block(req, res, rootSpan, null, actionParameters) expect(res.writeHead).to.have.been.calledOnceWithExactly(301, { - 'Location': '/you-have-been-blocked' + Location: '/you-have-been-blocked' }) expect(res.end).to.have.been.calledOnce }) }) }) + +describe('waf actions', () => { + const blocking = require('../../src/appsec/blocking') + + it('get block_request as blocking action', () => { + const blockRequestActionParameters = { + status_code: 401, + type: 'html' + } + const actions = { + block_request: blockRequestActionParameters + } + expect(blocking.getBlockingAction(actions)).to.be.deep.equal(blockRequestActionParameters) + }) + + it('get redirect_request as blocking action', () => { + const redirectRequestActionParameters = { + status_code: 301 + } + + const actions = { + redirect_request: redirectRequestActionParameters + } + expect(blocking.getBlockingAction(actions)).to.be.deep.equal(redirectRequestActionParameters) + }) + + it('get undefined when no actions', () => { + const actions = {} + expect(blocking.getBlockingAction(actions)).to.be.undefined + }) + + it('get undefined when generate_stack action', () => { + const actions = { + generate_stack: {} + } + expect(blocking.getBlockingAction(actions)).to.be.undefined + }) +}) diff --git a/packages/dd-trace/test/appsec/express-rules.json b/packages/dd-trace/test/appsec/express-rules.json index 8c5dfeaba31..e8dd910bd02 100644 --- a/packages/dd-trace/test/appsec/express-rules.json +++ b/packages/dd-trace/test/appsec/express-rules.json @@ -28,6 +28,31 @@ ], "transformers": ["lowercase"], "on_match": ["block"] + }, + { + "id": "test-rule-id-2", + "name": "test-rule-name-2", + "tags": { + "type": "security_scanner", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.path_params" + } + ], + "list": [ + "testattack" + ] + }, + "operator": "phrase_match" + } + ], + "transformers": ["lowercase"], + "on_match": ["block"] } ] } diff --git a/packages/dd-trace/test/appsec/graphq.test-utils.js b/packages/dd-trace/test/appsec/graphq.test-utils.js new file mode 100644 index 00000000000..2b07bb19865 --- /dev/null +++ b/packages/dd-trace/test/appsec/graphq.test-utils.js @@ -0,0 +1,228 @@ +'use strict' + +const axios = require('axios') +const path = require('path') +const fs = require('fs') +const { graphqlJson, json } = require('../../src/appsec/blocked_templates') +const agent = require('../plugins/agent') +const appsec = require('../../src/appsec') +const Config = require('../../src/config') + +const schema = ` +directive @case(format: String) on FIELD + +type Book { + title: String, + author: String +} + +type Query { + books(title: String): [Book!]! +}` + +const query = ` +query GetBooks ($title: String) { + books(title: $title) { + title, + author + } +}` + +function makeQuery (derivativeParam) { + return ` + query GetBooks ($title: String) { + books(title: $title) @case(format: "${derivativeParam}") { + title + author + } + }` +} + +const books = [ + { + title: 'Test title', + author: 'Test author' + } +] + +const resolvers = { + Query: { + books: (root, args, context) => { + return books.filter(book => { + return book.title.includes(args.title) + }) + } + } +} + +async function makeGraphqlRequest (port, variables, derivativeParam, extraHeaders = {}) { + const headers = { + 'content-type': 'application/json', + ...extraHeaders + } + + const query = makeQuery(derivativeParam) + return axios.post(`http://localhost:${port}/graphql`, { + operationName: 'GetBooks', + query, + variables + }, { headers, maxRedirects: 0 }) +} + +function graphqlCommonTests (config) { + describe('Block with content', () => { + beforeEach(() => { + appsec.enable(new Config({ appsec: { enabled: true, rules: path.join(__dirname, 'graphql-rules.json') } })) + }) + + afterEach(() => { + appsec.disable() + }) + + it('Should block an attack on variable', async () => { + try { + await makeGraphqlRequest(config.port, { title: 'testattack' }, 'lower') + + return Promise.reject(new Error('Request should not return 200')) + } catch (e) { + expect(e.response.status).to.be.equals(403) + expect(e.response.data).to.be.deep.equal(JSON.parse(graphqlJson)) + } + }) + + it('Should block an attack on directive', async () => { + try { + await makeGraphqlRequest(config.port, { title: 'Test' }, 'testattack') + + return Promise.reject(new Error('Request should not return 200')) + } catch (e) { + expect(e.response.status).to.be.equals(403) + expect(e.response.data).to.be.deep.equal(JSON.parse(graphqlJson)) + } + }) + + it('Should set appsec.blocked on blocked attack', (done) => { + agent.use(payload => { + expect(payload[0][0].meta['appsec.blocked']).to.be.equal('true') + done() + }) + + makeGraphqlRequest(config.port, { title: 'testattack' }, 'lower').then(() => { + done(new Error('block expected')) + }) + }) + + it('Should not block a safe request', async () => { + const response = await makeGraphqlRequest(config.port, { title: 'Test' }, 'lower') + + expect(response.data).to.be.deep.equal({ data: { books } }) + }) + + it('Should block an http attack with graphql response', async () => { + await makeGraphqlRequest(config.port, { title: 'Test' }, 'lower') + + try { + await makeGraphqlRequest(config.port, { title: 'testattack' }, 'lower', { customHeader: 'lower' }) + + return Promise.reject(new Error('Request should not return 200')) + } catch (e) { + expect(e.response.status).to.be.equals(403) + expect(e.response.data).to.be.deep.equal(JSON.parse(graphqlJson)) + } + }) + + it('Should block an http attack with json response when it is not a graphql endpoint', async () => { + await makeGraphqlRequest(config.port, { title: 'Test' }, 'lower') + + try { + await axios.get(`http://localhost:${config.port}/hello`, { headers: { customHeader: 'testattack' } }) + + return Promise.reject(new Error('Request should not return 200')) + } catch (e) { + expect(e.response.status).to.be.equals(403) + expect(e.response.data).to.be.deep.equal(JSON.parse(json)) + } + }) + }) + + describe('Block with custom content', () => { + const blockedTemplateGraphql = path.join(__dirname, 'graphql.block.json') + const customGraphqlJson = fs.readFileSync(blockedTemplateGraphql) + + beforeEach(() => { + appsec.enable(new Config({ + appsec: { + enabled: true, + rules: path.join(__dirname, 'graphql-rules.json'), + blockedTemplateGraphql + } + })) + }) + + afterEach(() => { + appsec.disable() + }) + + it('Should block an attack on variable', async () => { + try { + await makeGraphqlRequest(config.port, { title: 'testattack' }, 'lower') + + return Promise.reject(new Error('Request should not return 200')) + } catch (e) { + expect(e.response.status).to.be.equals(403) + expect(e.response.data).to.be.deep.equal(JSON.parse(customGraphqlJson)) + } + }) + }) + + describe('Block with redirect', () => { + beforeEach(() => { + appsec.enable(new Config({ + appsec: { + enabled: true, + rules: path.join(__dirname, 'graphql-rules-redirect.json') + } + })) + }) + + afterEach(() => { + appsec.disable() + }) + + it('Should block an attack', async () => { + try { + await makeGraphqlRequest(config.port, { title: 'testattack' }, 'lower') + + return Promise.reject(new Error('Request should not return 200')) + } catch (e) { + expect(e.response.status).to.be.equal(301) + expect(e.response.headers.location).to.be.equal('/you-have-been-blocked') + } + }) + + it('Should set appsec.blocked on blocked attack', (done) => { + agent.use(payload => { + expect(payload[0][0].meta['appsec.blocked']).to.be.equal('true') + done() + }) + + makeGraphqlRequest(config.port, { title: 'testattack' }, 'lower').then(() => { + done(new Error('block expected')) + }) + }) + + it('Should not block a safe request', async () => { + const response = await makeGraphqlRequest(config.port, { title: 'Test' }, 'lower') + + expect(response.data).to.be.deep.equal({ data: { books } }) + }) + }) +} + +module.exports = { + books, + schema, + query, + resolvers, + graphqlCommonTests +} diff --git a/packages/dd-trace/test/appsec/graphql-rules-redirect.json b/packages/dd-trace/test/appsec/graphql-rules-redirect.json new file mode 100644 index 00000000000..d6da1ed39b8 --- /dev/null +++ b/packages/dd-trace/test/appsec/graphql-rules-redirect.json @@ -0,0 +1,49 @@ +{ + "version": "2.2", + "metadata": { + "rules_version": "1.5.0" + }, + "rules": [ + { + "id": "test-rule-id-1", + "name": "test-rule-name-1", + "tags": { + "type": "security_scanner", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" + }, + { + "address": "server.request.headers.no_cookies" + } + ], + "list": [ + "testattack" + ] + }, + "operator": "phrase_match" + } + ], + "transformers": ["lowercase"], + "on_match": ["block"] + } + ], + "actions": [ + { + "id": "block", + "type": "redirect_request", + "parameters": { + "status_code": 301, + "location": "/you-have-been-blocked" + } + } + ] +} diff --git a/packages/dd-trace/test/appsec/graphql-rules.json b/packages/dd-trace/test/appsec/graphql-rules.json index e258dda5226..f4cadcdf9bb 100644 --- a/packages/dd-trace/test/appsec/graphql-rules.json +++ b/packages/dd-trace/test/appsec/graphql-rules.json @@ -17,6 +17,12 @@ "inputs": [ { "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" + }, + { + "address": "server.request.headers.no_cookies" } ], "list": [ diff --git a/packages/dd-trace/test/appsec/graphql.apollo-server-express.plugin.spec.js b/packages/dd-trace/test/appsec/graphql.apollo-server-express.plugin.spec.js new file mode 100644 index 00000000000..9ea9638eddf --- /dev/null +++ b/packages/dd-trace/test/appsec/graphql.apollo-server-express.plugin.spec.js @@ -0,0 +1,58 @@ +'use strict' + +const agent = require('../plugins/agent') +const { + schema, + resolvers, + graphqlCommonTests +} = require('./graphq.test-utils') + +withVersions('apollo-server-core', 'express', '>=4', expressVersion => { + withVersions('apollo-server-core', 'apollo-server-express', apolloServerExpressVersion => { + const config = {} + let express, expressServer, ApolloServer, gql + let app, server + + before(() => { + return agent.load(['express', 'graphql', 'apollo-server-core', 'http'], { client: false }) + }) + + before(() => { + const apolloServerExpress = + require(`../../../../versions/apollo-server-express@${apolloServerExpressVersion}`).get() + ApolloServer = apolloServerExpress.ApolloServer + gql = apolloServerExpress.gql + + express = require(`../../../../versions/express@${expressVersion}`).get() + }) + + before(async () => { + app = express() + + const typeDefs = gql(schema) + + server = new ApolloServer({ + typeDefs, + resolvers + }) + + await server.start() + + server.applyMiddleware({ app }) + + return new Promise(resolve => { + expressServer = app.listen({ port: config.port }, () => { + config.port = expressServer.address().port + resolve() + }) + }) + }) + + after(async () => { + await server.stop() + expressServer.close() + }) + + graphqlCommonTests(config) + }) +}) diff --git a/packages/dd-trace/test/appsec/graphql.apollo-server-fastify.plugin.spec.js b/packages/dd-trace/test/appsec/graphql.apollo-server-fastify.plugin.spec.js new file mode 100644 index 00000000000..f8f4721fe3d --- /dev/null +++ b/packages/dd-trace/test/appsec/graphql.apollo-server-fastify.plugin.spec.js @@ -0,0 +1,58 @@ +'use strict' + +const agent = require('../plugins/agent') +const { + schema, + resolvers, + graphqlCommonTests +} = require('./graphq.test-utils') + +withVersions('apollo-server-core', 'fastify', '3', fastifyVersion => { + withVersions('apollo-server-core', 'apollo-server-fastify', apolloServerFastifyVersion => { + const config = {} + let fastify, ApolloServer, gql + let app, server + + before(() => { + return agent.load(['fastify', 'graphql', 'apollo-server-core', 'http'], { client: false }) + }) + + before(() => { + const apolloServerFastify = + require(`../../../../versions/apollo-server-fastify@${apolloServerFastifyVersion}`).get() + ApolloServer = apolloServerFastify.ApolloServer + gql = apolloServerFastify.gql + + fastify = require(`../../../../versions/fastify@${fastifyVersion}`).get() + }) + + before(async () => { + app = fastify() + + const typeDefs = gql(schema) + + server = new ApolloServer({ + typeDefs, + resolvers + }) + + await server.start() + + app.register(server.createHandler()) + + return new Promise(resolve => { + app.listen({ port: config.port }, (data) => { + config.port = app.server.address().port + resolve() + }) + }) + }) + + after(async () => { + await server.stop() + await app.close() + }) + + graphqlCommonTests(config) + }) +}) diff --git a/packages/dd-trace/test/appsec/graphql.apollo-server.plugin.spec.js b/packages/dd-trace/test/appsec/graphql.apollo-server.plugin.spec.js new file mode 100644 index 00000000000..b0f527db141 --- /dev/null +++ b/packages/dd-trace/test/appsec/graphql.apollo-server.plugin.spec.js @@ -0,0 +1,43 @@ +'use strict' + +const path = require('path') +const agent = require('../plugins/agent') +const { + schema, + resolvers, + graphqlCommonTests +} = require('./graphq.test-utils') + +withVersions('apollo-server', '@apollo/server', apolloServerVersion => { + const config = {} + let ApolloServer, startStandaloneServer + let server + + before(() => { + return agent.load(['express', 'graphql', 'apollo-server', 'http'], { client: false }) + }) + + before(() => { + const apolloServerPath = require(`../../../../versions/@apollo/server@${apolloServerVersion}`).getPath() + + ApolloServer = require(apolloServerPath).ApolloServer + startStandaloneServer = require(path.join(apolloServerPath, '..', 'standalone')).startStandaloneServer + }) + + before(async () => { + server = new ApolloServer({ + typeDefs: schema, + resolvers + }) + + const { url } = await startStandaloneServer(server, { listen: { port: 0 } }) + + config.port = new URL(url).port + }) + + after(async () => { + await server.stop() + }) + + graphqlCommonTests(config) +}) diff --git a/packages/dd-trace/test/appsec/graphql.block.json b/packages/dd-trace/test/appsec/graphql.block.json new file mode 100644 index 00000000000..46d71c957d1 --- /dev/null +++ b/packages/dd-trace/test/appsec/graphql.block.json @@ -0,0 +1,7 @@ +{ + "errors": [ + { + "message": "custom blocking message" + } + ] +} diff --git a/packages/dd-trace/test/appsec/graphql.spec.js b/packages/dd-trace/test/appsec/graphql.spec.js new file mode 100644 index 00000000000..c8a7221828a --- /dev/null +++ b/packages/dd-trace/test/appsec/graphql.spec.js @@ -0,0 +1,251 @@ +const proxyquire = require('proxyquire') +const waf = require('../../src/appsec/waf') +const web = require('../../src/plugins/util/web') +const { storage } = require('../../../datadog-core') +const addresses = require('../../src/appsec/addresses') + +const { + startGraphqlResolve, + graphqlMiddlewareChannel, + apolloChannel, + apolloServerCoreChannel +} = require('../../src/appsec/channels') + +describe('GraphQL', () => { + let graphql + let blocking + + beforeEach(() => { + const getBlockingData = sinon.stub() + blocking = { + getBlockingData, + setTemplates: sinon.stub(), + block: sinon.stub() + } + + getBlockingData.returns({ + headers: { 'Content-type': 'application/json' }, + body: '{ "message": "blocked" }', + statusCode: 403 + }) + + graphql = proxyquire('../../src/appsec/graphql', { + './blocking': blocking + }) + }) + + afterEach(() => { + sinon.restore() + }) + + describe('enable', () => { + beforeEach(() => { + }) + + afterEach(() => { + graphql.disable() + sinon.restore() + }) + + it('Should subscribe to all channels', () => { + expect(graphqlMiddlewareChannel.start.hasSubscribers).to.be.false + expect(graphqlMiddlewareChannel.end.hasSubscribers).to.be.false + expect(apolloChannel.start.hasSubscribers).to.be.false + expect(apolloChannel.asyncEnd.hasSubscribers).to.be.false + expect(apolloServerCoreChannel.start.hasSubscribers).to.be.false + expect(apolloServerCoreChannel.asyncEnd.hasSubscribers).to.be.false + expect(startGraphqlResolve.hasSubscribers).to.be.false + + graphql.enable() + + expect(graphqlMiddlewareChannel.start.hasSubscribers).to.be.true + expect(graphqlMiddlewareChannel.end.hasSubscribers).to.be.true + expect(apolloChannel.start.hasSubscribers).to.be.true + expect(apolloChannel.asyncEnd.hasSubscribers).to.be.true + expect(apolloServerCoreChannel.start.hasSubscribers).to.be.true + expect(apolloServerCoreChannel.asyncEnd.hasSubscribers).to.be.true + expect(startGraphqlResolve.hasSubscribers).to.be.true + }) + }) + + describe('disable', () => { + it('Should unsubscribe from all channels', () => { + graphql.enable() + + expect(graphqlMiddlewareChannel.start.hasSubscribers).to.be.true + expect(graphqlMiddlewareChannel.end.hasSubscribers).to.be.true + expect(apolloChannel.start.hasSubscribers).to.be.true + expect(apolloChannel.asyncEnd.hasSubscribers).to.be.true + expect(apolloServerCoreChannel.start.hasSubscribers).to.be.true + expect(apolloServerCoreChannel.asyncEnd.hasSubscribers).to.be.true + expect(startGraphqlResolve.hasSubscribers).to.be.true + + graphql.disable() + + expect(graphqlMiddlewareChannel.start.hasSubscribers).to.be.false + expect(graphqlMiddlewareChannel.end.hasSubscribers).to.be.false + expect(apolloChannel.start.hasSubscribers).to.be.false + expect(apolloChannel.asyncEnd.hasSubscribers).to.be.false + expect(apolloServerCoreChannel.start.hasSubscribers).to.be.false + expect(apolloServerCoreChannel.asyncEnd.hasSubscribers).to.be.false + expect(startGraphqlResolve.hasSubscribers).to.be.false + }) + }) + + describe('onGraphqlStartResolve', () => { + beforeEach(() => { + sinon.stub(waf, 'run').returns(['']) + sinon.stub(storage, 'getStore').returns({ req: {} }) + sinon.stub(web, 'root').returns({}) + graphql.enable() + }) + + afterEach(() => { + sinon.restore() + graphql.disable() + }) + + it('Should not call waf if resolvers is undefined', () => { + const context = { + resolver: undefined + } + + startGraphqlResolve.publish({ context }) + + expect(waf.run).not.to.have.been.called + }) + + it('Should not call waf if resolvers is not an object', () => { + const context = { + resolver: '' + } + + startGraphqlResolve.publish({ context }) + + expect(waf.run).not.to.have.been.called + }) + + it('Should not call waf if req is unavailable', () => { + const context = {} + const resolverInfo = { + user: [{ id: '1234' }] + } + + storage.getStore().req = undefined + + startGraphqlResolve.publish({ context, resolverInfo }) + + expect(waf.run).not.to.have.been.called + }) + + it('Should call waf if resolvers is well formatted', () => { + const context = {} + + const resolverInfo = { + user: [{ id: '1234' }] + } + + startGraphqlResolve.publish({ context, resolverInfo }) + + expect(waf.run).to.have.been.calledOnceWithExactly({ + ephemeral: { + [addresses.HTTP_INCOMING_GRAPHQL_RESOLVER]: resolverInfo + } + }, {}) + }) + }) + + describe('block response', () => { + const req = {} + const res = {} + + beforeEach(() => { + sinon.stub(storage, 'getStore').returns({ req, res }) + + graphql.enable() + graphqlMiddlewareChannel.start.publish({ req, res }) + apolloChannel.start.publish() + }) + + afterEach(() => { + graphqlMiddlewareChannel.end.publish({ req }) + graphql.disable() + sinon.restore() + }) + + it('Should not call abort', () => { + const context = { + abortController: { + abort: sinon.stub() + } + } + + const resolverInfo = { + user: [{ id: '1234' }] + } + + const abortController = {} + + sinon.stub(waf, 'run').returns(['']) + + startGraphqlResolve.publish({ context, resolverInfo }) + + expect(waf.run).to.have.been.calledOnceWithExactly({ + ephemeral: { + [addresses.HTTP_INCOMING_GRAPHQL_RESOLVER]: resolverInfo + } + }, {}) + + expect(context.abortController.abort).not.to.have.been.called + + apolloChannel.asyncEnd.publish({ abortController }) + + expect(blocking.getBlockingData).not.to.have.been.called + }) + + it('Should call abort', () => { + const context = { + abortController: { + abort: sinon.stub() + } + } + + const resolverInfo = { + user: [{ id: '1234' }] + } + + const blockParameters = { + status_code: '401', + type: 'auto', + grpc_status_code: '10' + } + + const rootSpan = { setTag: sinon.stub() } + + const abortController = context.abortController + + sinon.stub(waf, 'run').returns({ + block_request: blockParameters + }) + + sinon.stub(web, 'root').returns(rootSpan) + + startGraphqlResolve.publish({ context, resolverInfo }) + + expect(waf.run).to.have.been.calledOnceWithExactly({ + ephemeral: { + [addresses.HTTP_INCOMING_GRAPHQL_RESOLVER]: resolverInfo + } + }, {}) + + expect(context.abortController.abort).to.have.been.called + + const abortData = {} + apolloChannel.asyncEnd.publish({ abortController, abortData }) + + expect(blocking.getBlockingData).to.have.been.calledOnceWithExactly(req, 'graphql', blockParameters) + + expect(rootSpan.setTag).to.have.been.calledOnceWithExactly('appsec.blocked', 'true') + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/iast/analyzers/code-injection-analyzer.express.plugin.spec.js b/packages/dd-trace/test/appsec/iast/analyzers/code-injection-analyzer.express.plugin.spec.js new file mode 100644 index 00000000000..4177dc78aba --- /dev/null +++ b/packages/dd-trace/test/appsec/iast/analyzers/code-injection-analyzer.express.plugin.spec.js @@ -0,0 +1,60 @@ +'use strict' + +const { prepareTestServerForIastInExpress } = require('../utils') +const axios = require('axios') +const path = require('path') +const os = require('os') +const fs = require('fs') +const { clearCache } = require('../../../../src/appsec/iast/vulnerability-reporter') + +describe('Code injection vulnerability', () => { + withVersions('express', 'express', '>4.18.0', version => { + let i = 0 + let evalFunctionsPath + + beforeEach(() => { + evalFunctionsPath = path.join(os.tmpdir(), `eval-methods-${i++}.js`) + fs.copyFileSync( + path.join(__dirname, 'resources', 'eval-methods.js'), + evalFunctionsPath + ) + }) + + afterEach(() => { + fs.unlinkSync(evalFunctionsPath) + clearCache() + }) + + prepareTestServerForIastInExpress('in express', version, + (testThatRequestHasVulnerability, testThatRequestHasNoVulnerability) => { + testThatRequestHasVulnerability({ + fn: (req, res) => { + // eslint-disable-next-line no-eval + res.send(require(evalFunctionsPath).runEval(req.query.script, 'test-result')) + }, + vulnerability: 'CODE_INJECTION', + makeRequest: (done, config) => { + axios.get(`http://localhost:${config.port}/?script=1%2B2`) + .then(res => { + expect(res.data).to.equal('test-result') + }) + .catch(done) + } + }) + + testThatRequestHasNoVulnerability({ + fn: (req, res) => { + res.send('' + require(evalFunctionsPath).runFakeEval(req.query.script)) + }, + vulnerability: 'CODE_INJECTION', + makeRequest: (done, config) => { + axios.get(`http://localhost:${config.port}/?script=1%2B2`).catch(done) + } + }) + + testThatRequestHasNoVulnerability((req, res) => { + res.send('' + require(evalFunctionsPath).runEval('1 + 2')) + }, 'CODE_INJECTION') + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/iast/analyzers/command-injection-analyzer.kafkajs.plugin.spec.js b/packages/dd-trace/test/appsec/iast/analyzers/command-injection-analyzer.kafkajs.plugin.spec.js new file mode 100644 index 00000000000..ecbb25ac865 --- /dev/null +++ b/packages/dd-trace/test/appsec/iast/analyzers/command-injection-analyzer.kafkajs.plugin.spec.js @@ -0,0 +1,61 @@ +'use strict' + +const { testOutsideRequestHasVulnerability } = require('../utils') + +const topic = 'test-topic' + +describe('command-injection-analyzer with kafkajs', () => { + withVersions('kafkajs', 'kafkajs', version => { + let kafka + let consumer + let producer + + afterEach(async function () { + this.timeout(20000) + await consumer?.disconnect() + await producer?.disconnect() + }) + + describe('outside request', () => { + testOutsideRequestHasVulnerability(async () => { + const lib = require(`../../../../../../versions/kafkajs@${version}`).get() + const Kafka = lib.Kafka + + kafka = new Kafka({ + clientId: 'my-app', + brokers: ['127.0.0.1:9092'] + }) + + consumer = await kafka.consumer({ groupId: 'iast-test' }) + producer = await kafka.producer() + + await consumer.connect() + await consumer.subscribe({ topic, fromBeginning: false }) + + await producer.connect() + + await consumer.run({ + eachMessage: ({ topic, message }) => { + try { + const { execSync } = require('child_process') + + const command = message.value.toString() + execSync(command) + } catch (e) { + // do nothing + } + } + }) + + const sendMessage = async (topic, messages) => { + await producer.send({ + topic, + messages + }) + } + + await sendMessage(topic, [{ key: 'key1', value: 'ls -la' }]) + }, 'COMMAND_INJECTION', 'kafkajs', 20000) + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/iast/analyzers/cookie-analyzer.spec.js b/packages/dd-trace/test/appsec/iast/analyzers/cookie-analyzer.spec.js new file mode 100644 index 00000000000..ba9c114a5c1 --- /dev/null +++ b/packages/dd-trace/test/appsec/iast/analyzers/cookie-analyzer.spec.js @@ -0,0 +1,110 @@ +'use strict' + +const { assert } = require('chai') +const CookieAnalyzer = require('../../../../src/appsec/iast/analyzers/cookie-analyzer') +const Analyzer = require('../../../../src/appsec/iast/analyzers/vulnerability-analyzer') +const Config = require('../../../../src/config') + +describe('CookieAnalyzer', () => { + const VULNERABILITY_TYPE = 'VULN_TYPE' + + it('should extends Analyzer', () => { + assert.isTrue(Analyzer.isPrototypeOf(CookieAnalyzer)) + }) + + describe('_createHashSource', () => { + let cookieAnalyzer + + beforeEach(() => { + cookieAnalyzer = new CookieAnalyzer(VULNERABILITY_TYPE, 'prop') + }) + + describe('default config', () => { + beforeEach(() => { + cookieAnalyzer.onConfigure(new Config({ iast: true })) + }) + + it('should create hash from vulnerability type and not long enough evidence value', () => { + const evidence = { + value: '0'.repeat(31) + } + + const vulnerability = cookieAnalyzer._createVulnerability(VULNERABILITY_TYPE, evidence, null, {}) + + assert.equal(vulnerability.hash, cookieAnalyzer._createHash(`${VULNERABILITY_TYPE}:${evidence.value}`)) + }) + + it('should create different hash from vulnerability type and long evidence value', () => { + const evidence = { + value: '0'.repeat(32) + } + + const vulnerability = cookieAnalyzer._createVulnerability(VULNERABILITY_TYPE, evidence, null, {}) + + assert.equal(vulnerability.hash, cookieAnalyzer._createHash(`FILTERED_${VULNERABILITY_TYPE}`)) + }) + }) + + describe('custom cookieFilterPattern', () => { + beforeEach(() => { + cookieAnalyzer.onConfigure(new Config({ + iast: { + enabled: true, + cookieFilterPattern: '^filtered$' + } + })) + }) + + it('should create hash from vulnerability with the default pattern', () => { + const evidence = { + value: 'notfiltered' + } + + const vulnerability = cookieAnalyzer._createVulnerability(VULNERABILITY_TYPE, evidence, null, {}) + + assert.equal(vulnerability.hash, cookieAnalyzer._createHash(`${VULNERABILITY_TYPE}:${evidence.value}`)) + }) + + it('should create different hash from vulnerability type and long evidence value', () => { + const evidence = { + value: 'filtered' + } + + const vulnerability = cookieAnalyzer._createVulnerability(VULNERABILITY_TYPE, evidence, null, {}) + + assert.equal(vulnerability.hash, cookieAnalyzer._createHash(`FILTERED_${VULNERABILITY_TYPE}`)) + }) + }) + + describe('invalid cookieFilterPattern maintains default behaviour', () => { + beforeEach(() => { + cookieAnalyzer.onConfigure(new Config({ + iast: { + enabled: true, + cookieFilterPattern: '(' + } + })) + }) + + it('should create hash from vulnerability type and not long enough evidence value', () => { + const evidence = { + value: '0'.repeat(31) + } + + const vulnerability = cookieAnalyzer._createVulnerability(VULNERABILITY_TYPE, evidence, null, {}) + + assert.equal(vulnerability.hash, cookieAnalyzer._createHash(`${VULNERABILITY_TYPE}:${evidence.value}`)) + }) + + it('should create different hash from vulnerability type and long evidence value', () => { + const evidence = { + value: '0'.repeat(32) + } + + const vulnerability = cookieAnalyzer._createVulnerability(VULNERABILITY_TYPE, evidence, null, {}) + + assert.equal(vulnerability.hash, cookieAnalyzer._createHash(`FILTERED_${VULNERABILITY_TYPE}`)) + }) + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/iast/analyzers/hardcoded-password-analyzer.spec.js b/packages/dd-trace/test/appsec/iast/analyzers/hardcoded-password-analyzer.spec.js new file mode 100644 index 00000000000..fdc51ce0153 --- /dev/null +++ b/packages/dd-trace/test/appsec/iast/analyzers/hardcoded-password-analyzer.spec.js @@ -0,0 +1,153 @@ +/* eslint-disable max-len */ +'use strict' + +const path = require('path') +const fs = require('fs') +const os = require('os') + +const agent = require('../../../plugins/agent') +const Config = require('../../../../src/config') + +const hardcodedPasswordAnalyzer = require('../../../../src/appsec/iast/analyzers/hardcoded-password-analyzer') +const iast = require('../../../../src/appsec/iast') + +const ruleId = 'hardcoded-password' +const samples = [ + { ident: 'hashpwd', value: 'hpu0-ig=3o5slyr0rkqszidgxw-bc23tivq8e1-qvt.4191vlwm8ddk.ce64m4q0kga' }, + { ident: 'passphrase', value: '8jqqn=-5k9fnu4ukjzg5ar-=yw4r3_no=cn25zpegfh3.ndid5lp04iknqwvz92npuniksena2sz9w1bipd_g_oyx2ij3xi7cyh8=y.pv_f_gnbupl' }, + { ident: 'password', value: '9z57r5sph9zkgnknefwbnrr5ilqiavsylk9b6qt2a9kg-g=ez=_fwat' }, + { ident: 'passw', value: '8=t6jaeqk0=-e9gspu0e6b91mcmayd2bvq37_nn0yyzzh20qn8t29eg-trn6r4w7.s4r59u160jcgenjb-rpn=1ga' }, + { ident: 'secret', value: 'ha=gcu1r-t.ckeirz9y3jf34zvg7o.qsh.' }, + { ident: 'passkey', value: 'k2urtbthyl=fd6z4wl6r26zlldy3.39ymm.p4l5wu_9mzg30sxx6absd696fgpjub4wu8a0bge-5_dp59xf493oayojzftf4iiavndwixmt-fngxn05naek8' }, + { ident: 'pass', value: 'm-ou4zr=vri2yl-0_ardsza7qbenvy3_2b1h8rsq2_n-.utj9nd3xyvqpg4xl37-nl0nkjwam1avbe9zl' }, + { ident: 'pwd', value: 'e8l=47jevvmz5=trchu6.uu3lwn0fb79bpt=fogw36r1srzb1o1w4f-nhihcni=kncnj9ubs0.2w2tey-b=u9f4-s5l_648p67hf4f2bccv6c4lr3mfcj8a77qv38dlzuhpyb=u' }, + { ident: 'pswd', value: '9rh_n.oooxynt-6._a7ho.2=v-gziqf3wz2vudn916jej_6xaq_1vczboi5rmt5_iuvxxf8oq.ghisjxum' }, + { ident: 'passwd', value: 'vtecj=v6b7-qc1m-s6c=zidew-hw-=c-=4d83icy28-guc3g-vvrimsdf=jml.acy=q7sdwaxh_rl-okx1z48pihg=w4=tc4' } +] + +describe('Hardcoded Password Analyzer', () => { + describe('unit test', () => { + const relFile = path.join('path', 'to', 'file.js') + const file = path.join(process.cwd(), relFile) + const line = 42 + const column = 3 + + let report + + beforeEach(() => { + report = sinon.stub(hardcodedPasswordAnalyzer, '_report') + }) + + afterEach(sinon.restore) + + samples.forEach((sample, sampleIndex) => { + // sample values are arrays containing the parts of the original token + it(`should match rule ${ruleId} with #${sampleIndex + 1} value ${sample.ident}...`, () => { + const ident = sample.ident + hardcodedPasswordAnalyzer.analyze({ + file, + literals: [{ + value: sample.value, + locations: [{ + ident, + line, + column + }] + }] + }) + + expect(report).to.have.been.calledOnceWithExactly({ file: relFile, line, column, ident, data: ruleId }) + }) + }) + + it('should not fail with a malformed secret', () => { + expect(() => hardcodedPasswordAnalyzer.analyze(undefined)).not.to.throw() + expect(() => hardcodedPasswordAnalyzer.analyze({ file: undefined })).not.to.throw() + expect(() => hardcodedPasswordAnalyzer.analyze({ file, literals: undefined })).not.to.throw() + expect(() => hardcodedPasswordAnalyzer.analyze({ file, literals: [{ value: undefined }] })).not.to.throw() + expect(() => hardcodedPasswordAnalyzer.analyze({ file, literals: [{ value: 'test' }] })).not.to.throw() + }) + + it('should not report secrets in line 0', () => { + hardcodedPasswordAnalyzer.analyze({ + file, + literals: [{ value: 'test', line: 0 }] + }) + + expect(report).not.to.have.been.called + }) + + it('should use ident as evidence', () => { + report.restore() + + const reportEvidence = sinon.stub(hardcodedPasswordAnalyzer, '_reportEvidence') + + const ident = 'passkey' + hardcodedPasswordAnalyzer.analyze({ + file, + literals: [{ + value: 'this_is_a_password', + locations: [{ + ident, + line, + column + }] + }] + }) + + const evidence = { value: ident } + expect(reportEvidence).to.be.calledOnceWithExactly({ file: relFile, line, column, ident, data: ruleId }, undefined, evidence) + }) + }) + + describe('full feature', () => { + const filename = 'hardcoded-password-functions' + const functionsPath = path.join(os.tmpdir(), filename) + + before(() => { + fs.copyFileSync(path.join(__dirname, 'resources', `${filename}.js`), functionsPath) + }) + + after(() => { + fs.unlinkSync(functionsPath) + }) + + describe('with iast enabled', () => { + beforeEach(() => { + return agent.load(undefined, undefined, { flushInterval: 1 }) + }) + + beforeEach(() => { + const tracer = require('../../../../') + iast.enable(new Config({ + experimental: { + iast: { + enabled: true, + requestSampling: 100 + } + } + }), tracer) + }) + + afterEach(() => { + iast.disable() + }) + + afterEach(() => { + return agent.close({ ritmReset: false }) + }) + + it('should detect vulnerability', (done) => { + agent + .use(traces => { + expect(traces[0][0].meta['_dd.iast.json']).to.include('"HARDCODED_PASSWORD"') + expect(traces[0][0].meta['_dd.iast.json']).to.include('"evidence":{"value":"pswd"}') + }) + .then(done) + .catch(done) + + require(functionsPath) + }) + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/iast/analyzers/hardcoded-secret-analyzer.spec.js b/packages/dd-trace/test/appsec/iast/analyzers/hardcoded-secret-analyzer.spec.js index 28734729558..67d00a8b53a 100644 --- a/packages/dd-trace/test/appsec/iast/analyzers/hardcoded-secret-analyzer.spec.js +++ b/packages/dd-trace/test/appsec/iast/analyzers/hardcoded-secret-analyzer.spec.js @@ -7,6 +7,7 @@ const os = require('os') const agent = require('../../../plugins/agent') const Config = require('../../../../src/config') +const { NameAndValue, ValueOnly } = require('../../../../src/appsec/iast/analyzers/hardcoded-rule-type') const hardcodedSecretAnalyzer = require('../../../../src/appsec/iast/analyzers/hardcoded-secret-analyzer') const { suite } = require('./resources/hardcoded-secrets-suite.json') const iast = require('../../../../src/appsec/iast') @@ -19,6 +20,7 @@ describe('Hardcoded Secret Analyzer', () => { const column = 3 let report + beforeEach(() => { report = sinon.stub(hardcodedSecretAnalyzer, '_report') }) @@ -29,18 +31,23 @@ describe('Hardcoded Secret Analyzer', () => { testCase.samples.forEach((sample, sampleIndex) => { // sample values are arrays containing the parts of the original token it(`should match rule ${testCase.id} with #${sampleIndex + 1} value ${sample[0]}...`, () => { + const value = sample.join('') + const ident = testCase.type === NameAndValue ? value.split(' = ')[0] : undefined + hardcodedSecretAnalyzer.analyze({ file, literals: [{ - value: sample.join(''), + value, locations: [{ line, - column + column, + ident }] }] }) - expect(report).to.have.been.calledOnceWithExactly({ file: relFile, line, column, data: testCase.id }) + expect([NameAndValue, ValueOnly]).to.be.include(testCase.type) + expect(report).to.have.been.calledOnceWithExactly({ file: relFile, line, column, ident, data: testCase.id }) }) }) }) diff --git a/packages/dd-trace/test/appsec/iast/analyzers/header-injection.express.plugin.spec.js b/packages/dd-trace/test/appsec/iast/analyzers/header-injection.express.plugin.spec.js index 3d825997654..bdb9734377a 100644 --- a/packages/dd-trace/test/appsec/iast/analyzers/header-injection.express.plugin.spec.js +++ b/packages/dd-trace/test/appsec/iast/analyzers/header-injection.express.plugin.spec.js @@ -83,6 +83,22 @@ describe('Header injection vulnerability', () => { vulnerability: 'HEADER_INJECTION' }) + testThatRequestHasNoVulnerability({ + testDescription: 'should not have HEADER_INJECTION vulnerability ' + + 'when is the header same header', + fn: (req, res) => { + setHeaderFunction('testheader', req.get('testheader'), res) + }, + vulnerability: 'HEADER_INJECTION', + makeRequest: (done, config) => { + return axios.get(`http://localhost:${config.port}/`, { + headers: { + testheader: 'headerValue' + } + }).catch(done) + } + }) + testThatRequestHasNoVulnerability({ testDescription: 'should not have HEADER_INJECTION vulnerability when the header value is not tainted', fn: (req, res) => { @@ -223,6 +239,37 @@ describe('Header injection vulnerability', () => { }).catch(done) } }) + + testThatRequestHasNoVulnerability({ + fn: (req, res) => { + setHeaderFunction('Access-Control-Allow-Origin', req.headers.origin, res) + setHeaderFunction('Access-Control-Allow-Headers', req.headers['access-control-request-headers'], res) + setHeaderFunction('Access-Control-Allow-Methods', req.headers['access-control-request-methods'], res) + }, + testDescription: 'Should not have vulnerability with CORS headers', + vulnerability: 'HEADER_INJECTION', + occurrencesAndLocation: { + occurrences: 1, + location: { + path: setHeaderFunctionFilename, + line: 4 + } + }, + cb: (headerInjectionVulnerabilities) => { + const evidenceString = headerInjectionVulnerabilities[0].evidence.valueParts + .map(part => part.value).join('') + expect(evidenceString).to.be.equal('custom: value') + }, + makeRequest: (done, config) => { + return axios.options(`http://localhost:${config.port}/`, { + headers: { + origin: 'http://custom-origin', + 'Access-Control-Request-Headers': 'TestHeader', + 'Access-Control-Request-Methods': 'GET' + } + }).catch(done) + } + }) }) }) }) diff --git a/packages/dd-trace/test/appsec/iast/analyzers/insecure-cookie-analyzer.spec.js b/packages/dd-trace/test/appsec/iast/analyzers/insecure-cookie-analyzer.spec.js index fbb3454c27e..af4bd911325 100644 --- a/packages/dd-trace/test/appsec/iast/analyzers/insecure-cookie-analyzer.spec.js +++ b/packages/dd-trace/test/appsec/iast/analyzers/insecure-cookie-analyzer.spec.js @@ -3,12 +3,20 @@ const { prepareTestServerForIast } = require('../utils') const Analyzer = require('../../../../src/appsec/iast/analyzers/vulnerability-analyzer') const { INSECURE_COOKIE } = require('../../../../src/appsec/iast/vulnerabilities') +const insecureCookieAnalyzer = require('../../../../src/appsec/iast/analyzers/insecure-cookie-analyzer') +const CookieAnalyzer = require('../../../../src/appsec/iast/analyzers/cookie-analyzer') + const analyzer = new Analyzer() describe('insecure cookie analyzer', () => { it('Expected vulnerability identifier', () => { expect(INSECURE_COOKIE).to.be.equals('INSECURE_COOKIE') }) + + it('InsecureCookieAnalyzer extends CookieAnalyzer', () => { + expect(CookieAnalyzer.isPrototypeOf(insecureCookieAnalyzer.constructor)).to.be.true + }) + // In these test, even when we are having multiple vulnerabilities, all the vulnerabilities // are in the same cookies method, and it is expected to detect both even when the max operations is 1 const iastConfig = { @@ -43,6 +51,12 @@ describe('insecure cookie analyzer', () => { res.setHeader('set-cookie', ['key=value; HttpOnly', 'key2=value2; Secure']) }, INSECURE_COOKIE, 1) + testThatRequestHasVulnerability((req, res) => { + const cookieNamePrefix = '0'.repeat(32) + res.setHeader('set-cookie', [cookieNamePrefix + 'key1=value', cookieNamePrefix + 'key2=value2']) + }, INSECURE_COOKIE, 1, undefined, undefined, + 'Should be detected as the same INSECURE_COOKIE vulnerability when the cookie name is long') + testThatRequestHasNoVulnerability((req, res) => { res.setHeader('set-cookie', 'key=value; Secure') }, INSECURE_COOKIE) diff --git a/packages/dd-trace/test/appsec/iast/analyzers/no-httponly-cookie-analyzer.spec.js b/packages/dd-trace/test/appsec/iast/analyzers/no-httponly-cookie-analyzer.spec.js index 3c9ed1bae19..743db43097c 100644 --- a/packages/dd-trace/test/appsec/iast/analyzers/no-httponly-cookie-analyzer.spec.js +++ b/packages/dd-trace/test/appsec/iast/analyzers/no-httponly-cookie-analyzer.spec.js @@ -3,6 +3,9 @@ const { prepareTestServerForIast } = require('../utils') const Analyzer = require('../../../../src/appsec/iast/analyzers/vulnerability-analyzer') const { NO_HTTPONLY_COOKIE } = require('../../../../src/appsec/iast/vulnerabilities') +const CookieAnalyzer = require('../../../../src/appsec/iast/analyzers/cookie-analyzer') +const noHttponlyCookieAnalyzer = require('../../../../src/appsec/iast/analyzers/no-httponly-cookie-analyzer') + const analyzer = new Analyzer() describe('no HttpOnly cookie analyzer', () => { @@ -10,6 +13,10 @@ describe('no HttpOnly cookie analyzer', () => { expect(NO_HTTPONLY_COOKIE).to.be.equals('NO_HTTPONLY_COOKIE') }) + it('NoHttponlyCookieAnalyzer extends CookieAnalyzer', () => { + expect(CookieAnalyzer.isPrototypeOf(noHttponlyCookieAnalyzer.constructor)).to.be.true + }) + // In these test, even when we are having multiple vulnerabilities, all the vulnerabilities // are in the same cookies method, and it is expected to detect both even when the max operations is 1 const iastConfig = { @@ -18,6 +25,7 @@ describe('no HttpOnly cookie analyzer', () => { maxConcurrentRequests: 1, maxContextOperations: 1 } + prepareTestServerForIast('no HttpOnly cookie analyzer', (testThatRequestHasVulnerability, testThatRequestHasNoVulnerability) => { testThatRequestHasVulnerability((req, res) => { @@ -47,6 +55,12 @@ describe('no HttpOnly cookie analyzer', () => { res.setHeader('set-cookie', ['key=value; HttpOnly', 'key2=value2; Secure']) }, NO_HTTPONLY_COOKIE, 1) + testThatRequestHasVulnerability((req, res) => { + const cookieNamePrefix = '0'.repeat(32) + res.setHeader('set-cookie', [cookieNamePrefix + 'key1=value', cookieNamePrefix + 'key2=value2']) + }, NO_HTTPONLY_COOKIE, 1, undefined, undefined, + 'Should be detected as the same NO_HTTPONLY_COOKIE vulnerability when the cookie name is long') + testThatRequestHasNoVulnerability((req, res) => { res.setHeader('set-cookie', 'key=value; HttpOnly') }, NO_HTTPONLY_COOKIE) diff --git a/packages/dd-trace/test/appsec/iast/analyzers/no-samesite-cookie-analyzer.spec.js b/packages/dd-trace/test/appsec/iast/analyzers/no-samesite-cookie-analyzer.spec.js index 03be8280795..0d7b1f26dc9 100644 --- a/packages/dd-trace/test/appsec/iast/analyzers/no-samesite-cookie-analyzer.spec.js +++ b/packages/dd-trace/test/appsec/iast/analyzers/no-samesite-cookie-analyzer.spec.js @@ -3,6 +3,9 @@ const { prepareTestServerForIast } = require('../utils') const Analyzer = require('../../../../src/appsec/iast/analyzers/vulnerability-analyzer') const { NO_SAMESITE_COOKIE } = require('../../../../src/appsec/iast/vulnerabilities') +const CookieAnalyzer = require('../../../../src/appsec/iast/analyzers/cookie-analyzer') +const noSamesiteCookieAnalyzer = require('../../../../src/appsec/iast/analyzers/no-samesite-cookie-analyzer') + const analyzer = new Analyzer() describe('no SameSite cookie analyzer', () => { @@ -10,6 +13,10 @@ describe('no SameSite cookie analyzer', () => { expect(NO_SAMESITE_COOKIE).to.be.equals('NO_SAMESITE_COOKIE') }) + it('NoSamesiteCookieAnalyzer extends CookieAnalyzer', () => { + expect(CookieAnalyzer.isPrototypeOf(noSamesiteCookieAnalyzer.constructor)).to.be.true + }) + // In these test, even when we are having multiple vulnerabilities, all the vulnerabilities // are in the same cookies method, and it is expected to detect both even when the max operations is 1 const iastConfig = { @@ -59,6 +66,12 @@ describe('no SameSite cookie analyzer', () => { res.setHeader('set-cookie', 'key=value; SameSite=strict') }, NO_SAMESITE_COOKIE) + testThatRequestHasVulnerability((req, res) => { + const cookieNamePrefix = '0'.repeat(32) + res.setHeader('set-cookie', [cookieNamePrefix + 'key1=value', cookieNamePrefix + 'key2=value2']) + }, NO_SAMESITE_COOKIE, 1, undefined, undefined, + 'Should be detected as the same NO_SAMESITE_COOKIE vulnerability when the cookie name is long') + testThatRequestHasNoVulnerability((req, res) => { res.setHeader('set-cookie', 'key=') }, NO_SAMESITE_COOKIE) diff --git a/packages/dd-trace/test/appsec/iast/analyzers/nosql-injection-mongodb-analyzer.mongoose.plugin.spec.js b/packages/dd-trace/test/appsec/iast/analyzers/nosql-injection-mongodb-analyzer.mongoose.plugin.spec.js index 787f737c156..f09264225a9 100644 --- a/packages/dd-trace/test/appsec/iast/analyzers/nosql-injection-mongodb-analyzer.mongoose.plugin.spec.js +++ b/packages/dd-trace/test/appsec/iast/analyzers/nosql-injection-mongodb-analyzer.mongoose.plugin.spec.js @@ -52,91 +52,173 @@ describe('nosql injection detection in mongodb - whole feature', () => { prepareTestServerForIastInExpress('Test with mongoose', expressVersion, (testThatRequestHasVulnerability, testThatRequestHasNoVulnerability) => { - testThatRequestHasVulnerability({ - fn: async (req, res) => { - Test.find({ - name: req.query.key, - value: [1, 2, - 'value', - false, req.query.key] - }).then(() => { - res.end() - }) - }, - vulnerability: 'NOSQL_MONGODB_INJECTION', - makeRequest: (done, config) => { - axios.get(`http://localhost:${config.port}/?key=value`).catch(done) - } - }) - - testThatRequestHasVulnerability({ - fn: async (req, res) => { - Test.find({ - name: { - child: [req.query.key] - } - }).then(() => { - res.end() - }) - }, - vulnerability: 'NOSQL_MONGODB_INJECTION', - makeRequest: (done, config) => { - axios.get(`http://localhost:${config.port}/?key=value`).catch(done) - } - }) - - testThatRequestHasVulnerability({ - testDescription: 'should have NOSQL_MONGODB_INJECTION vulnerability in correct file and line', - fn: async (req, res) => { - const filter = { - name: { - child: [req.query.key] - } + describe('using promises', () => { + testThatRequestHasVulnerability({ + fn: async (req, res) => { + Test.find({ + name: req.query.key, + value: [1, 2, + 'value', + false, req.query.key] + }).then(() => { + res.end() + }) + }, + vulnerability: 'NOSQL_MONGODB_INJECTION', + makeRequest: (done, config) => { + axios.get(`http://localhost:${config.port}/?key=value`).catch(done) } - require(tmpFilePath)(Test, filter, () => { - res.end() - }) - }, - vulnerability: 'NOSQL_MONGODB_INJECTION', - makeRequest: (done, config) => { - axios.get(`http://localhost:${config.port}/?key=value`).catch(done) - }, - occurrences: { - occurrences: 1, - location: { - path: vulnerableMethodFilename, - line: 4 + }) + + testThatRequestHasVulnerability({ + fn: async (req, res) => { + Test.find({ + name: { + child: [req.query.key] + } + }).then(() => { + res.end() + }) + }, + vulnerability: 'NOSQL_MONGODB_INJECTION', + makeRequest: (done, config) => { + axios.get(`http://localhost:${config.port}/?key=value`).catch(done) } - } - }) + }) - if (semver.satisfies(specificMongooseVersion, '>=6')) { - testThatRequestHasNoVulnerability({ - testDescription: 'should not have NOSQL_MONGODB_INJECTION vulnerability with mongoose.sanitizeFilter', + testThatRequestHasVulnerability({ + testDescription: 'should have NOSQL_MONGODB_INJECTION vulnerability using promise in exec method', fn: async (req, res) => { - const filter = mongoose.sanitizeFilter({ + Test.find({ name: { child: [req.query.key] } + }).exec().then(() => { + res.end() }) - Test.find(filter).then(() => { + }, + vulnerability: 'NOSQL_MONGODB_INJECTION', + makeRequest: (done, config) => { + axios.get(`http://localhost:${config.port}/?key=value`).catch(done) + } + }) + + testThatRequestHasVulnerability({ + testDescription: 'should have NOSQL_MONGODB_INJECTION vulnerability in correct file and line', + fn: async (req, res) => { + const filter = { + name: { + child: [req.query.key] + } + } + require(tmpFilePath)(Test, filter, () => { res.end() }) }, vulnerability: 'NOSQL_MONGODB_INJECTION', makeRequest: (done, config) => { axios.get(`http://localhost:${config.port}/?key=value`).catch(done) + }, + occurrences: { + occurrences: 1, + location: { + path: vulnerableMethodFilename, + line: 4 + } } }) - } - testThatRequestHasNoVulnerability(async (req, res) => { - Test.find({ - name: 'test' - }).then(() => { - res.end() + if (semver.satisfies(specificMongooseVersion, '>=6')) { + testThatRequestHasNoVulnerability({ + testDescription: 'should not have NOSQL_MONGODB_INJECTION vulnerability with mongoose.sanitizeFilter', + fn: async (req, res) => { + const filter = mongoose.sanitizeFilter({ + name: { + child: [req.query.key] + } + }) + Test.find(filter).then(() => { + res.end() + }) + }, + vulnerability: 'NOSQL_MONGODB_INJECTION', + makeRequest: (done, config) => { + axios.get(`http://localhost:${config.port}/?key=value`).catch(done) + } + }) + } + + testThatRequestHasNoVulnerability(async (req, res) => { + Test.find({ + name: 'test' + }).then(() => { + res.end() + }) + }, 'NOSQL_MONGODB_INJECTION') + }) + + if (semver.satisfies(specificMongooseVersion, '<7')) { + describe('using callbacks', () => { + testThatRequestHasNoVulnerability(async (req, res) => { + try { + Test.find({ + name: 'test' + }).exec(() => { + res.end() + }) + } catch (e) { + res.writeHead(500) + res.end() + } + }, 'NOSQL_MONGODB_INJECTION') + + testThatRequestHasVulnerability({ + textDescription: 'should have NOSQL_MONGODB_INJECTION vulnerability using callback in exec', + fn: async (req, res) => { + try { + Test.find({ + name: req.query.key, + value: [1, 2, + 'value', + false, req.query.key] + }).exec(() => { + res.end() + }) + } catch (e) { + res.writeHead(500) + res.end() + } + }, + vulnerability: 'NOSQL_MONGODB_INJECTION', + makeRequest: (done, config) => { + axios.get(`http://localhost:${config.port}/?key=value`).catch(done) + } + }) + + testThatRequestHasVulnerability({ + textDescription: 'should have NOSQL_MONGODB_INJECTION vulnerability using callback in find', + fn: async (req, res) => { + try { + Test.find({ + name: req.query.key, + value: [1, 2, + 'value', + false, req.query.key] + }, () => { + res.end() + }) + } catch (e) { + res.writeHead(500) + res.end() + } + }, + vulnerability: 'NOSQL_MONGODB_INJECTION', + makeRequest: (done, config) => { + axios.get(`http://localhost:${config.port}/?key=value`).catch(done) + } + }) }) - }, 'NOSQL_MONGODB_INJECTION') + } }) }) }) diff --git a/packages/dd-trace/test/appsec/iast/analyzers/nosql-injection-mongodb-analyzer.mquery.plugin.spec.js b/packages/dd-trace/test/appsec/iast/analyzers/nosql-injection-mongodb-analyzer.mquery.plugin.spec.js new file mode 100644 index 00000000000..7cf71f7a86e --- /dev/null +++ b/packages/dd-trace/test/appsec/iast/analyzers/nosql-injection-mongodb-analyzer.mquery.plugin.spec.js @@ -0,0 +1,344 @@ +'use strict' + +const { prepareTestServerForIastInExpress } = require('../utils') +const axios = require('axios') +const agent = require('../../../plugins/agent') +const os = require('os') +const path = require('path') +const semver = require('semver') +const fs = require('fs') + +describe('nosql injection detection with mquery', () => { + withVersions('express', 'express', '>4.18.0', expressVersion => { + withVersions('mongodb', 'mongodb', mongodbVersion => { + const mongodb = require(`../../../../../../versions/mongodb@${mongodbVersion}`) + + const satisfiesNodeVersionForMongo3and4 = + (semver.satisfies(process.version, '<14.20.1') && semver.satisfies(mongodb.version(), '>=3.3 <5')) + const satisfiesNodeVersionForMongo5 = + (semver.satisfies(process.version, '>=14.20.1 <16.20.1') && semver.satisfies(mongodb.version(), '5')) + const satisfiesNodeVersionForMongo6 = + (semver.satisfies(process.version, '>=16.20.1') && semver.satisfies(mongodb.version(), '>=6')) + + if (!satisfiesNodeVersionForMongo3and4 && !satisfiesNodeVersionForMongo5 && !satisfiesNodeVersionForMongo6) return + + const vulnerableMethodFilename = 'mquery-vulnerable-method.js' + let client, testCollection, tmpFilePath, dbName + + before(() => { + return agent.load(['mongodb', 'mquery'], { client: false }, { flushInterval: 1 }) + }) + + before(async () => { + const id = require('../../../../src/id') + dbName = id().toString() + const mongo = require(`../../../../../../versions/mongodb@${mongodbVersion}`).get() + + client = new mongo.MongoClient(`mongodb://localhost:27017/${dbName}`, { + useNewUrlParser: true, + useUnifiedTopology: true + }) + await client.connect() + + testCollection = client.db().collection('Test') + + await testCollection.insertMany([{ id: 1, name: 'value' }, { id: 2, name: 'value2' }]) + + const src = path.join(__dirname, 'resources', vulnerableMethodFilename) + + tmpFilePath = path.join(os.tmpdir(), vulnerableMethodFilename) + try { + fs.unlinkSync(tmpFilePath) + } catch (e) { + // ignore the error + } + fs.copyFileSync(src, tmpFilePath) + }) + + after(async () => { + fs.unlinkSync(tmpFilePath) + + await testCollection.deleteMany({}) + + await client.close() + }) + + withVersions('mquery', 'mquery', mqueryVersion => { + const vulnerableMethodFilename = 'mquery-vulnerable-method.js' + + const mqueryPkg = require(`../../../../../../versions/mquery@${mqueryVersion}`) + + let mquery, collection + + before(() => { + mquery = mqueryPkg.get() + collection = mquery().collection(testCollection) + }) + + prepareTestServerForIastInExpress('Test with mquery', expressVersion, + (testThatRequestHasVulnerability, testThatRequestHasNoVulnerability) => { + testThatRequestHasVulnerability({ + testDescription: 'should have NOSQL_MONGODB_INJECTION vulnerability [find exec]', + occurrences: 1, + fn: async (req, res) => { + try { + const result = await collection + .find({ + name: req.query.key + }) + .exec() + + expect(result).to.not.be.undefined + expect(result.length).to.equal(1) + expect(result[0].id).to.be.equal(1) + } catch (e) { + // do nothing + } + + res.end() + }, + vulnerability: 'NOSQL_MONGODB_INJECTION', + makeRequest: (done, config) => { + axios.get(`http://localhost:${config.port}/?key=value`).catch(done) + } + }) + + testThatRequestHasVulnerability({ + testDescription: 'should have NOSQL_MONGODB_INJECTION vulnerability [find then]', + occurrences: 1, + fn: (req, res) => { + try { + return collection + .find({ + name: req.query.key + }) + .then((result) => { + expect(result).to.not.be.undefined + expect(result.length).to.equal(1) + expect(result[0].id).to.be.equal(1) + + res.end() + }) + } catch (e) { + // do nothing + } + }, + vulnerability: 'NOSQL_MONGODB_INJECTION', + makeRequest: (done, config) => { + axios.get(`http://localhost:${config.port}/?key=value`).catch(done) + } + }) + + testThatRequestHasVulnerability({ + testDescription: 'should have NOSQL_MONGODB_INJECTION vulnerability [await find exec]', + occurrences: 1, + fn: async (req, res) => { + try { + await require(tmpFilePath).vulnerableFindExec(collection, { name: req.query.key }) + } catch (e) { + // do nothing + } + res.end() + }, + vulnerability: 'NOSQL_MONGODB_INJECTION', + makeRequest: (done, config) => { + axios.get(`http://localhost:${config.port}/?key=value`).catch(done) + } + }) + + testThatRequestHasVulnerability({ + testDescription: 'should have 2 NOSQL_MONGODB_INJECTION vulnerability [find where exec]', + fn: async (req, res) => { + try { + await require(tmpFilePath) + .vulnerableFindWhereExec(collection, { name: req.query.key }, { where: req.query.key2 }) + } catch (e) { + // do nothing + } + res.end() + }, + vulnerability: 'NOSQL_MONGODB_INJECTION', + occurrences: 2, + makeRequest: (done, config) => { + axios.get(`http://localhost:${config.port}/?key=value&key2=value2`).catch(done) + } + }) + + testThatRequestHasVulnerability({ + testDescription: 'should have 2 NOSQL_MONGODB_INJECTION vulnerabilities [await find where exec]', + occurrences: 2, + fn: async (req, res) => { + try { + await require(tmpFilePath) + .vulnerableFindWhereExec(collection, { name: req.query.key }, { where: req.query.key2 }) + } catch (e) { + // do nothing + } + res.end() + }, + vulnerability: 'NOSQL_MONGODB_INJECTION', + makeRequest: (done, config) => { + axios.get(`http://localhost:${config.port}/?key=value&key2=value2`).catch(done) + } + }) + + testThatRequestHasVulnerability({ + testDescription: 'should have 1 NOSQL_MONGODB_INJECTION vulnerability [await find where exec]', + occurrences: 1, + fn: async (req, res) => { + try { + await require(tmpFilePath) + .vulnerableFindWhereExec(collection, { name: req.query.key }, { where: 'not_tainted' }) + } catch (e) { + // do nothing + } + res.end() + }, + vulnerability: 'NOSQL_MONGODB_INJECTION', + makeRequest: (done, config) => { + axios.get(`http://localhost:${config.port}/?key=value&key2=value2`).catch(done) + } + }) + + testThatRequestHasVulnerability({ + testDescription: 'should have NOSQL_MONGODB_INJECTION vulnerability [find exec]', + occurrences: 2, + fn: async (req, res) => { + try { + const filter = { name: req.query.key } + const where = { key2: req.query.key2 } + await require(tmpFilePath).vulnerableFindWhere(collection, filter, where) + } catch (e) { + // do nothing + } + res.end() + }, + vulnerability: 'NOSQL_MONGODB_INJECTION', + makeRequest: (done, config) => { + axios.get(`http://localhost:${config.port}/?key=value&key2=value`).catch(done) + } + }) + + testThatRequestHasVulnerability({ + testDescription: 'should have NOSQL_MONGODB_INJECTION vulnerability in correct file and line [find]', + fn: async (req, res) => { + const filter = { + name: req.query.key + } + try { + await require(tmpFilePath).vulnerableFind(collection, filter) + } catch (e) { + // do nothing + } + res.end() + }, + vulnerability: 'NOSQL_MONGODB_INJECTION', + makeRequest: (done, config) => { + axios.get(`http://localhost:${config.port}/?key=value`).catch(done) + }, + occurrences: { + occurrences: 1, + location: { + path: vulnerableMethodFilename, + line: 5 + } + } + }) + + testThatRequestHasVulnerability({ + testDescription: 'should have NOSQL_MONGODB_INJECTION vulnerability in correct file and line [findOne]', + fn: async (req, res) => { + const filter = { + name: req.query.key + } + try { + await require(tmpFilePath).vulnerableFindOne(collection, filter) + } catch (e) { + // do nothing + } + res.end() + }, + vulnerability: 'NOSQL_MONGODB_INJECTION', + makeRequest: (done, config) => { + axios.get(`http://localhost:${config.port}/?key=value`).catch(done) + }, + occurrences: { + occurrences: 1, + location: { + path: vulnerableMethodFilename, + line: 10 + } + } + }) + + // this is a known issue. In this case promise is not resolved and exec method is not called but + // we are reporting the vulnerability + testThatRequestHasVulnerability({ + testDescription: 'should have no NOSQL_MONGODB_INJECTION vulnerability [find without call exec or await]', + fn: (req, res) => { + try { + require(tmpFilePath) + .vulnerableFind(collection, { name: req.query.key }) + } catch (e) { + // do nothing + } + res.end() + }, + vulnerability: 'NOSQL_MONGODB_INJECTION', + makeRequest: (done, config) => { + axios.get(`http://localhost:${config.port}/?key=value&key2=value2`).catch(done) + } + }) + + testThatRequestHasNoVulnerability(async (req, res) => { + try { + await collection + .find({ + name: 'test' + }) + } catch (e) { + // do nothing + } + res.end() + }, 'NOSQL_MONGODB_INJECTION') + + testThatRequestHasNoVulnerability(async (req, res) => { + try { + await collection + .find() + } catch (e) { + // do nothing + } + res.end() + }, 'NOSQL_MONGODB_INJECTION') + }) + + withVersions('express-mongo-sanitize', 'express-mongo-sanitize', expressMongoSanitizeVersion => { + prepareTestServerForIastInExpress('Test with sanitization middleware', expressVersion, (expressApp) => { + const mongoSanitize = + require(`../../../../../../versions/express-mongo-sanitize@${expressMongoSanitizeVersion}`).get() + expressApp.use(mongoSanitize()) + }, (testThatRequestHasVulnerability, testThatRequestHasNoVulnerability) => { + testThatRequestHasNoVulnerability({ + fn: async (req, res) => { + const filter = { + name: req.query.key + } + try { + await require(tmpFilePath).vulnerableFindOne(collection, filter) + } catch (e) { + // do nothing + } + res.end() + }, + vulnerability: 'NOSQL_MONGODB_INJECTION', + makeRequest: (done, config) => { + axios.get(`http://localhost:${config.port}/?key=value`).catch(done) + } + }) + }) + }) + }) + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/iast/analyzers/path-traversal-analyzer.spec.js b/packages/dd-trace/test/appsec/iast/analyzers/path-traversal-analyzer.spec.js index 216d63514cc..6c39799f916 100644 --- a/packages/dd-trace/test/appsec/iast/analyzers/path-traversal-analyzer.spec.js +++ b/packages/dd-trace/test/appsec/iast/analyzers/path-traversal-analyzer.spec.js @@ -45,6 +45,14 @@ const InjectionAnalyzer = proxyquire('../../../../src/appsec/iast/analyzers/inje }) describe('path-traversal-analyzer', () => { + before(() => { + pathTraversalAnalyzer.enable() + }) + + after(() => { + pathTraversalAnalyzer.disable() + }) + it('Analyzer should be subscribed to proper channel', () => { expect(pathTraversalAnalyzer._subscriptions).to.have.lengthOf(1) expect(pathTraversalAnalyzer._subscriptions[0]._channel.name).to.equals('apm:fs:operation:start') @@ -172,6 +180,7 @@ prepareTestServerForIast('integration test', (testThatRequestHasVulnerability, t return fn(callArgs) }, 'PATH_TRAVERSAL', { occurrences: 1 }) }) + describe('no vulnerable', () => { testThatRequestHasNoVulnerability(function () { return fn(args) @@ -221,9 +230,11 @@ prepareTestServerForIast('integration test', (testThatRequestHasVulnerability, t describe('test appendFile', () => { const filename = path.join(os.tmpdir(), 'test-appendfile') + beforeEach(() => { fs.writeFileSync(filename, '') }) + afterEach(() => { fs.unlinkSync(filename) }) @@ -233,9 +244,11 @@ prepareTestServerForIast('integration test', (testThatRequestHasVulnerability, t describe('test chmod', () => { const filename = path.join(os.tmpdir(), 'test-chmod') + beforeEach(() => { fs.writeFileSync(filename, '') }) + afterEach(() => { fs.unlinkSync(filename) }) @@ -245,9 +258,11 @@ prepareTestServerForIast('integration test', (testThatRequestHasVulnerability, t describe('test copyFile', () => { const src = path.join(os.tmpdir(), 'test-copyFile-src') const dest = path.join(os.tmpdir(), 'test-copyFile-dst') + beforeEach(() => { fs.writeFileSync(src, '') }) + afterEach(() => { fs.unlinkSync(src) fs.unlinkSync(dest) @@ -260,9 +275,11 @@ prepareTestServerForIast('integration test', (testThatRequestHasVulnerability, t describe('test cp', () => { const src = path.join(os.tmpdir(), 'test-cp-src') const dest = path.join(os.tmpdir(), 'test-cp-dst') + beforeEach(() => { fs.writeFileSync(src, '') }) + afterEach(() => { fs.unlinkSync(src) fs.unlinkSync(dest) @@ -273,7 +290,7 @@ prepareTestServerForIast('integration test', (testThatRequestHasVulnerability, t } describe('test createReadStream', () => { - runFsMethodTest(`test fs.createReadStream method`, 0, (args) => { + runFsMethodTest('test fs.createReadStream method', 0, (args) => { const rs = fs.createReadStream(...args) rs.close() }, __filename) @@ -281,14 +298,16 @@ prepareTestServerForIast('integration test', (testThatRequestHasVulnerability, t describe('test createWriteStream', () => { const filepath = path.join(os.tmpdir(), 'test-createWriteStream') + beforeEach(() => { fs.writeFileSync(filepath, '') }) + afterEach(() => { fs.unlinkSync(filepath) }) - runFsMethodTest(`test fs.createWriteStream method`, 0, (args) => { + runFsMethodTest('test fs.createWriteStream method', 0, (args) => { const rs = fs.createWriteStream(...args) return new Promise((resolve, reject) => { rs.close((err) => { @@ -305,9 +324,11 @@ prepareTestServerForIast('integration test', (testThatRequestHasVulnerability, t describe('test link', () => { const src = path.join(os.tmpdir(), 'test-link-src') const dest = path.join(os.tmpdir(), 'test-link-dst') + beforeEach(() => { fs.writeFileSync(src, '') }) + afterEach(() => { fs.unlinkSync(src) fs.unlinkSync(dest) @@ -348,9 +369,11 @@ prepareTestServerForIast('integration test', (testThatRequestHasVulnerability, t describe('test opendir', () => { const dirname = path.join(os.tmpdir(), 'test-opendir') + beforeEach(() => { fs.mkdirSync(dirname) }) + afterEach(() => { fs.rmdirSync(dirname) }) @@ -361,9 +384,11 @@ prepareTestServerForIast('integration test', (testThatRequestHasVulnerability, t describe('test readdir', () => { const dirname = path.join(os.tmpdir(), 'test-opendir') + beforeEach(() => { fs.mkdirSync(dirname) }) + afterEach(() => { fs.rmdirSync(dirname) }) @@ -382,6 +407,7 @@ prepareTestServerForIast('integration test', (testThatRequestHasVulnerability, t fs.writeFileSync(src, '') fs.linkSync(src, dest) }) + afterEach(() => { fs.unlinkSync(src) fs.unlinkSync(dest) @@ -393,7 +419,7 @@ prepareTestServerForIast('integration test', (testThatRequestHasVulnerability, t describe('test realpath', () => { runFsMethodTestThreeWay('realpath', 0, null, __filename) - runFsMethodTest(`test fs.realpath.native method`, 0, (args) => { + runFsMethodTest('test fs.realpath.native method', 0, (args) => { fs.realpath.native(...args, () => {}) }, __filename) }) @@ -401,9 +427,11 @@ prepareTestServerForIast('integration test', (testThatRequestHasVulnerability, t describe('test rename', () => { const src = path.join(os.tmpdir(), 'test-rename-src') const dest = path.join(os.tmpdir(), 'test-rename-dst') + beforeEach(() => { fs.writeFileSync(src, '') }) + afterEach(() => { fs.unlinkSync(dest) }) @@ -413,6 +441,7 @@ prepareTestServerForIast('integration test', (testThatRequestHasVulnerability, t describe('test rmdir', () => { const dirname = path.join(os.tmpdir(), 'test-rmdir') + beforeEach(() => { fs.mkdirSync(dirname) }) @@ -422,6 +451,7 @@ prepareTestServerForIast('integration test', (testThatRequestHasVulnerability, t if (fs.rm) { describe('test rm', () => { const filename = path.join(os.tmpdir(), 'test-rmdir') + beforeEach(() => { fs.writeFileSync(filename, '') }) @@ -437,9 +467,11 @@ prepareTestServerForIast('integration test', (testThatRequestHasVulnerability, t describe('test symlink', () => { const src = path.join(os.tmpdir(), 'test-symlink-src') const dest = path.join(os.tmpdir(), 'test-symlink-dst') + beforeEach(() => { fs.writeFileSync(src, '') }) + afterEach(() => { fs.unlinkSync(src) fs.unlinkSync(dest) @@ -450,9 +482,11 @@ prepareTestServerForIast('integration test', (testThatRequestHasVulnerability, t describe('test truncate', () => { const src = path.join(os.tmpdir(), 'test-truncate-src') + beforeEach(() => { fs.writeFileSync(src, 'aaaaaa') }) + afterEach(() => { fs.unlinkSync(src) }) @@ -461,6 +495,7 @@ prepareTestServerForIast('integration test', (testThatRequestHasVulnerability, t describe('test unlink', () => { const src = path.join(os.tmpdir(), 'test-unlink-src') + beforeEach(() => { fs.writeFileSync(src, '') }) @@ -469,16 +504,18 @@ prepareTestServerForIast('integration test', (testThatRequestHasVulnerability, t describe('test unwatchFile', () => { const listener = () => {} + beforeEach(() => { fs.watchFile(__filename, listener) }) - runFsMethodTest(`test fs.watchFile method`, 0, (args) => { + runFsMethodTest('test fs.watchFile method', 0, (args) => { fs.unwatchFile(...args) }, __filename, listener) }) describe('test writeFile', () => { const src = path.join(os.tmpdir(), 'test-writeFile-src') + afterEach(() => { fs.unlinkSync(src) }) @@ -486,7 +523,7 @@ prepareTestServerForIast('integration test', (testThatRequestHasVulnerability, t }) describe('test watch', () => { - runFsMethodTest(`test fs.watch method`, 0, (args) => { + runFsMethodTest('test fs.watch method', 0, (args) => { const watcher = fs.watch(...args, () => {}) watcher.close() }, __filename) @@ -494,10 +531,11 @@ prepareTestServerForIast('integration test', (testThatRequestHasVulnerability, t describe('test watchFile', () => { const listener = () => {} + afterEach(() => { fs.unwatchFile(__filename, listener) }) - runFsMethodTest(`test fs.watchFile method`, 0, (args) => { + runFsMethodTest('test fs.watchFile method', 0, (args) => { fs.watchFile(...args, listener) }, __filename) }) diff --git a/packages/dd-trace/test/appsec/iast/analyzers/resources/eval-methods.js b/packages/dd-trace/test/appsec/iast/analyzers/resources/eval-methods.js new file mode 100644 index 00000000000..d557e475f96 --- /dev/null +++ b/packages/dd-trace/test/appsec/iast/analyzers/resources/eval-methods.js @@ -0,0 +1,19 @@ +'use strict' + +function noop () {} + +const obj = { + eval: noop +} + +module.exports = { + runEval: (code, result) => { + const script = `(${code}, result)` + + // eslint-disable-next-line no-eval + return eval(script) + }, + runFakeEval: (code, result) => { + return obj.eval(`(${code}, result)`) + } +} diff --git a/packages/dd-trace/test/appsec/iast/analyzers/resources/fs-async-way-method.js b/packages/dd-trace/test/appsec/iast/analyzers/resources/fs-async-way-method.js index 16a599b295c..ef0bcc0d608 100644 --- a/packages/dd-trace/test/appsec/iast/analyzers/resources/fs-async-way-method.js +++ b/packages/dd-trace/test/appsec/iast/analyzers/resources/fs-async-way-method.js @@ -4,6 +4,7 @@ const fs = require('fs') module.exports = function (methodName, args, cb) { return new Promise((resolve, reject) => { + // eslint-disable-next-line n/handle-callback-err fs[methodName](...args, (err, res) => { resolve(cb(res)) }) diff --git a/packages/dd-trace/test/appsec/iast/analyzers/resources/hardcoded-password-functions.js b/packages/dd-trace/test/appsec/iast/analyzers/resources/hardcoded-password-functions.js new file mode 100644 index 00000000000..fe5342eb692 --- /dev/null +++ b/packages/dd-trace/test/appsec/iast/analyzers/resources/hardcoded-password-functions.js @@ -0,0 +1,7 @@ +'use strict' + +const pswd = 'A3TMAWZUKIWR6O0OGR7B' + +module.exports = { + pswd +} diff --git a/packages/dd-trace/test/appsec/iast/analyzers/resources/hardcoded-secret-functions.js b/packages/dd-trace/test/appsec/iast/analyzers/resources/hardcoded-secret-functions.js index a9df77503b0..6bee9f252f6 100644 --- a/packages/dd-trace/test/appsec/iast/analyzers/resources/hardcoded-secret-functions.js +++ b/packages/dd-trace/test/appsec/iast/analyzers/resources/hardcoded-secret-functions.js @@ -1,7 +1,7 @@ 'use strict' -const secret = 'A3TMAWZUKIWR6O0OGR7B' +const awsToken = 'A3TMAWZUKIWR6O0OGR7B' module.exports = { - secret + awsToken } diff --git a/packages/dd-trace/test/appsec/iast/analyzers/resources/hardcoded-secrets-suite.json b/packages/dd-trace/test/appsec/iast/analyzers/resources/hardcoded-secrets-suite.json index f57bf70f2e3..ad1f598415e 100644 --- a/packages/dd-trace/test/appsec/iast/analyzers/resources/hardcoded-secrets-suite.json +++ b/packages/dd-trace/test/appsec/iast/analyzers/resources/hardcoded-secrets-suite.json @@ -1,269 +1,740 @@ { "version": "1.0", "suite": [ + { + "id": "adafruit-api-key", + "samples": [["adafruit = rkulm44ib", "_sdozr6w_x2r4hop4wh_fp9"]], + "type": "NameAndValue" + }, + { + "id": "adobe-client-id", + "samples": [["adobe = cf561db3d1e3", "b2fcedf9c44bc82d3e82"]], + "type": "NameAndValue" + }, { "id": "adobe-client-secret", - "samples": [["p8e-EDAAd0", "2489877055E91DD4B9F6c72Fa1"]] + "samples": [["p8e-1eaBe2FC2445D9d9", "6D23F2bdcC7E914c"]], + "type": "ValueOnly" }, { "id": "age-secret-key", - "samples": [["AGE-SECRET", "-KEY-1QQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQ"]] + "samples": [["AGE-SECRET-KEY-1QQQQ", "QQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQ"]], + "type": "ValueOnly" + }, + { + "id": "airtable-api-key", + "samples": [["airtable = dilzcf1tf", "jm63ljkb"]], + "type": "NameAndValue" + }, + { + "id": "algolia-api-key", + "samples": [["algolia = 7d84rzxxtx", "5a8v098k9picto9olgluk6"]], + "type": "NameAndValue" }, { "id": "alibaba-access-key-id", - "samples": [["LTAI3vrhsg", "gqwovgecxxq3og"]] + "samples": [["LTAIm5b470p3y2lpbllg", "g6xn"]], + "type": "ValueOnly" + }, + { + "id": "asana-client-id", + "samples": [["asana = 594627604616", "1371"]], + "type": "NameAndValue" + }, + { + "id": "asana-client-secret", + "samples": [["asana = n9kwp5tu4cor", "l8n8qmxkoysnn3qp04du"]], + "type": "NameAndValue" + }, + { + "id": "atlassian-api-token", + "samples": [["atlassian = rhkw7fc6", "bpyhudx10y8c4pyq"], ["confluence = ivoe399", "3q30cmtgvomi4f3z3"], ["jira = 9gfrgcs4zyux1", "hqzoej6zof1"]], + "type": "NameAndValue" }, { "id": "authress-service-client-access-key", - "samples": [["sc_p0gtuom", "1ln3n7kt.13ew6.acc_ejjq9z2wbdcpn6ly923zma.6h6eh2alxu/z/mfr1a3o3nv_1gd9n6iietsjhxy+b6u/sjqp7og1z1bt19razf_5m2vr8+kdq7n_7zts52b-0vyrr=xngyl+94z5"], ["ext_8t7dcj", "2mexcn9i7e8d1dda5.773t.acc-ccaz13gd7vj3clks8gnm./z81y=0zgfsl/byb2kr4acvuma1y7-4sh1hsgbf8vt4s6vgl3tk//y49eopxlb7gqadj9f+0eyjwmc259q+o47vy/nn71wwt1xc"], ["scauth_p7p", "uuv3lnhtbxv06fqourrlt2.9uc0.acc_0a7m8-l2fahi02kgc4ylhgrl8w62sw.kzm7a4k6/82kwlly6u+_4qr1wwxqnewkw5_yq_lr7a4tu//b1ad"], ["authress_v", "yr5hwjqf4r.kpz2ly.acc-5ire1y8gxxt0ka.1+mcp516l_v_hdif3-8jafw4iktz8zfh5-57tkmqhrigi24dst74s6lvn"]] + "samples": [["sc_m0a0jyw7mkh3119f9", "i31pej.c7tgrq.acc_rnh-fbeyb36ms.ylwoai08zmjtb1mfss9f4tw58e9x5=v558s1a7tl"], ["ext_w161y9qm6mprnnf0", "1s.mcdx1q.acc-hixex9wwd0xagepi5wdlnrl8jm._swdfn=6j263t95lrd5n5_8jc1go4/3+mq1gwzu1inhvc+zryh98t-7-zmbsgzbm-/rdiz37ydfr4mggc3"], ["scauth_ub5olvytiubsv", "k5tfqti9qcj.5nilf.acc_oq5rkesestcsitvte71azts2d9w-qh.jd21c47xr0ob2/=737nkoxgh8_epmuv1le8qw+u6mmmoou78ff+ge=c1_u7f0rl/f15x"], ["authress_uj4lt7b.23m", "e.acc-jgwnulbgqbcof6b.i14j5w9c8wbbqf2s-phem5esw/ej5_qc-9yfz+t2mxuala5nozp52_drxm29szl/g"]], + "type": "ValueOnly" }, { "id": "aws-access-token", - "samples": [["A3TMFP3VCN", "RUS4U3CM2M"], ["AKIAH4YXF2", "KPEXC6X8DO"], ["AGPANBREO7", "N1NQFNIHI8"], ["AIDABWX6AW", "SF70OX56QP"], ["AROA804V27", "K62LFC6Q48"], ["AIPA6R18NP", "89QG9DRUZ8"], ["ANPAIRDAIZ", "M9HLNDBKTZ"], ["ANVA98YIVK", "5WL0L3K0PD"], ["ASIAHM4AY1", "O8FDV5P6PN"]] + "samples": [["A3TU9F74ZN", "63N12KNBYS"], ["AKIAUKU2X3", "KL9BYW3T1K"], ["AGPAA350M3", "RN8HDHX40J"], ["AIDALK90LL", "WTZMIHTHAA"], ["AROAA4X5HR", "DOYVHDMG7N"], ["AIPAHQKLZR", "IGXQIGP9CH"], ["ANPA45VNE2", "6MW0PQMFL3"], ["ANVAAT09LN", "PZTA4HJWE6"], ["ASIAMB1HDY", "N8C120KL7M"]], + "type": "ValueOnly" + }, + { + "id": "beamer-api-token", + "samples": [["beamer = b_rcqx3q8xs", "656ml3iiy7y_yf4f31r9=xboa1-dj=jzblo"]], + "type": "NameAndValue" + }, + { + "id": "bitbucket-client-id", + "samples": [["bitbucket = 5d3syr1k", "es47xqa0qzti1qlu1e9zw97l"]], + "type": "NameAndValue" + }, + { + "id": "bitbucket-client-secret", + "samples": [["bitbucket = fk079is6", "3kt7cr39tim3f9p4j3vtc0fxugthn8uscpah5110i4e3wvyj88xf0_4c"]], + "type": "NameAndValue" + }, + { + "id": "bittrex-access-key", + "samples": [["bittrex = 5m67uuwy4z", "05h8tc5jjuna8tjlqjn2dw"]], + "type": "NameAndValue" }, { "id": "clojars-api-token", - "samples": [["CLOJARS_4e", "lb8x3mghlnxqypcu7ntssngn2o09zir7ogdfrtm4tt1c3nbazzv8amv74v"]] + "samples": [["CLOJARS_bttt4d6r20jd", "gte26ftwmc1fulhwjykv5k8i131k5hfmfref0x0ensw91o8s"]], + "type": "ValueOnly" + }, + { + "id": "codecov-access-token", + "samples": [["codecov = d7m20xhrd0", "46mx7cc3ysrz4omxjenkcv"]], + "type": "NameAndValue" + }, + { + "id": "coinbase-access-token", + "samples": [["coinbase = 9eahkeycd", "lldx8ythl-pdroc_qcode4zj4qu8740crppvl12xwg-dhar29j5bj__"]], + "type": "NameAndValue" + }, + { + "id": "confluent-access-token", + "samples": [["confluent = eubuh3dj", "sek2gota"]], + "type": "NameAndValue" + }, + { + "id": "confluent-secret-key", + "samples": [["confluent = w44rdvlr", "kmwb8sxzm1s82ub1839w9x3sq67p6js94s25np2pvezqq9zb3ctkwzsx"]], + "type": "NameAndValue" + }, + { + "id": "contentful-delivery-api-token", + "samples": [["contentful = a7=ncsi", "8y61iof-gnk737omyi=7qky0akd0z1hce6zx"]], + "type": "NameAndValue" }, { "id": "databricks-api-token", - "samples": [["dapih3d1b8", "5a1g8bga4992c1hg4e9318a2cf"]] + "samples": [["dapifec04e728abhe5c8", "h9bdb728cdg490g7"]], + "type": "ValueOnly" + }, + { + "id": "datadog-access-token", + "samples": [["datadog = nbf9kw4eyj", "yxtledv2w9fz7kqlqfg06h2ljf25dx"]], + "type": "NameAndValue" + }, + { + "id": "defined-networking-api-token", + "samples": [["dnkey = dnkey-umapbe", "pb_ubbx6ze7v-g1_=ns7-c-m20-hyp=8qhcdii8ggtxc=mfsewoz=h9mcz4nbqhkg-f4dx22s"]], + "type": "NameAndValue" }, { "id": "digitalocean-access-token", - "samples": [["doo_v1_1c0", "26765063ee70bb5cfa0b0fecc05cc6c2fa7768d06da21a0cbba3bbac253f5"]] + "samples": [["doo_v1_903e3ec77a0ff", "4870ea4f4ede97a821824a877a86eb92e75963bafafcf931c9f"]], + "type": "ValueOnly" }, { "id": "digitalocean-pat", - "samples": [["dop_v1_58d", "fa4cbf2463d8e04f5a0b8821f698d4d4b138891b6a1004348727e7f12d6ef"]] + "samples": [["dop_v1_039a81dd37345", "b51af07f704ce17d314dc7cc822791a9e0b3d8b9983df73a0a9"]], + "type": "ValueOnly" }, { "id": "digitalocean-refresh-token", - "samples": [["dor_v1_02b", "fbbd80069ecf54b92ba631ea4faf84c710c9f8c437c6766a10c205287b330"]] + "samples": [["dor_v1_1654aa2db8b2e", "59ccbc8f7c35517114e61a1cb95b891967490ea61c34b9e2563"]], + "type": "ValueOnly" + }, + { + "id": "discord-api-token", + "samples": [["discord = 4518f022ba", "ae8de5a302b5ef89df0274c42c8ce062fce7a81f3ce9c80ba0ddc3"]], + "type": "NameAndValue" + }, + { + "id": "discord-client-id", + "samples": [["discord = 3137188207", "53100828"]], + "type": "NameAndValue" + }, + { + "id": "discord-client-secret", + "samples": [["discord = =1nirw33e7", "hxp4-823hnt07kixvlbau="]], + "type": "NameAndValue" }, { "id": "doppler-api-token", - "samples": [["dp.pt.6lel", "5ltfww6ng1lrf4ta4rrwp25ifilvlt63xu7qrl1"]] + "samples": [["dp.pt.203xsacdo3r6yw", "6544agurmkp3aezioemigbbaroi8o"]], + "type": "ValueOnly" + }, + { + "id": "droneci-access-token", + "samples": [["droneci = ewqa64fmd1", "q4fgvfgkfdtsi4ca4aq0dl"]], + "type": "NameAndValue" + }, + { + "id": "dropbox-api-token", + "samples": [["dropbox = jjfs536b9o", "7opcp"]], + "type": "NameAndValue" + }, + { + "id": "dropbox-long-lived-api-token", + "samples": [["dropbox = 7seeh7w1ah", "mAAAAAAAAAA9q332k4xq2s3xbzh84ui0q8878ikb1486r02likp3t6"]], + "type": "NameAndValue" + }, + { + "id": "dropbox-short-lived-api-token", + "samples": [["dropbox = sl.f62p74x", "ox-8ae8o0fx79rymrjbuemrz-qdm9o9ptz49=-89dbjk6hutuukf5jtcgk=xbwslihoj02nuoerddg639w5oe3xjg220_l8uv=cy5k64g3405p__qi0s4dhrxfc=_zud"]], + "type": "NameAndValue" }, { "id": "duffel-api-token", - "samples": [["duffel_tes", "t_yau4wm_3boiq6t8dqmaollcz7t7w9cuv2_=qje05lwv"], ["duffel_liv", "e_ls-f06bglojn-89cso6-6019ts7myhc12enou19cbok"]] + "samples": [["duffel_test_01-tbp06", "3qzlpui1a_pgqovrq766ox_yd1y26tdlvvo"], ["duffel_live_w8yt44pl", "vjimgucgw6sfsbh19t7t7ex7=y8uimbxhzc"]], + "type": "ValueOnly" }, { "id": "dynatrace-api-token", - "samples": [["dt0c01.yxq", "mhvm70vew2cwh90tdltw4.tt1q27qpmi4breb4e9e90vrdzcl4gbebv3xcvr8xxgp31gswsigur538by00wwjl"]] + "samples": [["dt0c01.rlulp60zdipsz", "7vk66pqxmls.t8i25749ngu7tc7n511hn4k00ykhwgfabz68r7hkvtl0bnn4sc1vuljzxscgxwng"]], + "type": "ValueOnly" }, { "id": "easypost-api-token", - "samples": [["EZAK3pyq8j", "xaqltfxlttzm74uu0azb71hbb7e0804ft9ttatj2omgg7klq"]] + "samples": [["EZAKu64f858tqv3stynl", "qztptpzphl94bx8gfzqqtytg2bdzpgd3y5h730"]], + "type": "ValueOnly" + }, + { + "id": "etsy-access-token", + "samples": [["etsy = cmsgjp5hyflrt", "565bstz8u7e"]], + "type": "NameAndValue" + }, + { + "id": "facebook", + "samples": [["facebook = 8842b1d35", "71fa9373d4ccd0f0fe9cd10"]], + "type": "NameAndValue" + }, + { + "id": "fastly-api-token", + "samples": [["fastly = 6nui0w1e96c", "pu-t=7pok-g2snf2hkrsn"]], + "type": "NameAndValue" + }, + { + "id": "finicity-api-token", + "samples": [["finicity = 043a72717", "1f5f2677e15e98fb1ef1ec3"]], + "type": "NameAndValue" + }, + { + "id": "finicity-client-secret", + "samples": [["finicity = 8hxdw6w9b", "jnm65n0aj4d"]], + "type": "NameAndValue" + }, + { + "id": "finnhub-access-token", + "samples": [["finnhub = 7uucb1m2rh", "3awvlv6fam"]], + "type": "NameAndValue" + }, + { + "id": "flickr-access-token", + "samples": [["flickr = 5q0h7ojrbvj", "0ro9h22rf820dnr481tdg"]], + "type": "NameAndValue" }, { "id": "flutterwave-public-key", - "samples": [["FLWPUBK_TE", "ST-e4358f76hf2a8g64169fa1d58322ac53-X"]] + "samples": [["FLWPUBK_TEST-cdh938b", "1633g7e855cg31f99adcdchag-X"]], + "type": "ValueOnly" }, { "id": "frameio-api-token", - "samples": [["fio-u-i0ah", "qyzz2wj04s=ospunes1u96k3l3lv_n105uefmzl__cfgwiv2ikv2=tkxn8n_"]] + "samples": [["fio-u-hmlf19k0uld46k", "w3eq4sd=5gp-r3-nowl_z-yycjd_=b9ih-wqcak8mite24rt3h"]], + "type": "ValueOnly" + }, + { + "id": "freshbooks-access-token", + "samples": [["freshbooks = 5bd34em", "dlpvtjfgloemshoczx4pulzsrdrwm6n7yfbwmeudmxwnp0xbq8ng9y21v"]], + "type": "NameAndValue" }, { "id": "gcp-api-key", - "samples": [["AIzagiRmKa", "zHN2TBlarh7C6DZLio_jJFWoRJLhI"]] + "samples": [["AIzaVUtdf9UOXO8_nGCl", "VCWINKQeRRWvHoaEp8Q"]], + "type": "ValueOnly" }, { "id": "github-app-token", - "samples": [["ghu_vAxxW1", "u41LHiUYdT0w9ayqGE7KpDFebuP52p"], ["ghs_iWPGeF", "Y6iNSLaUQ9bq5USqcBRoMzctOVNyHz"]] + "samples": [["ghu_miOyaaAEHmkgPtiQ", "n7Js3gS1qB6EMMkd8ZQx"], ["ghs_pq6nBjv5H426grlU", "KU5pN7y4VRU268QCLlYG"]], + "type": "ValueOnly" }, { "id": "github-fine-grained-pat", - "samples": [["github_pat", "_sB5usIsunOByK3zWYm5EEnUIgDXH9FZZwLOaERzWj8UVa2S6h8j6S4i5IodPMMkIQC1_gpi7Oakvu_xeMR"]] + "samples": [["github_pat_eHKzWsdJE", "mfQgZUBfG58HD0x1Hh1405CN10cOC8NZv_eRTyWi3BNk2BCHcyfabEEgE3fYHNExjabDNH88l"]], + "type": "ValueOnly" }, { "id": "github-oauth", - "samples": [["gho_0lR6rs", "IsgCJ9yOb4y0SQSQbjGKNBIQZ3Umjm"]] + "samples": [["gho_0pihKqpvB3TBNRmX", "dPiYBhXf1RpYUvp6PsDd"]], + "type": "ValueOnly" }, { "id": "github-pat", - "samples": [["ghp_IpfDBs", "Qe5xeVrVeGfLJk0CLBgILAB9Q4gtXI"]] + "samples": [["ghp_kkUgADKubdIQx1Xj", "fTxa3ebdDz6IOUcEQsqa"]], + "type": "ValueOnly" }, { "id": "gitlab-pat", - "samples": [["glpat-ujLI", "APYyTwkmmEHZwsUL"]] + "samples": [["glpat-5tYoXKy3zxl_ko", "5M7IhT"]], + "type": "ValueOnly" }, { "id": "gitlab-ptt", - "samples": [["glptt-dde6", "edfff934db1535930fe73dd2f41dd1a7d4af"]] + "samples": [["glptt-7b67f9365e7a4a", "720d6006590cbcdaabd9ab8ef2"]], + "type": "ValueOnly" }, { "id": "gitlab-rrt", - "samples": [["GR1348941C", "VV65wWADj8hvaRYrb30"]] + "samples": [["GR1348941FdDTk2KFD5R", "UR14pQqbQ"]], + "type": "ValueOnly" + }, + { + "id": "gitter-access-token", + "samples": [["gitter = dq3u6qahzog", "31wc3e70nd3rqgoth_ankbqbel45h"]], + "type": "NameAndValue" + }, + { + "id": "gocardless-api-token", + "samples": [["gocardless = live_jc", "k80j5eiw8dk6n=nos96jza83lv_rlct=71gvse"]], + "type": "NameAndValue" }, { "id": "grafana-api-key", - "samples": [["eyJrIjoi7p", "eYbtEnd3onRcF9VIU4jW89hU5ygaHl8IoOZsow05IZdnNrt7FCsFPkqf0GyQHDwDpJy1kndiEoOjf4GNEiUF322Dh41h8jxEna8D32WciKBkWH969kMjsqKwQvTqv6ke1J6F6DULL2gs1Rqa45iXXOV6QLwIiqqmjgtML396QAX0Jb59E3vzXMHmy9zuQLnfL4JUhePA5lHX1fwzPxEp40OWZGYLEPqEX1oPRlbkou4wFtvSNAfcH7kVpw4bFD6q4CA90ghoVdBJ6cRpVF8EtO7ZQTjfqAekyN1Dtwbqc0Gn6aYNfPG9F26qFlsh7B7V9aVskO6inRl2HTc04WAb4ef=="]] + "samples": [["eyJrIjoiipPw0rBskH3O", "sWJsrsvBHOjzy8ijTtewU0FhWFdUr5C75nIp4qHqaxVTGZ81IbY1mH41QmVmPaTLMl3K03pnlxVgevb7HlMPgKUytQYRUfb9eJHefyBPvuPyvXXeLYfnOEpBQFakfXTGWWCUo6bSk7ew5gCW23G59yvZeFNaXbuXbIcZUC0O7MCRk2GuCErlRXGhkm0jsA5JNGUwDnB0D18xvezl8dYTu"]], + "type": "ValueOnly" }, { "id": "grafana-cloud-api-token", - "samples": [["glc_ID6ub4", "wphFroRSiWzRDkxI4omBhbf6abaQ0gvjK/Wq8wOuuJYJtqOnLALSIaXVJmrPNAeCnPGENHLIZ3TYDWNvNzE0UcawoBgWE+gtqVhdkJaE0D4RH/RyY8Ca7qtK1VbLm0DbWltt1wKDZA/RkJppwu+I8GxAoXoOuOeA/c6hRj9nx+B7qblrsyb417fQztEeI8VKh/SIOHXMwXW+F2VNx8i7zmKD9qukffe1o2arIdckaPHelgK9X3sMaZX71PKkYztR4hVOcpFTf9hkerdcfxMoySbOfe24T4Mni2mEK8guxZsc4zcKxtEnmEelFHv9zMAnx5CoFZSjQe2Zm+VrBrHmzkAdYA+DMj6lYEr/x0FXbhWIOE/HxD7HEa0M4kPEb"]] + "samples": [["glc_EC60tMQh8R5EejUs", "ZC2jbaTxGyOmQnnZEAQvI0UpyVBh3bWDpx+R5lrquJt79qRVqf/e"]], + "type": "ValueOnly" }, { "id": "grafana-service-account-token", - "samples": [["glsa_riQAg", "SW4MluKRO2OSA1RImxf8PeFLjA6_D209Be6d"]] + "samples": [["glsa_iktMvFYw4qaL6dn", "VfeUlA9FYqHEq2Wa2_BEcf361F"]], + "type": "ValueOnly" }, { "id": "hashicorp-tf-api-token", - "samples": [["0slit8am7c", "u8hc.atlasv1.3bl5k2yx5=exmkjvb7a9kg21ud6y-oj-v5hbgagpd9ukqo0d1tvdigtwlwkg8=zq2gbu"]] + "samples": [["ibr2z2zl9sfyuu.atlas", "v1.nuya19ujvq8kj5502mc3c88pk=vo-2ctz9hghwvb14-o6vrth6qot5rryj32ngm6o"]], + "type": "ValueOnly" + }, + { + "id": "heroku-api-key", + "samples": [["heroku = 1b773c9b-f6", "54-f640-f42c-d94fadce53b8"]], + "type": "NameAndValue" + }, + { + "id": "hubspot-api-key", + "samples": [["hubspot = 8AFADC7D-3", "7D4-8827-F2A9-9152801C72B5"]], + "type": "NameAndValue" + }, + { + "id": "intercom-api-key", + "samples": [["intercom = kpnkpep4e", "m9koh36vb9csc4snn1e1pehiw8bw14vi1t3q8g6m==f5ksiy79c"]], + "type": "NameAndValue" + }, + { + "id": "jfrog-api-key", + "samples": [["jfrog = jd0h0b2kzj4j", "m42ubcqg521q9n8dxpz20f8w2ty3nsvyd5qoyxv2q5sqljhmdyxkucth5g1rn"], ["artifactory = 3ngdnd", "6to936l4xskslm0y5ldamits5dp3ceji6eptjxqy0yun5jfb6fr59m0ya6e0qr2sibw"], ["bintray = 6goi4pkyyf", "ww7o76xnjtn1tnt0hi5be4x8ez1jbtcc94kr3cu5552c4exazeib2wb299jhfnm"], ["xray = rk2ydb6fy9gjc", "ueac0u6kajn3rs43m0rscvtm87xhdn3q10p4ifp3qpqbp4vsclljy098uyd2"]], + "type": "NameAndValue" }, { "id": "jwt", - "samples": [["eymxaQKYwN", "DrjyU5BZi.eyztNWMW-IgrhIqwlbGiIAOxi90ui.ieoLMDPoDDYrTeUgI9"], ["eynTAssTgb", "kP5rLnybcH5X.ey/LdxrTJQcB4suJzP7r4wOdYHGQ."]] + "samples": [["eyLvLNDjVbdq6GITGdlf", ".eyTa3EXJ/12TJEh-E9h6Jwr.EWlGgRbms_oIRyV5=="], ["ey5uFXZGgWzkNUbgGhsq", ".eys0PBLqMvfnpcD19RjpyRLKo."]], + "type": "ValueOnly" + }, + { + "id": "kraken-access-token", + "samples": [["kraken = 1izs=c4d-wh", "l1czfu=enff4bv4a3slwby=4i6wu57re4f+a1o6d4at4wsx_gbbz1-g04lrm_qmt2b-=w"]], + "type": "NameAndValue" + }, + { + "id": "kucoin-access-token", + "samples": [["kucoin = f04d6accd6d", "4f666069ed081"]], + "type": "NameAndValue" + }, + { + "id": "launchdarkly-access-token", + "samples": [["launchdarkly = n0-bu", "r0np1z-theuwd0dlbv19yi=lu=rgcrnge1l"]], + "type": "NameAndValue" }, { "id": "linear-api-key", - "samples": [["lin_api_yy", "b9aiejfn7batneic9ky9nejcqc62s4vh589p4c"]] + "samples": [["lin_api_elrru4wvvxis", "k67tihavgeg6dpn56v5zv4ki1d1i"]], + "type": "ValueOnly" + }, + { + "id": "linkedin-client-secret", + "samples": [["linkedin = k8cf36v41", "xlio4lf"], ["linked-in = e5i0c50h", "3p01mqnn"]], + "type": "NameAndValue" + }, + { + "id": "lob-pub-api-key", + "samples": [["lob = live_pub_ab12b", "54b274d5dcd896a7dbf8a17b0d"], ["lob = test_pub_777d4", "5ac342ab9bd33113f569c3bd69"]], + "type": "NameAndValue" + }, + { + "id": "mailchimp-api-key", + "samples": [["mailchimp = 45f8082c", "841c4bd98ed285f2cae2b987-us20"]], + "type": "NameAndValue" + }, + { + "id": "mailgun-private-api-token", + "samples": [["mailgun = key-0f04d7", "5ae2882b884c921c9f65f49b95"]], + "type": "NameAndValue" + }, + { + "id": "mailgun-pub-key", + "samples": [["mailgun = pubkey-2d5", "6e51214c0448f4079ece0a711bbd3"]], + "type": "NameAndValue" + }, + { + "id": "mailgun-signing-key", + "samples": [["mailgun = ea7dbgh1aa", "158dbhff3c5gg9f50e90gh-dg3ac1ah-ec43c3ha"]], + "type": "NameAndValue" + }, + { + "id": "mapbox-api-token", + "samples": [["mapbox = pk.1urq7x66", "m7gtvxfhz3rng2wzlehjxx247zv3c1p8figdi6xrtpo47yu871e2.wjyemv5ble7nsi8wtrpr00"]], + "type": "NameAndValue" + }, + { + "id": "mattermost-access-token", + "samples": [["mattermost = i76jex6", "kxwss8r85k1pilqropw"]], + "type": "NameAndValue" + }, + { + "id": "messagebird-api-token", + "samples": [["messagebird = okylz6", "bphqm1op3k0az9euwe0"], ["message-bird = r52pr", "3rkra8l2w6b1p89a2sf1"], ["message_bird = zxh4y", "tuoofx18skd83lohdcc3"]], + "type": "NameAndValue" + }, + { + "id": "netlify-access-token", + "samples": [["netlify = h3l1py0de6", "dynp3ph3p-sr19716nhxwfbto6e14a"]], + "type": "NameAndValue" + }, + { + "id": "new-relic-browser-api-token", + "samples": [["new-relic = NRJS-a08", "b059b217cfb3bd8a"], ["newrelic = NRJS-5898", "011908c4832b5c3"], ["new_relic = NRJS-43d", "fe97fb2f8f1cb095"]], + "type": "NameAndValue" + }, + { + "id": "new-relic-user-api-id", + "samples": [["new-relic = tw5ffygb", "23vb83k7hkz79jtwmencixlvqwa1qe0os0yr53y0scidmqejdj2bkd3f"], ["newrelic = aum2sqcul", "pjuc0tpxtgpiiar0yhlzdp6t85jm7avftab5yatjwxuxzk1u4b4j14i"], ["new_relic = mad4jdzt", "ujzi8vcxxcdp5i3yeyr6gmoy2bsy7z9t8adqutvj9gvy12eehpzvv3nh"]], + "type": "NameAndValue" + }, + { + "id": "new-relic-user-api-key", + "samples": [["new-relic = NRAK-4oq", "alr7x36lx3z0bn4y2l6si7v4"], ["newrelic = NRAK-6jr8", "n6wbj6th7zmzjrh2w204di1"], ["new_relic = NRAK-5hd", "zw50tomk6cy2rlr5wt39pobo"]], + "type": "NameAndValue" }, { "id": "npm-access-token", - "samples": [["npm_kij9i4", "2j7tqsn3yk7081bmn4dp2yof3i7dn1"]] + "samples": [["npm_xq2vpnik4pntxw9e", "uoer0y92s7pvz1ep2asx"]], + "type": "ValueOnly" + }, + { + "id": "nytimes-access-token", + "samples": [["nytimes = j_0xmhk4vt", "pdlwee00yqyh88wjlqrr01"], ["new-york-times, = 0g", "09xad1b4xelx4naq3olkcd4bc97me5"], ["newyorktimes = yxpri", "dzv1wpk4lztcqa=ngpexev=g720"]], + "type": "NameAndValue" + }, + { + "id": "okta-access-token", + "samples": [["okta = qz-p95nd5x0z5", "a3qq48d8nsil7du1o672snyzegyz6"]], + "type": "NameAndValue" }, { "id": "openai-api-key", - "samples": [["sk-jboPY9q", "evozlPdR3rj52T3BlbkFJ3Xbh0WRnIqB5lVRSgnwf"]] + "samples": [["sk-uPehKekEk6osgmQOU", "Z8CT3BlbkFJeE6NKer6VNguUoUcwjLZ"]], + "type": "ValueOnly" + }, + { + "id": "plaid-api-token", + "samples": [["plaid = access-sandb", "ox-e877ac77-2d44-3fb1-d0cc-6ae47599c195"], ["plaid = access-devel", "opment-8062470b-29c1-b43c-bb00-2f38483ae631"], ["plaid = access-produ", "ction-ac5c0684-62d5-589e-0904-777feebfa890"]], + "type": "NameAndValue" + }, + { + "id": "plaid-client-id", + "samples": [["plaid = uzp2e3s0d3u6", "dg2dbz3nq137"]], + "type": "NameAndValue" + }, + { + "id": "plaid-secret-key", + "samples": [["plaid = xe1lwjpb36s4", "c9zgw0bfujp5ae74qc"]], + "type": "NameAndValue" }, { "id": "planetscale-api-token", - "samples": [["pscale_tkn", "_jk5fgt.9dcezw=7u._9z_h.5u0y1qj2iohp8=r3rne.rp7"]] + "samples": [["pscale_tkn_3sxfp.x=9", "gy-3_0cnjxug0tgwzv0og.tbo07v7"]], + "type": "ValueOnly" }, { "id": "planetscale-oauth-token", - "samples": [["pscale_oau", "th_zq0w.cysbk.zt8ms7n=kov.xvisl8ivp02_k413f43ox=6jwa0fdtwxeqca4yny"]] + "samples": [["pscale_oauth_.o1uu41", ".470.4=nbc_qw4s-w3he=pl7uukyfsg_p8z462giw1nx8flhurf4nz5y"]], + "type": "ValueOnly" }, { "id": "planetscale-password", - "samples": [["pscale_pw_", "3hqt3dfn0-r81kjiv_iw75__ktl.hg-w5zgxzze2x9=n.ctutuwkgv--j"]] + "samples": [["pscale_pw_n.hmxwr4bg", "viokget..9vpme_o_-yp1oz33u3nau7"]], + "type": "ValueOnly" }, { "id": "postman-api-token", - "samples": [["PMAK-53968", "a5c3624c04598edb18b-7754920c28f71499915a98a0e65b06e484"]] + "samples": [["PMAK-ac12ed38aa4c5b7", "392da490d-b5f036bae6245e1647b3a6fed0b336a6e0"]], + "type": "ValueOnly" }, { "id": "prefect-api-token", - "samples": [["pnu_harwid", "sxsm6rjkjd8hbh73kog2nz1vw73v17"]] + "samples": [["pnu_sitojveqkk6jkzka", "djwaknbgiyspq12okb9s"]], + "type": "ValueOnly" }, { "id": "private-key", - "samples": [["-----BEGIN", "W1K3XHIP8-AH8V9-GCH65LHJKRHK2857QYF05EWGYX23O3JUZ7J5JT5LUASG-9YAGPRIVATE KEY-----KEY----"]] + "samples": [["-----BEGIN_TRNTSFYV3", "CFNDLVN5JC287NAJXY_J1865ZXQZ5VXVU7LPRIVATE KEY------- KEY----"]], + "type": "ValueOnly" }, { "id": "pulumi-api-token", - "samples": [["pul-7190e2", "afe16c4e171c05b11568859442f3b5b64e"]] + "samples": [["pul-17a8fa9fb67e6914", "08cdfc3b5a2219e567303b9f"]], + "type": "ValueOnly" }, { "id": "pypi-upload-token", - "samples": [["pypi-AgEIc", "HlwaS5vcmcB4VlkGTKEYcWMbZBOoZmTZoyOzGMvyrgGcDlGIOxmk5ROMJIst1IP"]] + "samples": [["pypi-AgEIcHlwaS5vcmc", "3H4oKqyTw3I2UxUP6OWfe1IiIf6WDBU9zJgCudhVwNtHH4dGTM"]], + "type": "ValueOnly" + }, + { + "id": "rapidapi-access-token", + "samples": [["rapidapi = _yysvyxc0", "8a7fkt3bgwsdlw8vnxyhg58m2zj4vapzg3usd4xhh"]], + "type": "NameAndValue" }, { "id": "readme-api-token", - "samples": [["rdme_8witn", "1adpkz8i1n37dowa0smubx8sa8tjorpmt2r1sfvuj7o15200wyie59ldbvomgy3j7"]] + "samples": [["rdme_5v4va9mry0pj0tf", "bgjksqza0jeaoja3b4lqn8trcn84vl89u84k3ayjfvdm687ix6oood2"]], + "type": "ValueOnly" }, { "id": "rubygems-api-token", - "samples": [["rubygems_0", "e871386ebed3dd645df1310585e03bb700076d4db4d779c"]] + "samples": [["rubygems_36dc1700ec1", "105aacae67d3b05c58331c64a626fd5a62178"]], + "type": "ValueOnly" }, { "id": "scalingo-api-token", - "samples": [["tk-us-ydgc", "NbZ2IiL2NzT_jxoZX-3IMgy4oK37KjaIafoyNLrONYMR"]] + "samples": [["tk-us-xowZbnzohTVQTF", "UJ-sQd_oPr-KklZM76qYJwoQiIrYR0cPFn"]], + "type": "ValueOnly" + }, + { + "id": "sendbird-access-id", + "samples": [["sendbird = ac6b4476-", "60e3-2cac-f46a-8df83b591ed5"]], + "type": "NameAndValue" + }, + { + "id": "sendbird-access-token", + "samples": [["sendbird = fa244a58d", "a972bb203e91cd782a013a1a2286bad"]], + "type": "NameAndValue" }, { "id": "sendgrid-api-token", - "samples": [["SG.j0t3ooq", "hwruz_y.lzltpg.cf8hh2j=6t1=.9ohi5yorz231-lo-ufl6kxn9gg1-zya"]] + "samples": [["SG.iuig8=fxc7sbiqu57", "q==pnmfe5hv7jnd-8xc24p8q4f71tbogbhjwl_8icbd2y37uo"]], + "type": "ValueOnly" }, { "id": "sendinblue-api-token", - "samples": [["xkeysib-a3", "b81b566200b2d089ef0b5eaea3806a564fd12001a5c8649eb5f428867cab80-o0l46hpq577cafes"]] + "samples": [["xkeysib-f71abaa23d11", "357f6ab2485745ec6affe04259cd26f4530f8d960701b30f5853-ffhcjx375a6ubebk"]], + "type": "ValueOnly" + }, + { + "id": "sentry-access-token", + "samples": [["sentry = b4d3a9c926f", "fbce8d6dc0d0e8d6b17ff693c80e11a9b4db3a48c3d08d947cb24"]], + "type": "NameAndValue" }, { "id": "shippo-api-token", - "samples": [["shippo_liv", "e_44fb5ad5113f0f3396c17aa4d1537745440db445"], ["shippo_tes", "t_b067b13f1af9b23347ee48367558f2fb1f53e6c9"]] + "samples": [["shippo_live_e2b281ad", "1c3613a5631de33c72718a1fb64c2fd5"], ["shippo_test_772ec007", "7d23b44994f32f04d5598d105a16b75c"]], + "type": "ValueOnly" }, { "id": "shopify-access-token", - "samples": [["shpat_dBF5", "0bc0c3c02b8E5B54dEBDc9A7A5BF"]] + "samples": [["shpat_c7A03E319315F2", "B8Ce6fEb2Ca5797dBF"]], + "type": "ValueOnly" }, { "id": "shopify-custom-access-token", - "samples": [["shpca_dfc4", "bC9AeFe6c1D5Ce8bac29c205fAAC"]] + "samples": [["shpca_295d6d04bDfa91", "e3fE2f33be8070435B"]], + "type": "ValueOnly" }, { "id": "shopify-private-app-access-token", - "samples": [["shppa_0F65", "3386D98d09B6c71CEaa0d2C5eb8f"]] + "samples": [["shppa_c62F4FbfFD1CCb", "e3f132Aa971fAf9aC2"]], + "type": "ValueOnly" }, { "id": "shopify-shared-secret", - "samples": [["shpss_AC5f", "50c52C4DCDf69aC52893AF369A74"]] + "samples": [["shpss_1a594acae026c3", "ED6aD2D914d6be67b8"]], + "type": "ValueOnly" + }, + { + "id": "sidekiq-secret", + "samples": [["BUNDLE_ENTERPRISE__C", "ONTRIBSYS__COM = a2b3c962:5fdf45bc"], ["BUNDLE_GEMS__CONTRIB", "SYS__COM = 69d53348:60ef1e43"]], + "type": "NameAndValue" }, { "id": "slack-app-token", - "samples": [["xapp-4-J22", "2-45492-lh9a6"]] + "samples": [["xapp-0-I", "W-564-hk"]], + "type": "ValueOnly" }, { "id": "slack-bot-token", - "samples": [["xoxb-25169", "65025178-629147915426"]] + "samples": [["xoxb-9875249591247-4", "68400680878pYrEc"]], + "type": "ValueOnly" }, { "id": "slack-config-access-token", - "samples": [["xoxe.xoxb-", "6-LF0T5F67P5IWYCH24XT8IB7TKICBNV808OGTTI7XDRCWYXIJ9HGJQYJKHHJWC4U47RYK6YJVZ6518L4XA3CFPZKJE5LL9KUJ7IYG98MK8UTU3SM6MVWI1OPIHOFAM7WOLW30WYXCQ1H4FZPWTP8NYLRS2Q8U3L8A8FIHZ"], ["xoxe.xoxp-", "2-7U2AGVCS8TR79I0AEK8G0DIE3H6ARI67GKPDTRLHNQVJ4BB0OIDOXD74LXG9ZCA0WT0Y2WQX812KR9K2GGVJFSKDMV26KMSLP5XLKM722PP6KXMMNO3TQ1VDJXLH4TMWJQ4WL8R3I3RZCN2ZIPT5S85XJTCXA08IC2X42"]] + "samples": [["xoxe.xoxb-5-SDAW10KZ", "3KJ6C4OTD2714OGX566AS711CN54SB3UENS8MWR2AYDSV27E7N154JH6SS4Z9TBC8NXQ6HKIPDA34JFKQ06EK8Z4GWP4213QXATIFRIZDTQ1JNBJMUJIW62AX2KJ2PQXJTG3Z7F9SHB0R8V2UFSHSV30ISUMG8"], ["xoxe.xoxp-6-3RZHHUOP", "KZP5BSYHRZW6IFY0QVGN22UJNYYR5VNVE2H5N5GA5EZJK13X390WNBWSFGRN4HM6LB2Q72MUOGRYDL9W4AY641ICFZ0HO3FX127J8N0AXGHGT7P4H4S3TQCU30LHBEGUB36KNZYRKXIYTBSHK6ATLLEL0MP"]], + "type": "ValueOnly" }, { "id": "slack-config-refresh-token", - "samples": [["xoxe-0-79K", "XK6A54W8OARDKVGM1643S5IWK086SOE8TYINNSH3O1TIVZNIVNJRDF21BJ0E5ZQ95SUVN39IZAP0LAAVNZKEFX5N6QLD1C7YLUMCE152OA2C0BQBJF1GPL4IXGZ6R5QXD1P0Y7QV3RZE53W"]] + "samples": [["xoxe-0-8S9WF0BEQRGAT", "6HNB2HD7FDXJZZVTBJPP1YZZ652GFQTMK2IEMLXA8V3OLMQBQUOH0V1HRYE6P71ZUL97QRG0VPRG5LNUP00UBMVZABWYG1VA4E49NTU9OYVLDVMGKURYHGXI0K6P4ST44QL1Q"]], + "type": "ValueOnly" }, { "id": "slack-legacy-bot-token", - "samples": [["xoxb-93863", "72971490-tHHi79NSwtRjv5LSxZH15la1uv"]] + "samples": [["xoxb-1817137253-BxMr", "4ZODLqfMjh2hUDOrKS4y"]], + "type": "ValueOnly" }, { "id": "slack-legacy-token", - "samples": [["xoxo-8-038", "06-92105-4"]] + "samples": [["xoxo-711", "5-6-87-BF"]], + "type": "ValueOnly" }, { "id": "slack-legacy-workspace-token", - "samples": [["xoxa-4-yvu", "xRbbzJ"], ["xoxr-EchWx", "U9oknZJBpb28wGNCtcGNNdCuKq8EYTFEesWbW"]] + "samples": [["xoxa-5-ZgusHvIp8VzVD", "FhR75g0CjfZNW"], ["xoxr-7Jt", "eGonFc6Oo"]], + "type": "ValueOnly" }, { "id": "slack-user-token", - "samples": [["xoxp-30723", "4221103-4883165294-800633518510-HjrcUaoMJ4dMCu7OJACJkzqwL2WL"]] + "samples": [["xoxp-24101703864-785", "1525207873-7767242022749-LGooveXzoj6UyrACHAUidtMjy7HQMs"]], + "type": "ValueOnly" }, { "id": "slack-webhook-url", - "samples": [["https://ho", "oks.slack.com/services/i6l82GnghZ6EvnrTIp+5HRrsyQuvtpDFuQpCmHMuSk0aI"], ["https://ho", "oks.slack.com/workflows/JTk448L1kQO4JlX4z4W0Vf+psAcPnidRkNHbL1xG54rYz"]] + "samples": [["https://hooks.slack.", "com/services/rAgqwZnrqs4iQ/EvTsKk6kDzP7gz3r7X4j/MmJpt0CEfT/"], ["https://hooks.slack.", "com/workflows/nOHfnJ76G9ta05zz/xgqt/4JJDdw3gkfQM4kwLTO8o5P5"]], + "type": "ValueOnly" + }, + { + "id": "snyk-api-token", + "samples": [["snyk = f0da19d7-ad4e", "-00cb-4f8e-10175eeb0111"]], + "type": "NameAndValue" }, { "id": "square-access-token", - "samples": [["sq0atp--cM", "Qnx4JFfKxBtWxE2pzKy"]] + "samples": [["sq0atp-qp1F4jpL2dMzL", "ahB3bODAe"]], + "type": "ValueOnly" }, { "id": "square-secret", - "samples": [["sq0csp-yiz", "7FxTqbr_2svCmXCGzT6Ggvc-febrP17PjqqqOork"]] + "samples": [["sq0csp-E2qUIj46fGPEf", "leRsiw9GEprlYnbJshbCpoCGFXdzXj"]], + "type": "ValueOnly" + }, + { + "id": "squarespace-access-token", + "samples": [["squarespace = 1d9e18", "01-da45-965b-2990-622b24daecad"]], + "type": "NameAndValue" }, { "id": "stripe-access-token", - "samples": [["sk_test_6u", "n1199r2z0"], ["sk_live_fa", "7nys600l0qhhypdv6md6h1yymdb"], ["pk_test_b1", "3jajl5fzn"], ["pk_live_2l", "twjjbkd6teh9t5p06tox9zmx"]] + "samples": [["sk_test_xc1ff5ii8hq0", "8zqqyhsskt1uxf"], ["sk_live_m6a6qi3weib6", "5"], ["pk_test_6cp1syairohq", "w44a6"], ["pk_live_zzwcya0h52c8", "mbmhehbs5"]], + "type": "ValueOnly" + }, + { + "id": "sumologic-access-token", + "samples": [["sumo = n485gphrf9fml", "ynw8xq3cr7nk6n99ntbzz94skng4aihan4ura8ehbw67vvkcis2"]], + "type": "NameAndValue" }, { "id": "telegram-bot-api-token", - "samples": [["42168007:A", "iC1R4BmknR2pf9W3J2702PwV21koL7t5D7"]] + "samples": [["83667314:APlIUGqyOS3", "mbHj3SV86y6uHF_0xcAx8KXf"]], + "type": "ValueOnly" + }, + { + "id": "travisci-access-token", + "samples": [["travis = dwwzseykk8b", "i0m7c2s8po4"]], + "type": "NameAndValue" + }, + { + "id": "trello-access-token", + "samples": [["trello = edZZMfgICAQ", "-OmX0YqW-eFMknLmpOFub"]], + "type": "NameAndValue" }, { "id": "twilio-api-key", - "samples": [["SK8f34B8A4", "6537cDB8defA3de6D3E3f1a1"]] + "samples": [["SKdaBaC1A13fF32E69a9", "f02A81A146a5B1"]], + "type": "ValueOnly" + }, + { + "id": "twitch-api-token", + "samples": [["twitch = 8txvh9ggmt1", "8ph46tcet6vsn0ya546"]], + "type": "NameAndValue" + }, + { + "id": "twitter-access-secret", + "samples": [["twitter = c1arlmsjyi", "iu9kwtjy76dpi00a2515ljihmofzthnrmym"]], + "type": "NameAndValue" + }, + { + "id": "twitter-access-token", + "samples": [["twitter = 3446869594", "078745478412913-wqrQZczkjl0H2Tc4JmkIAKx7AL"]], + "type": "NameAndValue" + }, + { + "id": "twitter-api-key", + "samples": [["twitter = d0y1yimyky", "10a7i2rnngsirmm"]], + "type": "NameAndValue" + }, + { + "id": "twitter-api-secret", + "samples": [["twitter = j5t9kmqwxi", "0pfyhn35f5sqr5tsq0p5jlpy9rflnpkoj9bbhy8e"]], + "type": "NameAndValue" + }, + { + "id": "twitter-bearer-token", + "samples": [["twitter = AAAAAAAAAA", "AAAAAAAAAAAALhmFZEsXWsEOi8llgF3LIJoFbOcT%o53hD2yL208Ff83rurdtFNSN4kc3RElCJXHyLHPfXLoQ5gDMvnRw%AELnqotZF5yOG"]], + "type": "NameAndValue" + }, + { + "id": "typeform-api-token", + "samples": [["typeform = tfp_gj5bf", "ku12zvl4mwzt5-=n5.jqwf3uycwpe7vd24c4vt288_iz4=w4omplwz"]], + "type": "NameAndValue" }, { "id": "vault-batch-token", - "samples": [["hvb.7b57r-", "7sz2660kyeuwpzllx3epou2t1px91koj_kba11k4d9n5mc1rdwqb3bxm1farj3un1lyexkv8tmlrh9hz0f5f5ms8z1lv05umo-dbpw54auur1ge4_n2d9pto51q8--h03-ci5gjdlbel4bhtwtxmcrp-r1i0g95a"]] + "samples": [["hvb.i92y0vox64vcivtr", "6atv2re8t32alyg1vbmazyd__t5aqgiow8tcdqc356e94omhy6thjs4i9vwjydsnunqoeeyf_vh4ugfwgxovzafdelet54e1o3oq_9ctinj0k-03pacb18zuimk5-elfqsq230yju6nl3r4exne52dasd6ixyjudbdj4u9-0n5t6un22tfhy-s"]], + "type": "ValueOnly" }, { "id": "vault-service-token", - "samples": [["hvs.kodv8r", "fyp--0r_8cmrsgq_2c7mzqvtj3d9ktz40gve1xzwpgceow-7_mya0w40c82hwu0oot2lxmiepk9sorey3i-96zee"]] + "samples": [["hvs.cn-7v44m162y330-", "wg9ru96d1237qw867ucwlv8x80o7yq_ktm8c7xces8g4b97f_73j0e0bklu5v_ql4tnez5-jr_doodw-"]], + "type": "ValueOnly" + }, + { + "id": "yandex-access-token", + "samples": [["yandex = t1.UM292-==", ".LTzNlTywGlF2htK5fBvEO_l8eAPsCDS-9lbklfKy5_-k5u6aZyWs-NGMa1blEGuC3GuyS2lAtybP-BCm5g3Fj1=="]], + "type": "NameAndValue" + }, + { + "id": "yandex-api-key", + "samples": [["yandex = AQVN59SvoNM", "g7IgHENG9MraeDK42DwCt35Pb5Fd"]], + "type": "NameAndValue" + }, + { + "id": "yandex-aws-access-token", + "samples": [["yandex = YChXlgK95PF", "aJYc1oYsoxIN7fXOGZbZBpGel8-wy"]], + "type": "NameAndValue" + }, + { + "id": "zendesk-secret-key", + "samples": [["zendesk = glcp5nd8jo", "9uk8hm9bw4htn3iox2lbdtk6jp7pch"]], + "type": "NameAndValue" } ] } diff --git a/packages/dd-trace/test/appsec/iast/analyzers/resources/mquery-vulnerable-method.js b/packages/dd-trace/test/appsec/iast/analyzers/resources/mquery-vulnerable-method.js new file mode 100644 index 00000000000..785d6cc7e97 --- /dev/null +++ b/packages/dd-trace/test/appsec/iast/analyzers/resources/mquery-vulnerable-method.js @@ -0,0 +1,38 @@ +'use strict' + +function vulnerableFind (collection, filter) { + return collection + .find(filter) +} + +async function vulnerableFindOne (collection, filter) { + return collection + .findOne(filter) +} + +function vulnerableFindWhere (collection, filter, where) { + return collection + .find(filter) + .where(where) +} + +function vulnerableFindExec (collection, filter) { + return collection + .find(filter) + .exec() +} + +function vulnerableFindWhereExec (collection, filter, where) { + return collection + .find(filter) + .where(where) + .exec() +} + +module.exports = { + vulnerableFind, + vulnerableFindOne, + vulnerableFindWhere, + vulnerableFindExec, + vulnerableFindWhereExec +} diff --git a/packages/dd-trace/test/appsec/iast/analyzers/resources/random-functions.js b/packages/dd-trace/test/appsec/iast/analyzers/resources/random-functions.js new file mode 100644 index 00000000000..f608645242d --- /dev/null +++ b/packages/dd-trace/test/appsec/iast/analyzers/resources/random-functions.js @@ -0,0 +1,24 @@ +function weakRandom () { + return Math.random() +} + +function safeRandom () { + const { randomBytes } = require('node:crypto') + return randomBytes(256) +} + +function customRandom () { + const Math = { + random: function () { + return 4 // chosen by fair dice roll - guaranteed to be random + } + } + + return Math.random() +} + +module.exports = { + weakRandom, + safeRandom, + customRandom +} diff --git a/packages/dd-trace/test/appsec/iast/analyzers/sql-injection-analyzer.pg.plugin.spec.js b/packages/dd-trace/test/appsec/iast/analyzers/sql-injection-analyzer.pg.plugin.spec.js index 8820aeb2eb2..082034ce307 100644 --- a/packages/dd-trace/test/appsec/iast/analyzers/sql-injection-analyzer.pg.plugin.spec.js +++ b/packages/dd-trace/test/appsec/iast/analyzers/sql-injection-analyzer.pg.plugin.spec.js @@ -79,6 +79,7 @@ describe('sql-injection-analyzer with pg', () => { describe('with pool', () => { let pool + beforeEach(() => { pool = new pg.Pool({ host: '127.0.0.1', diff --git a/packages/dd-trace/test/appsec/iast/analyzers/sql-injection-analyzer.sequelize.plugin.spec.js b/packages/dd-trace/test/appsec/iast/analyzers/sql-injection-analyzer.sequelize.plugin.spec.js index bd9c4aa0d4a..b54c64b4186 100644 --- a/packages/dd-trace/test/appsec/iast/analyzers/sql-injection-analyzer.sequelize.plugin.spec.js +++ b/packages/dd-trace/test/appsec/iast/analyzers/sql-injection-analyzer.sequelize.plugin.spec.js @@ -3,6 +3,7 @@ const fs = require('fs') const os = require('os') const path = require('path') +const semver = require('semver') const { prepareTestServerForIast } = require('../utils') const { storage } = require('../../../../../datadog-core') const iastContextFunctions = require('../../../../src/appsec/iast/iast-context') @@ -11,7 +12,13 @@ const vulnerabilityReporter = require('../../../../src/appsec/iast/vulnerability describe('sql-injection-analyzer with sequelize', () => { withVersions('sequelize', 'sequelize', sequelizeVersion => { - withVersions('mysql2', 'mysql2', (mysqlVersion) => { + /** + * mysql2 3.9.4 causes an error when using it with sequelize 4.x, making sequelize plugin test to fail. + * Constraint the test combination of sequelize and mysql2 to force run mysql2 <3.9.4 with sequelize 4.x + */ + const sequelizeSpecificVersion = require(`../../../../../../versions/sequelize@${sequelizeVersion}`).version() + const compatibleMysql2VersionRange = semver.satisfies(sequelizeSpecificVersion, '>=5') ? '>=1' : '>=1 <3.9.4' + withVersions('mysql2', 'mysql2', compatibleMysql2VersionRange, () => { let sequelize prepareTestServerForIast('sequelize + mysql2', diff --git a/packages/dd-trace/test/appsec/iast/analyzers/sql-injection-analyzer.spec.js b/packages/dd-trace/test/appsec/iast/analyzers/sql-injection-analyzer.spec.js index 23b40545401..7716f0ae478 100644 --- a/packages/dd-trace/test/appsec/iast/analyzers/sql-injection-analyzer.spec.js +++ b/packages/dd-trace/test/appsec/iast/analyzers/sql-injection-analyzer.spec.js @@ -78,6 +78,9 @@ describe('sql-injection-analyzer', () => { }, '../overhead-controller': { hasQuota: () => true } }) + sinon.stub(ProxyAnalyzer.prototype, '_reportEvidence') + const reportEvidence = ProxyAnalyzer.prototype._reportEvidence + const InjectionAnalyzer = proxyquire('../../../../src/appsec/iast/analyzers/injection-analyzer', { '../taint-tracking/operations': TaintTrackingMock, './vulnerability-analyzer': ProxyAnalyzer @@ -91,11 +94,11 @@ describe('sql-injection-analyzer', () => { }, '../vulnerability-reporter': { addVulnerability } }) - proxiedSqlInjectionAnalyzer.analyze(TAINTED_QUERY, dialect) - expect(addVulnerability).to.have.been.calledOnce - expect(addVulnerability).to.have.been.calledWithMatch({}, { - type: 'SQL_INJECTION', - evidence: { dialect: dialect } + proxiedSqlInjectionAnalyzer.analyze(TAINTED_QUERY, undefined, dialect) + expect(reportEvidence).to.have.been.calledOnce + expect(reportEvidence).to.have.been.calledWithMatch(TAINTED_QUERY, {}, { + value: TAINTED_QUERY, + dialect }) }) @@ -169,12 +172,12 @@ describe('sql-injection-analyzer', () => { const sqlInjectionAnalyzer = require('../../../../src/appsec/iast/analyzers/sql-injection-analyzer') const knexDialects = { - 'mssql': 'MSSQL', - 'oracle': 'ORACLE', - 'mysql': 'MYSQL', - 'redshift': 'REDSHIFT', - 'postgresql': 'POSTGRES', - 'sqlite3': 'SQLITE' + mssql: 'MSSQL', + oracle: 'ORACLE', + mysql: 'MYSQL', + redshift: 'REDSHIFT', + postgresql: 'POSTGRES', + sqlite3: 'SQLITE' } Object.keys(knexDialects).forEach((knexDialect) => { diff --git a/packages/dd-trace/test/appsec/iast/analyzers/unvalidated-redirect-analyzer.spec.js b/packages/dd-trace/test/appsec/iast/analyzers/unvalidated-redirect-analyzer.spec.js index c941f007adb..d038f7d023b 100644 --- a/packages/dd-trace/test/appsec/iast/analyzers/unvalidated-redirect-analyzer.spec.js +++ b/packages/dd-trace/test/appsec/iast/analyzers/unvalidated-redirect-analyzer.spec.js @@ -77,6 +77,7 @@ describe('unvalidated-redirect-analyzer', () => { } let report + beforeEach(() => { sinon.stub(overheadController, 'hasQuota').returns(1) report = sinon.stub(unvalidatedRedirectAnalyzer, '_report') diff --git a/packages/dd-trace/test/appsec/iast/analyzers/vulnerability-analyzer.spec.js b/packages/dd-trace/test/appsec/iast/analyzers/vulnerability-analyzer.spec.js index 4d86990d887..332e0c29e35 100644 --- a/packages/dd-trace/test/appsec/iast/analyzers/vulnerability-analyzer.spec.js +++ b/packages/dd-trace/test/appsec/iast/analyzers/vulnerability-analyzer.spec.js @@ -17,6 +17,7 @@ describe('vulnerability-analyzer', () => { let pathLine let iastContextHandler let rewriter + beforeEach(() => { vulnerabilityReporter = { createVulnerability: sinon.stub().returns(VULNERABILITY), @@ -43,6 +44,7 @@ describe('vulnerability-analyzer', () => { '../taint-tracking/rewriter': rewriter }) }) + afterEach(() => { sinon.restore() }) @@ -167,6 +169,7 @@ describe('vulnerability-analyzer', () => { describe('addSub', () => { let iastPluginAddSub + beforeEach(() => { iastPluginAddSub = sinon.spy() diff --git a/packages/dd-trace/test/appsec/iast/analyzers/weak-hash-analyzer.spec.js b/packages/dd-trace/test/appsec/iast/analyzers/weak-hash-analyzer.spec.js index 54a5f3ef90a..f856485a522 100644 --- a/packages/dd-trace/test/appsec/iast/analyzers/weak-hash-analyzer.spec.js +++ b/packages/dd-trace/test/appsec/iast/analyzers/weak-hash-analyzer.spec.js @@ -66,6 +66,7 @@ describe('weak-hash-analyzer', () => { describe('some locations should be excluded', () => { let locationPrefix + before(() => { if (process.platform === 'win32') { locationPrefix = 'C:\\path\\to\\project' @@ -137,6 +138,11 @@ describe('weak-hash-analyzer', () => { } expect(weakHashAnalyzer._isExcluded(location)).to.be.true }) + + it('undefined location', () => { + const location = undefined + expect(weakHashAnalyzer._isExcluded(location)).to.be.false + }) }) describe('full feature', () => { diff --git a/packages/dd-trace/test/appsec/iast/analyzers/weak-randomness-analyzer.spec.js b/packages/dd-trace/test/appsec/iast/analyzers/weak-randomness-analyzer.spec.js new file mode 100644 index 00000000000..a80c257760a --- /dev/null +++ b/packages/dd-trace/test/appsec/iast/analyzers/weak-randomness-analyzer.spec.js @@ -0,0 +1,115 @@ +'use strict' + +const fs = require('fs') +const os = require('os') +const path = require('path') +const proxyquire = require('proxyquire') + +const { prepareTestServerForIast } = require('../utils') +const { clearCache } = require('../../../../src/appsec/iast/vulnerability-reporter') +const weakRandomnessAnalyzer = require('../../../../src/appsec/iast/analyzers/weak-randomness-analyzer') + +describe('weak-randomness-analyzer', () => { + weakRandomnessAnalyzer.configure(true) + + it('should subscribe to Math random call channel', () => { + expect(weakRandomnessAnalyzer._subscriptions).to.have.lengthOf(1) + expect(weakRandomnessAnalyzer._subscriptions[0]._channel.name).to.equals('datadog:random:call') + }) + + it('should detect Math.random as vulnerable', () => { + const isVulnerable = weakRandomnessAnalyzer._isVulnerable(Math.random) + expect(isVulnerable).to.be.true + }) + + it('should not detect custom random as vulnerable', () => { + function random () { + return 4 // chosen by fair dice roll - guaranteed to be random + } + const isVulnerable = weakRandomnessAnalyzer._isVulnerable(random) + expect(isVulnerable).to.be.false + }) + + it('should not detect vulnerability when checking empty object', () => { + const isVulnerable = weakRandomnessAnalyzer._isVulnerable({}) + expect(isVulnerable).to.be.false + }) + + it('should not detect vulnerability when no target', () => { + const isVulnerable = weakRandomnessAnalyzer._isVulnerable() + expect(isVulnerable).to.be.false + }) + + it('should report "WEAK_RANDOMNESS" vulnerability', () => { + const addVulnerability = sinon.stub() + const iastContext = { + rootSpan: { + context () { + return { + toSpanId () { + return '123' + } + } + } + } + } + const ProxyAnalyzer = proxyquire('../../../../src/appsec/iast/analyzers/vulnerability-analyzer', { + '../iast-context': { + getIastContext: () => iastContext + }, + '../overhead-controller': { hasQuota: () => true }, + '../vulnerability-reporter': { addVulnerability } + }) + const proxiedWeakRandomnessAnalyzer = proxyquire('../../../../src/appsec/iast/analyzers/weak-randomness-analyzer', + { + './vulnerability-analyzer': ProxyAnalyzer + }) + proxiedWeakRandomnessAnalyzer.analyze(Math.random) + expect(addVulnerability).to.have.been.calledOnce + expect(addVulnerability).to.have.been.calledWithMatch({}, { type: 'WEAK_RANDOMNESS' }) + }) + + describe('Math.random instrumentation', () => { + const randomFunctionsPath = path.join(os.tmpdir(), 'random-functions.js') + + beforeEach(() => { + fs.copyFileSync( + path.join(__dirname, 'resources', 'random-functions.js'), + randomFunctionsPath + ) + }) + + afterEach(() => { + fs.unlinkSync(randomFunctionsPath) + clearCache() + }) + + prepareTestServerForIast('full feature', (testThatRequestHasVulnerability, testThatRequestHasNoVulnerability) => { + describe('should detect weak randomness when calling Math.random', () => { + testThatRequestHasVulnerability(() => { + require(randomFunctionsPath).weakRandom() + }, + 'WEAK_RANDOMNESS', + { + occurrences: 1, + location: { + path: randomFunctionsPath, + line: 2 + } + }) + }) + + describe('should not detect weak randomness when calling safe random function', () => { + testThatRequestHasNoVulnerability(() => { + require(randomFunctionsPath).safeRandom() + }, 'WEAK_RANDOMNESS') + }) + + describe('should not detect weak randomness when calling custom random function', () => { + testThatRequestHasNoVulnerability(() => { + require(randomFunctionsPath).customRandom() + }, 'WEAK_RANDOMNESS') + }) + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/iast/context/context-plugin.spec.js b/packages/dd-trace/test/appsec/iast/context/context-plugin.spec.js new file mode 100644 index 00000000000..e08e3565c41 --- /dev/null +++ b/packages/dd-trace/test/appsec/iast/context/context-plugin.spec.js @@ -0,0 +1,259 @@ +'use strict' + +const proxyquire = require('proxyquire') +const { IastPlugin } = require('../../../../src/appsec/iast/iast-plugin') +const { TagKey } = require('../../../../src/appsec/iast/telemetry/iast-metric') +const { storage } = require('../../../../../datadog-core') +const { IAST_ENABLED_TAG_KEY } = require('../../../../src/appsec/iast/tags') + +describe('IastContextPlugin', () => { + let IastContextPlugin, addSub, getAndRegisterSubscription + let plugin + let acquireRequest, initializeRequestContext, releaseRequest + let saveIastContext, getIastContext, cleanIastContext + let createTransaction, removeTransaction + let sendVulnerabilities + + beforeEach(() => { + addSub = sinon.stub(IastPlugin.prototype, 'addSub') + getAndRegisterSubscription = sinon.stub(IastPlugin.prototype, '_getAndRegisterSubscription') + + acquireRequest = sinon.stub() + initializeRequestContext = sinon.stub() + releaseRequest = sinon.stub() + + saveIastContext = sinon.stub() + getIastContext = sinon.stub() + cleanIastContext = sinon.stub() + + createTransaction = sinon.stub() + removeTransaction = sinon.stub() + + sendVulnerabilities = sinon.stub() + + IastContextPlugin = proxyquire('../../../../src/appsec/iast/context/context-plugin', { + '../iast-plugin': { IastPlugin }, + '../overhead-controller': { + acquireRequest, + initializeRequestContext, + releaseRequest + }, + '../iast-context': { + saveIastContext, + getIastContext, + cleanIastContext + }, + '../taint-tracking/operations': { + createTransaction, + removeTransaction + }, + '../vulnerability-reporter': { + sendVulnerabilities + } + }) + + plugin = new IastContextPlugin() + }) + + afterEach(sinon.restore) + + describe('startCtxOn', () => { + const channelName = 'start' + const tag = {} + + it('should add a subscription to the channel', () => { + plugin.startCtxOn(channelName, tag) + + expect(addSub).to.be.calledOnceWith(channelName) + expect(getAndRegisterSubscription).to.be.calledOnceWith({ channelName, tag, tagKey: TagKey.SOURCE_TYPE }) + }) + + it('should call startContext when event is published', () => { + plugin.startCtxOn(channelName, tag) + + const startContext = sinon.stub(plugin, 'startContext') + .returns({ isRequestAcquired: true, iastContext: {}, store: {} }) + + addSub.firstCall.args[1]() + + expect(startContext).to.be.calledOnce + }) + }) + + describe('finishCtxOn', () => { + const channelName = 'finish' + + it('should add a subscription to the channel', () => { + plugin.finishCtxOn(channelName) + + expect(addSub).to.be.calledOnceWith(channelName) + }) + + it('should call finishContext when event is published', () => { + plugin.finishCtxOn(channelName) + + const finishContext = sinon.stub(plugin, 'finishContext') + .returns({ isRequestAcquired: true, iastContext: {}, store: {} }) + + addSub.firstCall.args[1]() + + expect(finishContext).to.be.calledOnce + }) + }) + + describe('startContext', () => { + const topContext = {} + const rootSpan = { + context: () => { + return { + toSpanId: () => 'span-id' + } + }, + + addTags: () => {} + } + + const store = { + span: rootSpan + } + + let getStore + + beforeEach(() => { + getStore = sinon.stub(storage, 'getStore') + getStore.returns(store) + }) + + it('should obtain needed info from data before starting iast context', () => { + const data = {} + + sinon.stub(plugin, 'getTopContext').returns(topContext) + sinon.stub(plugin, 'getRootSpan').returns(rootSpan) + + plugin.startContext(data) + + expect(plugin.getTopContext).to.be.calledOnce + expect(plugin.getRootSpan).to.be.calledWith(store) + }) + + it('should call overheadController before starting iast context', () => { + plugin.startContext({}) + + expect(acquireRequest).to.be.calledOnceWith(rootSpan) + }) + + it('should add _dd.iast.enabled:0 tag in the rootSpan', () => { + const addTags = sinon.stub(rootSpan, 'addTags') + plugin.startContext({}) + + expect(addTags).to.be.calledOnceWith({ [IAST_ENABLED_TAG_KEY]: 0 }) + }) + + it('should not fail if store does not contain span', () => { + getStore.returns({}) + + plugin.startContext({}) + + expect(acquireRequest).to.be.calledOnceWith(undefined) + }) + + describe('if acquireRequest', () => { + let context, newIastContext + + beforeEach(() => { + acquireRequest.returns(true) + + context = {} + newIastContext = sinon.stub(plugin, 'newIastContext').returns(context) + + saveIastContext.returns(context) + }) + + it('should add _dd.iast.enabled: 1 tag in the rootSpan', () => { + const addTags = sinon.stub(rootSpan, 'addTags') + plugin.startContext({}) + + expect(addTags).to.be.calledOnceWith({ [IAST_ENABLED_TAG_KEY]: 1 }) + }) + + it('should create and save new IAST context and store it', () => { + const data = {} + plugin.startContext(data) + + expect(newIastContext).to.be.calledOnceWith(rootSpan) + expect(saveIastContext).to.be.calledOnceWith(store, topContext, context) + }) + + it('should create new taint-tracking transaction', () => { + const data = {} + plugin.startContext(data) + + expect(createTransaction).to.be.calledOnceWith('span-id', context) + }) + + it('should obtain needed info from data before starting iast context', () => { + plugin.startContext({}) + + expect(initializeRequestContext).to.be.calledOnceWith(context) + }) + }) + }) + + describe('finishContext', () => { + const store = {} + + beforeEach(() => { + sinon.stub(storage, 'getStore').returns(store) + }) + + it('should send the vulnerabilities if any', () => { + const rootSpan = {} + const vulnerabilities = [] + + getIastContext.returns({ + rootSpan: {}, + vulnerabilities: [] + }) + + plugin.finishContext() + + expect(sendVulnerabilities).to.be.calledOnceWith(vulnerabilities, rootSpan) + }) + + it('should remove the taint-tracking transaction', () => { + const iastContext = { + rootSpan: {}, + vulnerabilities: [] + } + + getIastContext.returns(iastContext) + + plugin.finishContext() + + expect(removeTransaction).to.be.calledOnceWith(iastContext) + }) + + it('should clear iastContext and releaseRequest from OCE', () => { + const iastContext = { + rootSpan: {}, + vulnerabilities: [] + } + + cleanIastContext.returns(true) + getIastContext.returns(iastContext) + + plugin.finishContext() + + expect(cleanIastContext).to.be.calledOnce + expect(releaseRequest).to.be.calledOnce + }) + + it('should not fail if there is no iastContext', () => { + getIastContext.returns(undefined) + + plugin.finishContext() + + expect(cleanIastContext).to.be.calledOnce + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/iast/context/kafka-ctx-plugin.spec.js b/packages/dd-trace/test/appsec/iast/context/kafka-ctx-plugin.spec.js new file mode 100644 index 00000000000..ef0b3f07775 --- /dev/null +++ b/packages/dd-trace/test/appsec/iast/context/kafka-ctx-plugin.spec.js @@ -0,0 +1,42 @@ +'use strict' + +const proxyquire = require('proxyquire') +const dc = require('dc-polyfill') +const IastContextPlugin = require('../../../../src/appsec/iast/context/context-plugin') + +const afterStartCh = dc.channel('dd-trace:kafkajs:consumer:afterStart') +const beforeFinishCh = dc.channel('dd-trace:kafkajs:consumer:beforeFinish') + +describe('KafkaContextPlugin', () => { + const message = { key: 'key', value: 'value' } + let plugin + let startContext, finishContext + + beforeEach(() => { + startContext = sinon.stub(IastContextPlugin.prototype, 'startContext') + finishContext = sinon.stub(IastContextPlugin.prototype, 'finishContext') + + plugin = proxyquire('../../../../src/appsec/iast/context/kafka-ctx-plugin', { + './context-plugin': IastContextPlugin + }) + + plugin.enable() + }) + + afterEach(() => { + plugin.disable() + sinon.restore() + }) + + it('should start iast context on dd-trace:kafkajs:consumer:afterStart', () => { + afterStartCh.publish({ message }) + + expect(startContext).to.be.calledOnce + }) + + it('should finish iast context on dd-trace:kafkajs:consumer:beforeFinish', () => { + beforeFinishCh.publish() + + expect(finishContext).to.be.calledOnce + }) +}) diff --git a/packages/dd-trace/test/appsec/iast/iast-log.spec.js b/packages/dd-trace/test/appsec/iast/iast-log.spec.js index b04dbec0faa..bd62a45e06c 100644 --- a/packages/dd-trace/test/appsec/iast/iast-log.spec.js +++ b/packages/dd-trace/test/appsec/iast/iast-log.spec.js @@ -1,9 +1,5 @@ const { expect } = require('chai') const proxyquire = require('proxyquire') -const { calculateDDBasePath } = require('../../../src/util') - -const ddBasePath = calculateDDBasePath(__dirname) -const EOL = '\n' describe('IAST log', () => { let iastLog @@ -86,7 +82,7 @@ describe('IAST log', () => { iastLog.errorAndPublish('error') expect(log.error).to.be.calledOnceWith('error') - expect(telemetryLog.publish).to.be.calledOnceWith({ message: 'error', level: 'ERROR' }) + expect(telemetryLog.publish).to.not.be.called // handled by log.error() }) it('should chain multiple error calls', () => { @@ -96,58 +92,7 @@ describe('IAST log', () => { expect(log.error.getCall(0).args[0]).to.be.eq('error') expect(log.error.getCall(1).args[0]).to.be.eq('errorAndPublish') expect(log.error.getCall(2).args[0]).to.be.eq('error2') - expect(telemetryLog.publish).to.be.calledOnceWith({ message: 'errorAndPublish', level: 'ERROR' }) - }) - - it('should include original message and dd frames', () => { - const ddFrame = `at T (${ddBasePath}packages/dd-trace/test/telemetry/logs/log_collector.spec.js:29:21)` - const stack = new Error('Error 1') - .stack.replace(`Error 1${EOL}`, `Error 1${EOL}${ddFrame}${EOL}`) - - const ddFrames = stack - .split(EOL) - .filter(line => line.includes(ddBasePath)) - .map(line => line.replace(ddBasePath, '')) - - iastLog.errorAndPublish({ message: 'Error 1', stack }) - - expect(telemetryLog.publish).to.be.calledOnce - const log = telemetryLog.publish.getCall(0).args[0] - - expect(log.message).to.be.eq('Error 1') - expect(log.level).to.be.eq('ERROR') - - log.stack_trace.split(EOL).forEach((frame, index) => { - if (index !== 0) { - expect(ddFrames.indexOf(frame) !== -1).to.be.true - } - }) - }) - - it('should not include original message if first frame is not a dd frame', () => { - const thirdPartyFrame = `at callFn (/this/is/not/a/dd/frame/runnable.js:366:21) - at T (${ddBasePath}packages/dd-trace/test/telemetry/logs/log_collector.spec.js:29:21)` - const stack = new Error('Error 1') - .stack.replace(`Error 1${EOL}`, `Error 1${EOL}${thirdPartyFrame}${EOL}`) - - const ddFrames = stack - .split(EOL) - .filter(line => line.includes(ddBasePath)) - .map(line => line.replace(ddBasePath, '')) - - iastLog.errorAndPublish({ message: 'Error 1', stack }) - - expect(telemetryLog.publish).to.be.calledOnce - - const log = telemetryLog.publish.getCall(0).args[0] - expect(log.message).to.be.eq('omitted') - expect(log.level).to.be.eq('ERROR') - - log.stack_trace.split(EOL).forEach((frame, index) => { - if (index !== 0) { - expect(ddFrames.indexOf(frame) !== -1).to.be.true - } - }) + expect(telemetryLog.publish).to.not.be.called // handled by log.error() }) }) }) diff --git a/packages/dd-trace/test/appsec/iast/iast-plugin.spec.js b/packages/dd-trace/test/appsec/iast/iast-plugin.spec.js index ca8a3381676..1c3af349794 100644 --- a/packages/dd-trace/test/appsec/iast/iast-plugin.spec.js +++ b/packages/dd-trace/test/appsec/iast/iast-plugin.spec.js @@ -37,6 +37,7 @@ describe('IAST Plugin', () => { addSub (channelName, handler) { addSubMock(channelName, handler) } + configure (config) { configureMock(config) } @@ -54,7 +55,7 @@ describe('IAST Plugin', () => { errorAndPublish: logError }, './iast-context': { - getIastContext: getIastContext + getIastContext }, './telemetry': { isEnabled: () => false @@ -184,6 +185,9 @@ describe('IAST Plugin', () => { }) describe('with appsec telemetry enabled', () => { + const vulnTags = [`${VULNERABILITY_TYPE}:injection`] + const sourceTags = [`${SOURCE_TYPE}:http.source`] + let iastTelemetry beforeEach(() => { @@ -191,6 +195,7 @@ describe('IAST Plugin', () => { addSub (channelName, handler) { addSubMock(channelName, handler) } + configure (config) { configureMock(config) } @@ -247,7 +252,7 @@ describe('IAST Plugin', () => { expect(getTelemetryHandler).to.be.calledOnceWith(iastPlugin.pluginSubs[1]) }) - it('should register an pluginSubscription and increment a sink metric when a sink module is loaded', () => { + it('should register a pluginSubscription and increment a sink metric when a sink module is loaded', () => { iastPlugin.addSub({ moduleName: 'sink', channelName: 'datadog:sink:start', @@ -257,11 +262,27 @@ describe('IAST Plugin', () => { iastPlugin.configure(true) const metric = getInstrumentedMetric(VULNERABILITY_TYPE) - const metricAdd = sinon.stub(metric, 'add') + const metricInc = sinon.stub(metric, 'inc') + + loadChannel.publish({ name: 'sink' }) + + expect(metricInc).to.be.calledOnceWith(undefined, vulnTags) + }) + + it('should register and increment a sink metric when a sink module is loaded using a tracingChannel', () => { + iastPlugin.addSub({ + channelName: 'tracing:datadog:sink:start', + tag: 'injection', + tagKey: VULNERABILITY_TYPE + }, handler) + iastPlugin.configure(true) + + const metric = getInstrumentedMetric(VULNERABILITY_TYPE) + const metricInc = sinon.stub(metric, 'inc') loadChannel.publish({ name: 'sink' }) - expect(metricAdd).to.be.calledOnceWith(1, 'injection') + expect(metricInc).to.be.calledOnceWith(undefined, vulnTags) }) it('should register an pluginSubscription and increment a source metric when a source module is loaded', () => { @@ -274,11 +295,11 @@ describe('IAST Plugin', () => { iastPlugin.configure(true) const metric = getInstrumentedMetric(SOURCE_TYPE) - const metricAdd = sinon.stub(metric, 'add') + const metricInc = sinon.stub(metric, 'inc') loadChannel.publish({ name: 'source' }) - expect(metricAdd).to.be.calledOnceWith(1, 'http.source') + expect(metricInc).to.be.calledOnceWith(undefined, sourceTags) }) it('should increment a sink metric when event is received', () => { @@ -291,12 +312,12 @@ describe('IAST Plugin', () => { iastPlugin.configure(true) const metric = getExecutedMetric(VULNERABILITY_TYPE) - const metricAdd = sinon.stub(metric, 'add') + const metricInc = sinon.stub(metric, 'inc') const telemetryHandler = addSubMock.secondCall.args[1] telemetryHandler() - expect(metricAdd).to.be.calledOnceWith(1, 'injection') + expect(metricInc).to.be.calledOnceWith(undefined, vulnTags) }) it('should increment a source metric when event is received', () => { @@ -309,30 +330,33 @@ describe('IAST Plugin', () => { iastPlugin.configure(true) const metric = getExecutedMetric(SOURCE_TYPE) - const metricAdd = sinon.stub(metric, 'add') + const metricInc = sinon.stub(metric, 'inc') const telemetryHandler = addSubMock.secondCall.args[1] telemetryHandler() - expect(metricAdd).to.be.calledOnceWith(1, 'http.source') + expect(metricInc).to.be.calledOnceWith(undefined, sourceTags) }) it('should increment a source metric when event is received for every tag', () => { iastPlugin.addSub({ moduleName: 'source', channelName: 'datadog:source:start', - tag: [ 'http.source', 'http.source2', 'http.source3' ], + tag: ['http.source', 'http.source2', 'http.source3'], tagKey: SOURCE_TYPE }, handler) iastPlugin.configure(true) const metric = getExecutedMetric(SOURCE_TYPE) - const metricAdd = sinon.stub(metric, 'add') + const metricInc = sinon.stub(metric, 'inc') const telemetryHandler = addSubMock.secondCall.args[1] telemetryHandler() - expect(metricAdd).to.be.calledOnceWith(1, [ 'http.source', 'http.source2', 'http.source3' ]) + expect(metricInc).to.be.calledThrice + expect(metricInc.firstCall).to.be.calledWith(undefined, [`${SOURCE_TYPE}:http.source`]) + expect(metricInc.secondCall).to.be.calledWith(undefined, [`${SOURCE_TYPE}:http.source2`]) + expect(metricInc.thirdCall).to.be.calledWith(undefined, [`${SOURCE_TYPE}:http.source3`]) }) }) @@ -361,17 +385,17 @@ describe('IAST Plugin', () => { const metric = { inc: sinon.spy() } - const tag = 'tag1' + const tags = 'tag1' const iastContext = {} iastPlugin._execHandlerAndIncMetric({ handler, metric, - tag, + tags, iastContext }) expect(handler).to.be.calledOnce - expect(metric.inc).to.be.calledOnceWithExactly(tag, iastContext) + expect(metric.inc).to.be.calledOnceWithExactly(iastContext, tags) }) }) }) diff --git a/packages/dd-trace/test/appsec/iast/index.spec.js b/packages/dd-trace/test/appsec/iast/index.spec.js index 803c8221d27..f770694ede4 100644 --- a/packages/dd-trace/test/appsec/iast/index.spec.js +++ b/packages/dd-trace/test/appsec/iast/index.spec.js @@ -7,6 +7,7 @@ const iastContextFunctions = require('../../../src/appsec/iast/iast-context') const overheadController = require('../../../src/appsec/iast/overhead-controller') const vulnerabilityReporter = require('../../../src/appsec/iast/vulnerability-reporter') const { testInRequest } = require('./utils') +const { IAST_MODULE } = require('../../../src/appsec/rasp/fs-plugin') describe('IAST Index', () => { beforeEach(() => { @@ -102,6 +103,8 @@ describe('IAST Index', () => { let mockVulnerabilityReporter let mockIast let mockOverheadController + let appsecFsPlugin + let analyzers const config = new Config({ experimental: { @@ -125,9 +128,18 @@ describe('IAST Index', () => { startGlobalContext: sinon.stub(), finishGlobalContext: sinon.stub() } + appsecFsPlugin = { + enable: sinon.stub(), + disable: sinon.stub() + } + analyzers = { + enableAllAnalyzers: sinon.stub() + } mockIast = proxyquire('../../../src/appsec/iast', { './vulnerability-reporter': mockVulnerabilityReporter, - './overhead-controller': mockOverheadController + './overhead-controller': mockOverheadController, + '../rasp/fs-plugin': appsecFsPlugin, + './analyzers': analyzers }) }) @@ -136,6 +148,22 @@ describe('IAST Index', () => { mockIast.disable() }) + describe('enable', () => { + it('should enable AppsecFsPlugin', () => { + mockIast.enable(config) + expect(appsecFsPlugin.enable).to.have.been.calledOnceWithExactly(IAST_MODULE) + expect(analyzers.enableAllAnalyzers).to.have.been.calledAfter(appsecFsPlugin.enable) + }) + }) + + describe('disable', () => { + it('should disable AppsecFsPlugin', () => { + mockIast.enable(config) + mockIast.disable() + expect(appsecFsPlugin.disable).to.have.been.calledOnceWithExactly(IAST_MODULE) + }) + }) + describe('managing overhead controller global context', () => { it('should start global context refresher on iast enabled', () => { mockIast.enable(config) @@ -143,9 +171,24 @@ describe('IAST Index', () => { }) it('should finish global context refresher on iast disabled', () => { + mockIast.enable(config) + mockIast.disable() expect(mockOverheadController.finishGlobalContext).to.have.been.calledOnce }) + + it('should start global context only once when calling enable multiple times', () => { + mockIast.enable(config) + mockIast.enable(config) + + expect(mockOverheadController.startGlobalContext).to.have.been.calledOnce + }) + + it('should not finish global context if not enabled before ', () => { + mockIast.disable(config) + + expect(mockOverheadController.finishGlobalContext).to.have.been.not.called + }) }) describe('managing vulnerability reporter', () => { @@ -156,6 +199,8 @@ describe('IAST Index', () => { }) it('should stop vulnerability reporter on iast disabled', () => { + mockIast.enable(config) + mockIast.disable() expect(mockVulnerabilityReporter.stop).to.have.been.calledOnce }) diff --git a/packages/dd-trace/test/appsec/iast/overhead-controller.spec.js b/packages/dd-trace/test/appsec/iast/overhead-controller.spec.js index e9145b85ea5..7bde02537d9 100644 --- a/packages/dd-trace/test/appsec/iast/overhead-controller.spec.js +++ b/packages/dd-trace/test/appsec/iast/overhead-controller.spec.js @@ -39,33 +39,40 @@ describe('Overhead controller', () => { describe('Global context', () => { let originalSetInterval let originalClearInterval + before(() => { originalSetInterval = global.setInterval originalClearInterval = global.clearInterval }) + beforeEach(() => { global.setInterval = sinon.spy(global.setInterval) global.clearInterval = sinon.spy(global.clearInterval) }) + afterEach(() => { sinon.restore() }) + after(() => { global.setInterval = originalSetInterval global.clearInterval = originalClearInterval }) + it('should not start refresher interval when already started', () => { overheadController.startGlobalContext() overheadController.startGlobalContext() expect(global.setInterval).to.have.been.calledOnce overheadController.finishGlobalContext() }) + it('should stop refresher interval once when already finished', () => { overheadController.startGlobalContext() overheadController.finishGlobalContext() overheadController.finishGlobalContext() expect(global.clearInterval).to.have.been.calledOnce }) + it('should restart refresher when already finished', () => { overheadController.startGlobalContext() overheadController.finishGlobalContext() @@ -216,6 +223,7 @@ describe('Overhead controller', () => { }) }) }) + describe('full feature', () => { describe('multiple request at same time', () => { const TEST_REQUEST_STARTED = 'test-request-started' diff --git a/packages/dd-trace/test/appsec/iast/path-line.spec.js b/packages/dd-trace/test/appsec/iast/path-line.spec.js index 6dd5f9a77d7..11905bcb880 100644 --- a/packages/dd-trace/test/appsec/iast/path-line.spec.js +++ b/packages/dd-trace/test/appsec/iast/path-line.spec.js @@ -9,12 +9,15 @@ class CallSiteMock { this.lineNumber = lineNumber this.columnNumber = columnNumber } + getLineNumber () { return this.lineNumber } + getColumnNumber () { return this.columnNumber } + getFileName () { return this.fileName } @@ -37,14 +40,16 @@ describe('path-line', function () { 'diagnostics_channel' ] let mockPath, pathLine, mockProcess + beforeEach(() => { mockPath = {} mockProcess = {} pathLine = proxyquire('../../../src/appsec/iast/path-line', { - 'path': mockPath, - 'process': mockProcess + path: mockPath, + process: mockProcess }) }) + describe('getFirstNonDDPathAndLine', () => { it('call does not fail', () => { const obj = pathLine.getFirstNonDDPathAndLine() diff --git a/packages/dd-trace/test/appsec/iast/taint-tracking/plugin.spec.js b/packages/dd-trace/test/appsec/iast/taint-tracking/plugin.spec.js index c8d49ee933e..59b7c524aae 100644 --- a/packages/dd-trace/test/appsec/iast/taint-tracking/plugin.spec.js +++ b/packages/dd-trace/test/appsec/iast/taint-tracking/plugin.spec.js @@ -6,8 +6,6 @@ const taintTrackingOperations = require('../../../../src/appsec/iast/taint-track const dc = require('dc-polyfill') const { HTTP_REQUEST_COOKIE_VALUE, - HTTP_REQUEST_COOKIE_NAME, - HTTP_REQUEST_HEADER_NAME, HTTP_REQUEST_HEADER_VALUE, HTTP_REQUEST_PATH_PARAM, HTTP_REQUEST_URI @@ -44,12 +42,13 @@ describe('IAST Taint tracking plugin', () => { }) it('Should subscribe to body parser, qs, cookie and process_params channel', () => { - expect(taintTrackingPlugin._subscriptions).to.have.lengthOf(5) + expect(taintTrackingPlugin._subscriptions).to.have.lengthOf(6) expect(taintTrackingPlugin._subscriptions[0]._channel.name).to.equals('datadog:body-parser:read:finish') expect(taintTrackingPlugin._subscriptions[1]._channel.name).to.equals('datadog:qs:parse:finish') expect(taintTrackingPlugin._subscriptions[2]._channel.name).to.equals('apm:express:middleware:next') expect(taintTrackingPlugin._subscriptions[3]._channel.name).to.equals('datadog:cookie:parse:finish') expect(taintTrackingPlugin._subscriptions[4]._channel.name).to.equals('datadog:express:process_params:start') + expect(taintTrackingPlugin._subscriptions[5]._channel.name).to.equals('apm:graphql:resolve:start') }) describe('taint sources', () => { @@ -203,9 +202,7 @@ describe('IAST Taint tracking plugin', () => { expect(taintTrackingOperations.taintObject).to.be.calledOnceWith( iastContext, cookies, - HTTP_REQUEST_COOKIE_VALUE, - true, - HTTP_REQUEST_COOKIE_NAME + HTTP_REQUEST_COOKIE_VALUE ) }) @@ -243,9 +240,7 @@ describe('IAST Taint tracking plugin', () => { expect(taintTrackingOperations.taintObject).to.be.calledOnceWith( iastContext, req.headers, - HTTP_REQUEST_HEADER_VALUE, - true, - HTTP_REQUEST_HEADER_NAME + HTTP_REQUEST_HEADER_VALUE ) expect(taintTrackingOperations.newTaintedString).to.be.calledOnceWith( diff --git a/packages/dd-trace/test/appsec/iast/taint-tracking/plugins/kafka.spec.js b/packages/dd-trace/test/appsec/iast/taint-tracking/plugins/kafka.spec.js new file mode 100644 index 00000000000..5f334412598 --- /dev/null +++ b/packages/dd-trace/test/appsec/iast/taint-tracking/plugins/kafka.spec.js @@ -0,0 +1,106 @@ +'use strict' + +const proxyquire = require('proxyquire') +const { SourceIastPlugin } = require('../../../../../src/appsec/iast/iast-plugin') +const { KAFKA_MESSAGE_KEY, KAFKA_MESSAGE_VALUE } = require('../../../../../src/appsec/iast/taint-tracking/source-types') + +describe('Kafka consumer plugin', () => { + let kafkaConsumerPlugin + let addSub, handler + let getIastContext + let newTaintedObject, newTaintedString + let iastContext + + beforeEach(() => { + addSub = sinon.stub(SourceIastPlugin.prototype, 'addSub') + newTaintedObject = sinon.stub() + newTaintedString = sinon.stub().callsFake((arg0, arg1) => arg1) + + iastContext = {} + getIastContext = sinon.stub().returns(iastContext) + + kafkaConsumerPlugin = proxyquire('../../../../../src/appsec/iast/taint-tracking/plugins/kafka', { + '../../iast-plugin': { + SourceIastPlugin + }, + '../operations': { + newTaintedObject, + newTaintedString + }, + '../../iast-context': { + getIastContext + } + }) + + kafkaConsumerPlugin.enable(true) + + handler = addSub.firstCall.args[1] + }) + + afterEach(sinon.restore) + + it('should subscribe to dd-trace:kafkajs:consumer:afterStart channel', () => { + expect(addSub).to.be.calledOnceWith({ + channelName: 'dd-trace:kafkajs:consumer:afterStart', + tag: [KAFKA_MESSAGE_KEY, KAFKA_MESSAGE_VALUE] + }) + }) + + it('should taint kafka message', () => { + const message = { + key: Buffer.from('key'), + value: Buffer.from('value') + } + + handler({ message }) + + expect(newTaintedObject).to.be.calledTwice + + expect(newTaintedObject.firstCall).to.be.calledWith(iastContext, message.key, undefined, KAFKA_MESSAGE_KEY) + expect(newTaintedObject.secondCall).to.be.calledWith(iastContext, message.value, undefined, KAFKA_MESSAGE_VALUE) + }) + + it('should taint key Buffer.toString method', () => { + const message = { + key: Buffer.from('keyToString'), + value: Buffer.from('valueToString') + } + + handler({ message }) + + const keyStr = message.key.toString() + + expect(newTaintedString).to.be.calledOnceWith(iastContext, keyStr, undefined, KAFKA_MESSAGE_KEY) + }) + + it('should taint value Buffer.toString method', () => { + const message = { + key: Buffer.from('keyToString'), + value: Buffer.from('valueToString') + } + + handler({ message }) + + const valueStr = message.value.toString() + + expect(newTaintedString).to.be.calledOnceWith(iastContext, valueStr, undefined, KAFKA_MESSAGE_VALUE) + }) + + it('should not fail with an unknown kafka message', () => { + const message = {} + + expect(() => { + handler({ message }) + }).to.not.throw() + }) + + it('should not fail with an unknown kafka message II', () => { + const message = { + key: 'key' + } + + expect(() => { + handler({ message }) + }).to.not.throw() + }) +}) diff --git a/packages/dd-trace/test/appsec/iast/taint-tracking/resources/propagationFunctions.js b/packages/dd-trace/test/appsec/iast/taint-tracking/resources/propagationFunctions.js index 7183074bbd2..de37c351789 100644 --- a/packages/dd-trace/test/appsec/iast/taint-tracking/resources/propagationFunctions.js +++ b/packages/dd-trace/test/appsec/iast/taint-tracking/resources/propagationFunctions.js @@ -6,6 +6,19 @@ function insertStr (str) { return `pre_${str}_suf` } +function templateLiteralEndingWithNumberParams (str) { + const num1 = 1 + const num2 = 2 + return `${str}Literal${num1}${num2}` +} + +function templateLiteralWithTaintedAtTheEnd (str) { + const num1 = 1 + const num2 = 2 + const hello = 'world' + return `Literal${num1}${num2}-${hello}-${str}` +} + function appendStr (str) { let pre = 'pre_' pre += str @@ -52,6 +65,14 @@ function sliceStr (str) { return str.slice(1, 4) } +function toLowerCaseStr (str) { + return str.toLowerCase() +} + +function toUpperCaseStr (str) { + return str.toUpperCase() +} + function replaceStr (str) { return str.replace('ls', 'sl') } @@ -60,20 +81,45 @@ function replaceRegexStr (str) { return str.replace(/ls/g, 'ls') } +function jsonParseStr (str) { + return JSON.parse(str) +} + +function arrayJoin (str) { + return [str, str].join(str) +} + +function arrayInVariableJoin (str) { + const testArr = [str, str] + return testArr.join(',') +} + +function arrayProtoJoin (str) { + return Array.prototype.join.call([str, str], ',') +} + module.exports = { - concatSuffix, - insertStr, appendStr, - trimStr, - trimStartStr, - trimEndStr, - trimProtoStr, + arrayInVariableJoin, + arrayJoin, + arrayProtoJoin, + concatProtoStr, concatStr, + concatSuffix, concatTaintedStr, - concatProtoStr, - substringStr, - substrStr, - sliceStr, + insertStr, + jsonParseStr, + replaceRegexStr, replaceStr, - replaceRegexStr + sliceStr, + substrStr, + substringStr, + templateLiteralEndingWithNumberParams, + templateLiteralWithTaintedAtTheEnd, + toLowerCaseStr, + toUpperCaseStr, + trimEndStr, + trimProtoStr, + trimStartStr, + trimStr } diff --git a/packages/dd-trace/test/appsec/iast/taint-tracking/resources/propagationLodashFunctions.js b/packages/dd-trace/test/appsec/iast/taint-tracking/resources/propagationLodashFunctions.js new file mode 100644 index 00000000000..94b504ab2ee --- /dev/null +++ b/packages/dd-trace/test/appsec/iast/taint-tracking/resources/propagationLodashFunctions.js @@ -0,0 +1,44 @@ +'use strict' + +function trimLodash (_, str) { + return _.trim(str) +} + +function trimStartLodash (_, str) { + return _.trimStart(str) +} + +function trimEndLodash (_, str) { + return _.trimEnd(str) +} + +function toLowerLodash (_, str) { + return _.toLower(str) +} + +function toUpperLodash (_, str) { + return _.toUpper(str) +} + +function arrayJoinLodashWithoutSeparator (_, str) { + return _.join([str, str]) +} + +function arrayJoinLodashWithSeparator (_, str) { + return _.join([str, str], str) +} + +function startCaseLodash (_, str) { + return _.startCase(str) +} + +module.exports = { + arrayJoinLodashWithoutSeparator, + arrayJoinLodashWithSeparator, + toLowerLodash, + toUpperLodash, + startCaseLodash, + trimEndLodash, + trimLodash, + trimStartLodash +} diff --git a/packages/dd-trace/test/appsec/iast/taint-tracking/rewriter-telemetry.spec.js b/packages/dd-trace/test/appsec/iast/taint-tracking/rewriter-telemetry.spec.js index 05cdbd9c8b3..8b3b3a76ad4 100644 --- a/packages/dd-trace/test/appsec/iast/taint-tracking/rewriter-telemetry.spec.js +++ b/packages/dd-trace/test/appsec/iast/taint-tracking/rewriter-telemetry.spec.js @@ -7,7 +7,7 @@ const { Verbosity } = require('../../../../src/appsec/iast/telemetry/verbosity') describe('rewriter telemetry', () => { let iastTelemetry, rewriter, getRewriteFunction - let instrumentedPropagationAdd + let instrumentedPropagationInc beforeEach(() => { iastTelemetry = { @@ -27,7 +27,7 @@ describe('rewriter telemetry', () => { } } } - instrumentedPropagationAdd = sinon.stub(INSTRUMENTED_PROPAGATION, 'add') + instrumentedPropagationInc = sinon.stub(INSTRUMENTED_PROPAGATION, 'inc') }) afterEach(() => { @@ -40,7 +40,7 @@ describe('rewriter telemetry', () => { const rewriteFn = getRewriteFunction(rewriter) rewriteFn('const a = b + c', 'test.js') - expect(instrumentedPropagationAdd).to.not.be.called + expect(instrumentedPropagationInc).to.not.be.called }) it('should increase information metrics with MANDATORY verbosity', () => { @@ -49,7 +49,7 @@ describe('rewriter telemetry', () => { const rewriteFn = getRewriteFunction(rewriter) const result = rewriteFn('const a = b + c', 'test.js') - expect(instrumentedPropagationAdd).to.be.calledOnceWith(result.metrics.instrumentedPropagation) + expect(instrumentedPropagationInc).to.be.calledOnceWith(undefined, result.metrics.instrumentedPropagation) }) it('should increase information metrics with INFORMATION verbosity', () => { @@ -58,7 +58,7 @@ describe('rewriter telemetry', () => { const rewriteFn = getRewriteFunction(rewriter) const result = rewriteFn('const a = b + c', 'test.js') - expect(instrumentedPropagationAdd).to.be.calledOnceWith(result.metrics.instrumentedPropagation) + expect(instrumentedPropagationInc).to.be.calledOnceWith(undefined, result.metrics.instrumentedPropagation) }) it('should increase debug metrics with DEBUG verbosity', () => { @@ -67,6 +67,6 @@ describe('rewriter telemetry', () => { const rewriteFn = getRewriteFunction(rewriter) const result = rewriteFn('const a = b + c', 'test.js') - expect(instrumentedPropagationAdd).to.be.calledOnceWith(result.metrics.instrumentedPropagation) + expect(instrumentedPropagationInc).to.be.calledOnceWith(undefined, result.metrics.instrumentedPropagation) }) }) diff --git a/packages/dd-trace/test/appsec/iast/taint-tracking/rewriter.spec.js b/packages/dd-trace/test/appsec/iast/taint-tracking/rewriter.spec.js index d822a417657..36dd400afb0 100644 --- a/packages/dd-trace/test/appsec/iast/taint-tracking/rewriter.spec.js +++ b/packages/dd-trace/test/appsec/iast/taint-tracking/rewriter.spec.js @@ -13,12 +13,7 @@ describe('IAST Rewriter', () => { }) describe('Enabling rewriter', () => { - let rewriter, iastTelemetry - - const shimmer = { - wrap: sinon.spy(), - unwrap: sinon.spy() - } + let rewriter, iastTelemetry, shimmer class Rewriter { rewrite (content, filename) { @@ -35,32 +30,108 @@ describe('IAST Rewriter', () => { iastTelemetry = { add: sinon.spy() } + + shimmer = { + wrap: sinon.spy(), + unwrap: sinon.spy() + } + + const kSymbolPrepareStackTrace = Symbol('kTestSymbolPrepareStackTrace') + rewriter = proxyquire('../../../../src/appsec/iast/taint-tracking/rewriter', { - '@datadog/native-iast-rewriter': { Rewriter, getPrepareStackTrace: function () {} }, + '@datadog/native-iast-rewriter': { + Rewriter, + getPrepareStackTrace: function (fn) { + const testWrap = function testWrappedPrepareStackTrace (_, callsites) { + return fn(_, callsites) + } + Object.defineProperty(testWrap, kSymbolPrepareStackTrace, { + value: true + }) + return testWrap + }, + kSymbolPrepareStackTrace + }, '../../../../../datadog-shimmer': shimmer, '../../telemetry': iastTelemetry }) }) afterEach(() => { - sinon.restore() + sinon.reset() }) it('Should wrap module compile method on taint tracking enable', () => { rewriter.enableRewriter() expect(shimmer.wrap).to.be.calledOnce expect(shimmer.wrap.getCall(0).args[1]).eq('_compile') + + rewriter.disableRewriter() }) it('Should unwrap module compile method on taint tracking disable', () => { rewriter.disableRewriter() + expect(shimmer.unwrap).to.be.calledOnce expect(shimmer.unwrap.getCall(0).args[1]).eq('_compile') }) + + it('Should keep original prepareStackTrace fn when calling enable and then disable', () => { + const orig = Error.prepareStackTrace + + rewriter.enableRewriter() + + const testPrepareStackTrace = (_, callsites) => { + // do nothing + } + Error.prepareStackTrace = testPrepareStackTrace + + rewriter.disableRewriter() + + expect(Error.prepareStackTrace).to.be.eq(testPrepareStackTrace) + + Error.prepareStackTrace = orig + }) + + it('Should keep original prepareStackTrace fn when calling disable only', () => { + const orig = Error.prepareStackTrace + + const testPrepareStackTrace = (_, callsites) => { + // do nothing + } + Error.prepareStackTrace = testPrepareStackTrace + + rewriter.disableRewriter() + + expect(Error.prepareStackTrace).to.be.eq(testPrepareStackTrace) + + Error.prepareStackTrace = orig + }) + + it('Should keep original prepareStackTrace fn when calling disable if not marked with the Symbol', () => { + const orig = Error.prepareStackTrace + + rewriter.enableRewriter() + + // remove iast property to avoid wrapping the new testPrepareStackTrace fn + delete Error.prepareStackTrace + + const testPrepareStackTrace = (_, callsites) => { + // do nothing + } + Error.prepareStackTrace = testPrepareStackTrace + + rewriter.disableRewriter() + + expect(Error.prepareStackTrace).to.be.eq(testPrepareStackTrace) + + Error.prepareStackTrace = orig + }) }) describe('getOriginalPathAndLineFromSourceMap', () => { let rewriter, getOriginalPathAndLineFromSourceMap, argvs + beforeEach(() => { getOriginalPathAndLineFromSourceMap = sinon.spy() rewriter = proxyquire('../../../../src/appsec/iast/taint-tracking/rewriter', { diff --git a/packages/dd-trace/test/appsec/iast/taint-tracking/sources/graphql.sources.test-utils.js b/packages/dd-trace/test/appsec/iast/taint-tracking/sources/graphql.sources.test-utils.js new file mode 100644 index 00000000000..80ae36e1202 --- /dev/null +++ b/packages/dd-trace/test/appsec/iast/taint-tracking/sources/graphql.sources.test-utils.js @@ -0,0 +1,118 @@ +'use strict' + +const axios = require('axios') +const agent = require('../../../../plugins/agent') +const iast = require('../../../../../src/appsec/iast') +const Config = require('../../../../../src/config') +const vulnerabilityReporter = require('../../../../../src/appsec/iast/vulnerability-reporter') + +const schema = ` +type Book { + title: String, + author: String +} + +type Query { + books(title: String): [Book!]! +}` + +const query = ` +query GetBooks ($title: String) { + books(title: $title) { + title, + author + } +}` + +const queryWithHardcodedArgument = ` +query GetBooks { + books(title: "ls") { + title, + author + } +}` + +const books = [ + { + title: 'Test title', + author: 'Test author' + } +] + +const resolvers = { + Query: { + books: (root, args, context) => { + const { execSync } = require('child_process') + execSync(args.title) + return books.filter(book => { + return book.title.includes(args.title) + }) + } + } +} + +async function makeGraphqlRequest (port, query, variables = {}) { + const headers = { + 'content-type': 'application/json' + } + + return axios.post(`http://localhost:${port}/graphql`, { + operationName: 'GetBooks', + query, + variables + }, { headers, maxRedirects: 0 }) +} + +function graphqlCommonTests (config) { + describe('Graphql sources tests', () => { + beforeEach(() => { + iast.enable(new Config({ + experimental: { + iast: { + enabled: true, + requestSampling: 100 + } + } + })) + vulnerabilityReporter.clearCache() + }) + + afterEach(() => { + iast.disable() + }) + + it('Should detect COMMAND_INJECTION vulnerability with hardcoded query', (done) => { + agent.use(payload => { + expect(payload[0][0].meta).to.have.property('_dd.iast.json') + + const iastJson = JSON.parse(payload[0][0].meta['_dd.iast.json']) + expect(iastJson.vulnerabilities[0].type).to.be.equal('COMMAND_INJECTION') + done() + }) + + makeGraphqlRequest(config.port, queryWithHardcodedArgument) + }) + + it('Should detect COMMAND_INJECTION vulnerability with query and variables', (done) => { + agent.use(payload => { + expect(payload[0][0].meta).to.have.property('_dd.iast.json') + + const iastJson = JSON.parse(payload[0][0].meta['_dd.iast.json']) + expect(iastJson.vulnerabilities[0].type).to.be.equal('COMMAND_INJECTION') + done() + }) + + makeGraphqlRequest(config.port, query, { + title: 'test' + }) + }) + }) +} + +module.exports = { + books, + schema, + query, + resolvers, + graphqlCommonTests +} diff --git a/packages/dd-trace/test/appsec/iast/taint-tracking/sources/plugin.apollo-server-express.plugin.spec.js b/packages/dd-trace/test/appsec/iast/taint-tracking/sources/plugin.apollo-server-express.plugin.spec.js new file mode 100644 index 00000000000..91b6e2849f6 --- /dev/null +++ b/packages/dd-trace/test/appsec/iast/taint-tracking/sources/plugin.apollo-server-express.plugin.spec.js @@ -0,0 +1,58 @@ +'use strict' + +const agent = require('../../../../plugins/agent') +const { + schema, + resolvers, + graphqlCommonTests +} = require('./graphql.sources.test-utils') + +withVersions('graphql', 'express', '>=4', expressVersion => { + withVersions('graphql', 'apollo-server-express', apolloServerExpressVersion => { + const config = {} + let express, expressServer, ApolloServer, gql + let app, server + + before(() => { + return agent.load(['express', 'graphql', 'http'], { client: false }) + }) + + before(() => { + const apolloServerExpress = + require(`../../../../../../../versions/apollo-server-express@${apolloServerExpressVersion}`).get() + ApolloServer = apolloServerExpress.ApolloServer + gql = apolloServerExpress.gql + + express = require(`../../../../../../../versions/express@${expressVersion}`).get() + }) + + before(async () => { + app = express() + + const typeDefs = gql(schema) + + server = new ApolloServer({ + typeDefs, + resolvers + }) + + await server.start() + + server.applyMiddleware({ app }) + + return new Promise(resolve => { + expressServer = app.listen({ port: config.port }, () => { + config.port = expressServer.address().port + resolve() + }) + }) + }) + + after(async () => { + await server.stop() + expressServer.close() + }) + + graphqlCommonTests(config) + }) +}) diff --git a/packages/dd-trace/test/appsec/iast/taint-tracking/sources/plugin.apollo-server.plugin.spec.js b/packages/dd-trace/test/appsec/iast/taint-tracking/sources/plugin.apollo-server.plugin.spec.js new file mode 100644 index 00000000000..bc6f0b7f079 --- /dev/null +++ b/packages/dd-trace/test/appsec/iast/taint-tracking/sources/plugin.apollo-server.plugin.spec.js @@ -0,0 +1,43 @@ +'use strict' + +const path = require('path') +const agent = require('../../../../plugins/agent') +const { + schema, + resolvers, + graphqlCommonTests +} = require('./graphql.sources.test-utils') + +withVersions('apollo-server', '@apollo/server', apolloServerVersion => { + const config = {} + let ApolloServer, startStandaloneServer + let server + + before(() => { + return agent.load(['express', 'graphql', 'apollo-server', 'http'], { client: false }) + }) + + before(() => { + const apolloServerPath = require(`../../../../../../../versions/@apollo/server@${apolloServerVersion}`).getPath() + + ApolloServer = require(apolloServerPath).ApolloServer + startStandaloneServer = require(path.join(apolloServerPath, '..', 'standalone')).startStandaloneServer + }) + + before(async () => { + server = new ApolloServer({ + typeDefs: schema, + resolvers + }) + + const { url } = await startStandaloneServer(server, { listen: { port: config.port } }) + + config.port = new URL(url).port + }) + + after(async () => { + await server.stop() + }) + + graphqlCommonTests(config) +}) diff --git a/packages/dd-trace/test/appsec/iast/taint-tracking/sources/taint-tracking.express.plugin.spec.js b/packages/dd-trace/test/appsec/iast/taint-tracking/sources/taint-tracking.express.plugin.spec.js index 7e1626a2b6f..7465f6b2408 100644 --- a/packages/dd-trace/test/appsec/iast/taint-tracking/sources/taint-tracking.express.plugin.spec.js +++ b/packages/dd-trace/test/appsec/iast/taint-tracking/sources/taint-tracking.express.plugin.spec.js @@ -1,7 +1,6 @@ 'use strict' const axios = require('axios') -const getPort = require('get-port') const semver = require('semver') const agent = require('../../../../plugins/agent') const Config = require('../../../../../src/config') @@ -59,13 +58,13 @@ describe('URI sourcing with express', () => { res.status(200).send() }) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/path/vulnerable`) - .then(() => done()) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/path/vulnerable`) + .then(() => done()) + .catch(done) }) }) }) @@ -137,13 +136,13 @@ describe('Path params sourcing with express', () => { res.status(200).send() }) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/tainted1/tainted2`) - .then(() => done()) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/tainted1/tainted2`) + .then(() => done()) + .catch(done) }) }) @@ -172,13 +171,13 @@ describe('Path params sourcing with express', () => { app.use('/:parameterParent', nestedRouter) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/tainted1/tainted2`) - .then(() => done()) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/tainted1/tainted2`) + .then(() => done()) + .catch(done) }) }) @@ -192,13 +191,13 @@ describe('Path params sourcing with express', () => { app.param('parameter1', checkParamIsTaintedAndNext) app.param('parameter2', checkParamIsTaintedAndNext) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/tainted1/tainted2`) - .then(() => done()) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/tainted1/tainted2`) + .then(() => done()) + .catch(done) }) }) @@ -216,13 +215,13 @@ describe('Path params sourcing with express', () => { app.param('parameter1') app.param('parameter2') - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/tainted1/tainted2`) - .then(() => done()) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/tainted1/tainted2`) + .then(() => done()) + .catch(done) }) }) }) diff --git a/packages/dd-trace/test/appsec/iast/taint-tracking/sources/taint-tracking.headers.spec.js b/packages/dd-trace/test/appsec/iast/taint-tracking/sources/taint-tracking.headers.spec.js index 4a846b875b4..f192db37d7f 100644 --- a/packages/dd-trace/test/appsec/iast/taint-tracking/sources/taint-tracking.headers.spec.js +++ b/packages/dd-trace/test/appsec/iast/taint-tracking/sources/taint-tracking.headers.spec.js @@ -6,10 +6,7 @@ const { storage } = require('../../../../../../datadog-core') const iast = require('../../../../../src/appsec/iast') const iastContextFunctions = require('../../../../../src/appsec/iast/iast-context') const { isTainted, getRanges } = require('../../../../../src/appsec/iast/taint-tracking/operations') -const { - HTTP_REQUEST_HEADER_NAME, - HTTP_REQUEST_HEADER_VALUE -} = require('../../../../../src/appsec/iast/taint-tracking/source-types') +const { HTTP_REQUEST_HEADER_VALUE } = require('../../../../../src/appsec/iast/taint-tracking/source-types') const { testInRequest } = require('../../utils') describe('Headers sourcing', () => { @@ -23,13 +20,6 @@ describe('Headers sourcing', () => { expect(isHeaderValueTainted).to.be.true const taintedHeaderValueRanges = getRanges(iastContext, headerValue) expect(taintedHeaderValueRanges[0].iinfo.type).to.be.equal(HTTP_REQUEST_HEADER_VALUE) - // @see packages/dd-trace/test/appsec/iast/taint-tracking/taint-tracking-operations.spec.js - if (headerName.length >= 10) { - const isHeaderNameTainted = isTainted(iastContext, headerName) - expect(isHeaderNameTainted).to.be.true - const taintedHeaderNameRanges = getRanges(iastContext, headerName) - expect(taintedHeaderNameRanges[0].iinfo.type).to.be.equal(HTTP_REQUEST_HEADER_NAME) - } }) } diff --git a/packages/dd-trace/test/appsec/iast/taint-tracking/taint-tracking-impl.spec.js b/packages/dd-trace/test/appsec/iast/taint-tracking/taint-tracking-impl.spec.js index 03994deff5f..d356753d607 100644 --- a/packages/dd-trace/test/appsec/iast/taint-tracking/taint-tracking-impl.spec.js +++ b/packages/dd-trace/test/appsec/iast/taint-tracking/taint-tracking-impl.spec.js @@ -6,26 +6,33 @@ const path = require('path') const { prepareTestServerForIast, copyFileToTmp } = require('../utils') const { storage } = require('../../../../../datadog-core') const iastContextFunctions = require('../../../../src/appsec/iast/iast-context') -const { newTaintedString, isTainted } = require('../../../../src/appsec/iast/taint-tracking/operations') +const { newTaintedString, isTainted, getRanges } = require('../../../../src/appsec/iast/taint-tracking/operations') const { clearCache } = require('../../../../src/appsec/iast/vulnerability-reporter') const { expect } = require('chai') const propagationFns = [ - 'concatSuffix', - 'insertStr', 'appendStr', - 'trimStr', - 'trimStartStr', - 'trimEndStr', - 'trimProtoStr', + 'arrayInVariableJoin', + 'arrayJoin', + 'arrayProtoJoin', + 'concatProtoStr', 'concatStr', + 'concatSuffix', 'concatTaintedStr', - 'concatProtoStr', - 'substringStr', - 'substrStr', - 'sliceStr', + 'insertStr', + 'replaceRegexStr', 'replaceStr', - 'replaceRegexStr' + 'sliceStr', + 'substrStr', + 'substringStr', + 'templateLiteralEndingWithNumberParams', + 'templateLiteralWithTaintedAtTheEnd', + 'toLowerCaseStr', + 'toUpperCaseStr', + 'trimEndStr', + 'trimProtoStr', + 'trimStartStr', + 'trimStr' ] const commands = [ @@ -43,6 +50,7 @@ const propagationFunctions = require(propagationFunctionsFile) describe('TaintTracking', () => { let instrumentedFunctionsFile + beforeEach(() => { instrumentedFunctionsFile = copyFileToTmp(propagationFunctionsFile) }) @@ -82,10 +90,57 @@ describe('TaintTracking', () => { }) }) }) + + describe('using JSON.parse', () => { + testThatRequestHasVulnerability(function () { + const store = storage.getStore() + const iastContext = iastContextFunctions.getIastContext(store) + + const json = '{"command":"ls -la"}' + const jsonTainted = newTaintedString(iastContext, json, 'param', 'request.type') + + const propFnInstrumented = require(instrumentedFunctionsFile).jsonParseStr + const propFnOriginal = propagationFunctions.jsonParseStr + + const result = propFnInstrumented(jsonTainted) + expect(isTainted(iastContext, result.command)).to.be.true + expect(getRanges(iastContext, result.command)).to.be.deep + .eq([{ + start: 0, + end: 6, + iinfo: { + parameterName: 'command', + parameterValue: 'ls -la', + type: 'request.type' + }, + secureMarks: 0 + }]) + + const resultOrig = propFnOriginal(jsonTainted) + expect(result).deep.eq(resultOrig) + + try { + const childProcess = require('child_process') + childProcess.execSync(result.command, { stdio: 'ignore' }) + } catch (e) { + // do nothing + } + }, 'COMMAND_INJECTION') + }) }) describe('should not catch original Error', () => { - const filtered = ['concatSuffix', 'insertStr', 'appendStr', 'concatTaintedStr'] + const filtered = [ + 'appendStr', + 'arrayInVariableJoin', + 'arrayJoin', + 'arrayProtoJoin', + 'concatSuffix', + 'concatTaintedStr', + 'insertStr', + 'templateLiteralEndingWithNumberParams', + 'templateLiteralWithTaintedAtTheEnd' + ] propagationFns.forEach((propFn) => { if (filtered.includes(propFn)) return it(`invoking ${propFn} with null argument`, () => { diff --git a/packages/dd-trace/test/appsec/iast/taint-tracking/taint-tracking-operations.spec.js b/packages/dd-trace/test/appsec/iast/taint-tracking/taint-tracking-operations.spec.js index 0523cad1374..c105eb5b97c 100644 --- a/packages/dd-trace/test/appsec/iast/taint-tracking/taint-tracking-operations.spec.js +++ b/packages/dd-trace/test/appsec/iast/taint-tracking/taint-tracking-operations.spec.js @@ -23,12 +23,14 @@ function getExpectedMethods () { describe('IAST TaintTracking Operations', () => { let taintTrackingOperations let taintTrackingImpl + let operationsTaintObject let taintedUtilsMock const taintedUtils = { createTransaction: id => id, removeTransaction: id => id, setMaxTransactions: () => {}, newTaintedString: (id, value) => value, + newTaintedObject: (id, value) => value, isTainted: id => id, getRanges: id => id, concat: id => id, @@ -50,14 +52,19 @@ describe('IAST TaintTracking Operations', () => { beforeEach(() => { taintedUtilsMock = sinon.spy(taintedUtils) + operationsTaintObject = proxyquire('../../../../src/appsec/iast/taint-tracking/operations-taint-object', { + '@datadog/native-iast-taint-tracking': taintedUtilsMock + }) taintTrackingImpl = proxyquire('../../../../src/appsec/iast/taint-tracking/taint-tracking-impl', { '@datadog/native-iast-taint-tracking': taintedUtilsMock, + './operations-taint-object': operationsTaintObject, '../../../../../datadog-core': datadogCore }) taintTrackingOperations = proxyquire('../../../../src/appsec/iast/taint-tracking/operations', { '@datadog/native-iast-taint-tracking': taintedUtilsMock, '../../../../../datadog-core': datadogCore, './taint-tracking-impl': taintTrackingImpl, + './operations-taint-object': operationsTaintObject, '../telemetry': iastTelemetry }) }) @@ -153,75 +160,6 @@ describe('IAST TaintTracking Operations', () => { expect(result).to.be.deep.equal(expected) }) - it('Should call newTaintedString in object keys when keyTainting is true', () => { - const iastContext = {} - const transactionId = 'id' - taintTrackingOperations.createTransaction(transactionId, iastContext) - - const VALUE_TYPE = 'value.type' - const KEY_TYPE = 'key.type' - - const obj = { - key1: 'parent', - key2: { - key3: 'child' - } - } - - const result = taintTrackingOperations.taintObject(iastContext, obj, VALUE_TYPE, true, KEY_TYPE) - expect(taintedUtilsMock.newTaintedString.getCall(0)).to.have.been - .calledWithExactly(transactionId, 'key2', 'key2', KEY_TYPE) - expect(taintedUtilsMock.newTaintedString.getCall(1)).to.have.been - .calledWithExactly(transactionId, 'child', 'key2.key3', VALUE_TYPE) - expect(taintedUtilsMock.newTaintedString.getCall(2)).to.have.been - .calledWithExactly(transactionId, 'key3', 'key2.key3', KEY_TYPE) - expect(taintedUtilsMock.newTaintedString.getCall(3)).to.have.been - .calledWithExactly(transactionId, 'parent', 'key1', VALUE_TYPE) - expect(taintedUtilsMock.newTaintedString.getCall(4)).to.have.been - .calledWithExactly(transactionId, 'key1', 'key1', KEY_TYPE) - expect(result).to.equal(obj) - - taintTrackingOperations.removeTransaction() - }) - - it('Should taint object keys when taintingKeys is true', () => { - const taintTrackingOperations = require('../../../../src/appsec/iast/taint-tracking/operations') - const iastContext = {} - const transactionId = 'id' - taintTrackingOperations.createTransaction(transactionId, iastContext) - - const VALUE_TYPE = 'value.type' - const KEY_TYPE = 'key.type' - - const obj = { - keyLargerThan10Chars: 'parent', - anotherKeyLargerThan10Chars: { - shortKey: 'child' - } - } - - const checkValueAndKeyAreTainted = (target, key) => { - // Strings shorter than 10 characters are not tainted directly, but a new instance of the string is created - // in dd-native-iast-taint-tracking. This leads to object keys that meet this condition not being detected - // as tainted - if (key && key.length >= 10) { - const isKeyTainted = taintTrackingOperations.isTainted(iastContext, key) - expect(isKeyTainted).to.be.true - } - - const obj = key ? target[key] : target - if (!key || typeof obj === 'object') { - Object.keys(obj).forEach(k => checkValueAndKeyAreTainted(obj, k)) - } else if (typeof obj === 'string') { - const isValueTainted = taintTrackingOperations.isTainted(iastContext, obj) - expect(isValueTainted).to.be.true - } - } - - taintTrackingOperations.taintObject(iastContext, obj, VALUE_TYPE, true, KEY_TYPE) - checkValueAndKeyAreTainted(obj, null) - }) - it('Should handle the exception', () => { const iastContext = {} const transactionId = 'id' @@ -237,11 +175,14 @@ describe('IAST TaintTracking Operations', () => { } const logSpy = sinon.spy(iastLogStub) - const taintTrackingOperations = proxyquire('../../../../src/appsec/iast/taint-tracking/operations', { + const operationsTaintObject = proxyquire('../../../../src/appsec/iast/taint-tracking/operations-taint-object', { '@datadog/native-iast-taint-tracking': taintedUtils, + '../iast-log': logSpy + }) + const taintTrackingOperations = proxyquire('../../../../src/appsec/iast/taint-tracking/operations', { '../../../../../datadog-core': datadogCore, - '../iast-log': logSpy, - './taint-tracking-impl': taintTrackingImpl + './taint-tracking-impl': taintTrackingImpl, + './operations-taint-object': operationsTaintObject }) taintTrackingOperations.createTransaction(transactionId, iastContext) @@ -311,12 +252,12 @@ describe('IAST TaintTracking Operations', () => { telemetry: { enabled: true, metrics: true } }, 'INFORMATION') - const requestTaintedAdd = sinon.stub(REQUEST_TAINTED, 'add') + const requestTaintedInc = sinon.stub(REQUEST_TAINTED, 'inc') taintTrackingOperations.enableTaintOperations(iastTelemetry.verbosity) taintTrackingOperations.removeTransaction(iastContext) - expect(requestTaintedAdd).to.be.calledOnceWith(5, null, iastContext) + expect(requestTaintedInc).to.be.calledOnceWith(iastContext, 5) }) }) @@ -336,6 +277,7 @@ describe('IAST TaintTracking Operations', () => { describe('enableTaintTracking', () => { let context + beforeEach(() => { context = { [taintTrackingOperations.IAST_TRANSACTION_ID]: 'id' } iastContextFunctions.saveIastContext( @@ -382,7 +324,7 @@ describe('IAST TaintTracking Operations', () => { global._ddiast.plusOperator('helloworld', 'hello', 'world') expect(taintedUtils.concat).to.be.called - expect(executedPropagationIncrease).to.be.calledOnceWith(null, context) + expect(executedPropagationIncrease).to.be.calledOnceWith(context) }) }) @@ -399,6 +341,7 @@ describe('IAST TaintTracking Operations', () => { expect(taintedUtils.newTaintedString).to.be .calledWithExactly(iastContext[taintTrackingOperations.IAST_TRANSACTION_ID], value, param, type) }) + it('Given iastContext with undefined IAST_TRANSACTION_ID should not call TaintedUtils.newTaintedString', () => { const iastContext = {} taintTrackingOperations.newTaintedString(iastContext) @@ -419,6 +362,40 @@ describe('IAST TaintTracking Operations', () => { }) }) + describe('newTaintedObject', () => { + it('Given not null iastContext with defined IAST_TRANSACTION_ID should call TaintedUtils.newTaintedObject', () => { + const iastContext = { + [taintTrackingOperations.IAST_TRANSACTION_ID]: 'id' + } + const value = Buffer.from('value') + const param = 'param' + const type = 'REQUEST' + taintTrackingOperations.newTaintedObject(iastContext, value, param, type) + expect(taintedUtils.newTaintedObject).to.be.called + expect(taintedUtils.newTaintedObject).to.be + .calledWithExactly(iastContext[taintTrackingOperations.IAST_TRANSACTION_ID], value, param, type) + }) + + it('Given iastContext with undefined IAST_TRANSACTION_ID should not call TaintedUtils.newTaintedObject', () => { + const iastContext = {} + taintTrackingOperations.newTaintedObject(iastContext) + expect(taintedUtils.newTaintedObject).not.to.be.called + }) + + it('Given null iastContext should call not TaintedUtils.newTaintedObject', () => { + const iastContext = null + taintTrackingOperations.newTaintedObject(iastContext) + expect(taintedUtils.newTaintedObject).not.to.be.called + }) + + it('Given null iastContext should return the string passed as parameter', () => { + const iastContext = null + const value = Buffer.from('test') + const result = taintTrackingOperations.newTaintedObject(iastContext, value) + expect(result).to.be.equal(value) + }) + }) + describe('isTainted', () => { it('Given not null iastContext with defined IAST_TRANSACTION_ID should call TaintedUtils.isTainted', () => { const iastContext = { @@ -432,6 +409,7 @@ describe('IAST TaintTracking Operations', () => { value ) }) + it('Given iastContext with undefined IAST_TRANSACTION_ID should not call TaintedUtils.isTainted', () => { const iastContext = {} taintTrackingOperations.isTainted(iastContext) diff --git a/packages/dd-trace/test/appsec/iast/taint-tracking/taint-tracking.lodash.plugin.spec.js b/packages/dd-trace/test/appsec/iast/taint-tracking/taint-tracking.lodash.plugin.spec.js new file mode 100644 index 00000000000..d92433959ec --- /dev/null +++ b/packages/dd-trace/test/appsec/iast/taint-tracking/taint-tracking.lodash.plugin.spec.js @@ -0,0 +1,98 @@ +'use strict' + +const fs = require('fs') +const path = require('path') + +const { prepareTestServerForIast, copyFileToTmp } = require('../utils') +const { storage } = require('../../../../../datadog-core') +const iastContextFunctions = require('../../../../src/appsec/iast/iast-context') +const { newTaintedString, isTainted } = require('../../../../src/appsec/iast/taint-tracking/operations') +const { clearCache } = require('../../../../src/appsec/iast/vulnerability-reporter') + +const commands = [ + ' ls -la ', + ' ls -la', + 'ls -la ', + 'ls -la', + ' ls -la 人 ', + ' ls -la 𠆢𠆢𠆢 ', + ' ls -ls �', + ' w ', + 'w' +] + +const propagationLodashFns = [ + 'toLowerLodash', + 'toUpperLodash', + 'trimLodash', + 'trimStartLodash', + 'trimEndLodash', + 'arrayJoinLodashWithoutSeparator', + 'arrayJoinLodashWithSeparator' +] + +const propagationLodashFunctionsFile = path.join(__dirname, 'resources/propagationLodashFunctions.js') +const propagationLodashFunctions = require(propagationLodashFunctionsFile) + +describe('TaintTracking lodash', () => { + let instrumentedFunctionsFile + + beforeEach(() => { + instrumentedFunctionsFile = copyFileToTmp(propagationLodashFunctionsFile) + }) + + afterEach(() => { + fs.unlinkSync(instrumentedFunctionsFile) + clearCache() + }) + + prepareTestServerForIast('should propagate strings with lodash', (testThatRequestHasVulnerability) => { + propagationLodashFns.forEach((propFn) => { + describe(`using ${propFn}()`, () => { + commands.forEach((command) => { + describe(`with command: '${command}'`, () => { + testThatRequestHasVulnerability(function () { + const _ = require('../../../../../../versions/lodash').get() + const store = storage.getStore() + const iastContext = iastContextFunctions.getIastContext(store) + const commandTainted = newTaintedString(iastContext, command, 'param', 'Request') + + const propFnInstrumented = require(instrumentedFunctionsFile)[propFn] + const propFnOriginal = propagationLodashFunctions[propFn] + + const commandResult = propFnInstrumented(_, commandTainted) + expect(isTainted(iastContext, commandResult)).to.be.true + + const commandResultOrig = propFnOriginal(_, commandTainted) + expect(commandResult).eq(commandResultOrig) + + try { + const childProcess = require('child_process') + childProcess.execSync(commandResult, { stdio: 'ignore' }) + } catch (e) { + // do nothing + } + }, 'COMMAND_INJECTION') + }) + }) + }) + }) + }) + + describe('lodash method with no taint tracking', () => { + it('should return the original result', () => { + const _ = require('../../../../../../versions/lodash').get() + const store = storage.getStore() + const iastContext = iastContextFunctions.getIastContext(store) + const taintedValue = newTaintedString(iastContext, 'tainted', 'param', 'Request') + + const propFnInstrumented = require(instrumentedFunctionsFile).startCaseLodash + const propFnOriginal = propagationLodashFunctions.startCaseLodash + + const originalResult = propFnOriginal(_, taintedValue) + const instrumentedResult = propFnInstrumented(_, taintedValue) + expect(instrumentedResult).to.be.equal(originalResult) + expect(isTainted(iastContext, instrumentedResult)).to.be.false + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/iast/telemetry/iast-metric.spec.js b/packages/dd-trace/test/appsec/iast/telemetry/iast-metric.spec.js index f862158ddf3..c1fcea37e86 100644 --- a/packages/dd-trace/test/appsec/iast/telemetry/iast-metric.spec.js +++ b/packages/dd-trace/test/appsec/iast/telemetry/iast-metric.spec.js @@ -10,72 +10,80 @@ const { INSTRUMENTED_SINK, INSTRUMENTED_SOURCE } = require('../../../../src/appsec/iast/telemetry/iast-metric') +const { globalNamespace } = require('../../../../src/appsec/iast/telemetry/namespaces') + +const { Namespace } = require('../../../../src/telemetry/metrics') describe('Metrics', () => { - let IastMetric, reqNamespace, inc, context + let IastMetric, NoTaggedIastMetric, reqNamespace, inc, context + beforeEach(() => { context = {} inc = sinon.stub() const metricMock = { inc } reqNamespace = { - count: sinon.stub().returns(metricMock) + count: sinon.stub(globalNamespace, 'count').returns(metricMock) } const metric = proxyquire('../../../../src/appsec/iast/telemetry/iast-metric', { './namespaces': { - getNamespaceFromContext: () => reqNamespace + getNamespaceFromContext: () => globalNamespace } }) IastMetric = metric.IastMetric + NoTaggedIastMetric = metric.NoTaggedIastMetric }) - afterEach(sinon.restore) + afterEach(() => { + globalNamespace.iastMetrics.clear() + sinon.restore() + }) it('should increase by one the metric value', () => { - const metric = new IastMetric('test.metric', 'REQUEST') + const metric = new NoTaggedIastMetric('test.metric', 'REQUEST') - metric.inc(undefined, context) + metric.inc(context) - expect(reqNamespace.count).to.be.calledOnceWith(metric.name, undefined) + expect(reqNamespace.count).to.be.calledOnceWith(metric.name) expect(inc).to.be.calledOnceWith(1) }) it('should add by 42 the metric value', () => { - const metric = new IastMetric('test.metric', 'REQUEST', 'tagKey') + const metric = new NoTaggedIastMetric('test.metric', 'REQUEST', 'tagKey') - metric.add(42, undefined, context) + metric.inc(context, 42) - expect(reqNamespace.count).to.be.calledOnceWith(metric.name, undefined) + expect(reqNamespace.count).to.be.calledOnceWith(metric.name) expect(inc).to.be.calledOnceWith(42) }) it('should increase by one the metric tag value', () => { const metric = new IastMetric('test.metric', 'REQUEST', 'tagKey') - metric.inc('tag1', context) + metric.inc(context, 'tagKey:tag1') - expect(reqNamespace.count).to.be.calledOnceWith(metric.name, { tagKey: 'tag1' }) + expect(reqNamespace.count).to.be.calledOnceWith(metric.name, 'tagKey:tag1') expect(inc).to.be.calledOnceWith(1) }) it('should add by 42 the metric tag value', () => { const metric = new IastMetric('test.metric', 'REQUEST', 'tagKey') - metric.add(42, 'tag1', context) + metric.inc(context, 'tagKey:tag1', 42) - expect(reqNamespace.count).to.be.calledOnceWith(metric.name, { tagKey: 'tag1' }) + expect(reqNamespace.count).to.be.calledOnceWith(metric.name, 'tagKey:tag1') expect(inc).to.be.calledOnceWith(42) }) - it('should add by 42 the each metric tag value', () => { + it('should format tags according with its tagKey', () => { const metric = new IastMetric('test.metric', 'REQUEST', 'tagKey') - metric.add(42, ['tag1', 'tag2'], context) + metric.formatTags('tag1', 'tag2').forEach(tag => metric.inc(context, tag, 42)) expect(reqNamespace.count).to.be.calledTwice - expect(reqNamespace.count.firstCall.args).to.be.deep.equals([metric.name, { tagKey: 'tag1' }]) - expect(reqNamespace.count.secondCall.args).to.be.deep.equals([metric.name, { tagKey: 'tag2' }]) + expect(reqNamespace.count.firstCall.args).to.be.deep.equals([metric.name, ['tagKey:tag1']]) + expect(reqNamespace.count.secondCall.args).to.be.deep.equals([metric.name, ['tagKey:tag2']]) }) it('getExecutedMetric should return a metric depending on tag', () => { @@ -95,4 +103,33 @@ describe('Metrics', () => { metric = getInstrumentedMetric(TagKey.SOURCE_TYPE) expect(metric).to.be.equal(INSTRUMENTED_SOURCE) }) + + describe('NoTaggedIastMetric', () => { + it('should define an empty array as its tags', () => { + const noTagged = new NoTaggedIastMetric('notagged', 'scope') + + expect(noTagged.name).to.be.eq('notagged') + expect(noTagged.scope).to.be.eq('scope') + expect(noTagged.tags).to.be.deep.eq([]) + }) + + it('should increment in 1 the metric', () => { + const noTagged = new NoTaggedIastMetric('notagged', 'scope') + noTagged.inc() + + expect(inc).to.be.calledOnceWith(1) + }) + + it('should reuse previous metric when calling add multiple times', () => { + sinon.restore() + const superCount = sinon.stub(Namespace.prototype, 'count').returns({ inc: () => {} }) + + const noTagged = new NoTaggedIastMetric('notagged') + + noTagged.inc(undefined, 42) + noTagged.inc(undefined, 42) + + expect(superCount).to.be.calledOnceWith('notagged') + }) + }) }) diff --git a/packages/dd-trace/test/appsec/iast/telemetry/logs.spec.js b/packages/dd-trace/test/appsec/iast/telemetry/logs.spec.js new file mode 100644 index 00000000000..a8b5c358b9f --- /dev/null +++ b/packages/dd-trace/test/appsec/iast/telemetry/logs.spec.js @@ -0,0 +1,60 @@ +'use strict' + +const proxyquire = require('proxyquire') +const dc = require('dc-polyfill') + +const telemetryLog = dc.channel('datadog:telemetry:log') + +describe('Telemetry logs', () => { + let telemetry + let clock + let start, send + + before(() => { + clock = sinon.useFakeTimers() + }) + + after(() => { + clock.restore() + telemetry.stop() + }) + + it('should be started and send logs when log received via the datadog:telemetry:log channel', () => { + start = sinon.stub() + send = sinon.spy() + + telemetry = proxyquire('../../../../src/telemetry', { + '../exporters/common/docker': { + id () { + return 'test docker id' + } + }, + './logs': { + start, + send + } + }) + + const config = { + telemetry: { enabled: true, heartbeatInterval: 3000, logCollection: true }, + version: '1.2.3-beta4', + appsec: { enabled: false }, + profiling: { enabled: false }, + env: 'preprod', + tags: { + 'runtime-id': '1a2b3c' + } + } + + telemetry.start(config, { + _pluginsByName: {} + }) + + telemetryLog.publish({ message: 'This is an Error', level: 'ERROR' }) + + clock.tick(3000) + + expect(start).to.be.calledOnceWith(config) + expect(send).to.be.calledOnceWith(config) + }) +}) diff --git a/packages/dd-trace/test/appsec/iast/telemetry/namespaces.spec.js b/packages/dd-trace/test/appsec/iast/telemetry/namespaces.spec.js index 4b2e6a5ca78..6c4aa361e77 100644 --- a/packages/dd-trace/test/appsec/iast/telemetry/namespaces.spec.js +++ b/packages/dd-trace/test/appsec/iast/telemetry/namespaces.spec.js @@ -4,9 +4,13 @@ const { initRequestNamespace, finalizeRequestNamespace, DD_IAST_METRICS_NAMESPACE, - globalNamespace + globalNamespace, + + IastNamespace } = require('../../../../src/appsec/iast/telemetry/namespaces') +const { Namespace } = require('../../../../src/telemetry/metrics') + const REQUEST_TAINTED = 'request.tainted' const EXECUTED_SINK = 'executed.sink' const TAG_PREFIX = '_dd.iast.telemetry' @@ -27,6 +31,7 @@ describe('IAST metric namespaces', () => { }) afterEach(() => { + globalNamespace.clear() sinon.restore() }) @@ -42,7 +47,7 @@ describe('IAST metric namespaces', () => { const tag = rootSpan.addTags.getCalls()[0].args[0] expect(tag).to.has.property(`${TAG_PREFIX}.${REQUEST_TAINTED}`) - expect(tag[`${TAG_PREFIX}.${REQUEST_TAINTED}`]).to.be.eq(10) + expect(tag[`${TAG_PREFIX}.${REQUEST_TAINTED}`]).to.be.equal(10) expect(context[DD_IAST_METRICS_NAMESPACE]).to.be.undefined }) @@ -59,30 +64,165 @@ describe('IAST metric namespaces', () => { const calls = rootSpan.addTags.getCalls() const reqTaintedTag = calls[0].args[0] expect(reqTaintedTag).to.has.property(`${TAG_PREFIX}.${REQUEST_TAINTED}`) - expect(reqTaintedTag[`${TAG_PREFIX}.${REQUEST_TAINTED}`]).to.be.eq(15) + expect(reqTaintedTag[`${TAG_PREFIX}.${REQUEST_TAINTED}`]).to.be.equal(15) const execSinkTag = calls[1].args[0] expect(execSinkTag).to.has.property(`${TAG_PREFIX}.${EXECUTED_SINK}`) - expect(execSinkTag[`${TAG_PREFIX}.${EXECUTED_SINK}`]).to.be.eq(1) + expect(execSinkTag[`${TAG_PREFIX}.${EXECUTED_SINK}`]).to.be.equal(1) }) it('should merge all kind of metrics in global Namespace as gauges', () => { - namespace.count(REQUEST_TAINTED, { tag1: 'test' }).inc(10) + namespace.count(REQUEST_TAINTED, ['tag1:test']).inc(10) namespace.count(EXECUTED_SINK).inc(1) const metric = { inc: sinon.spy() } - sinon.stub(globalNamespace, 'count').returns(metric) + const count = sinon.stub(Namespace.prototype, 'count').returns(metric) finalizeRequestNamespace(context, rootSpan) - expect(globalNamespace.count).to.be.calledTwice - expect(globalNamespace.count.firstCall.args).to.be.deep.equal([REQUEST_TAINTED, ['tag1:test']]) + expect(count).to.be.calledTwice + expect(count.firstCall.args).to.be.deep.equal([REQUEST_TAINTED, ['tag1:test']]) expect(metric.inc).to.be.calledTwice expect(metric.inc.firstCall.args[0]).to.equal(10) - expect(globalNamespace.count.secondCall.args).to.be.deep.equal([EXECUTED_SINK, []]) + expect(count.secondCall.args).to.be.deep.equal([EXECUTED_SINK, undefined]) expect(metric.inc.secondCall.args[0]).to.equal(1) }) + + it('should cache metrics from different request namespaces', () => { + const context2 = {} + const namespace2 = initRequestNamespace(context2) + namespace2.count(REQUEST_TAINTED, { tag1: 'test' }).inc(10) + + finalizeRequestNamespace(context2) + + const context3 = {} + const namespace3 = initRequestNamespace(context3) + namespace3.count(REQUEST_TAINTED, { tag1: 'test' }).inc(10) + + finalizeRequestNamespace(context3) + + expect(globalNamespace.iastMetrics.size).to.be.equal(1) + }) + + it('should clear metric and distribution collections and iast metrics cache', () => { + namespace.count(REQUEST_TAINTED, ['tag1:test']).inc(10) + namespace.distribution('test.distribution', ['tag2:test']).track(10) + + finalizeRequestNamespace(context) + + expect(namespace.iastMetrics.size).to.be.equal(0) + expect(namespace.metrics.size).to.be.equal(0) + expect(namespace.distributions.size).to.be.equal(0) + }) +}) + +describe('IastNamespace', () => { + describe('getIastMetric', () => { + it('should create an IastMetric map with metric name as its key', () => { + const namespace = new IastNamespace() + + const metrics = namespace.getIastMetrics('metric.name') + + expect(metrics).to.not.undefined + expect(metrics instanceof Map).to.be.true + }) + + it('should reuse the same map if created before', () => { + const namespace = new IastNamespace() + + expect(namespace.getIastMetrics('metric.name')).to.be.equal(namespace.getIastMetrics('metric.name')) + }) + }) + + describe('getMetric', () => { + beforeEach(sinon.restore) + + it('should register a new count type metric and store it in the map', () => { + const namespace = new IastNamespace() + + const metric = namespace.getMetric('metric.name', ['key:tag1']) + + expect(metric).to.not.be.undefined + expect(metric.metric).to.be.equal('metric.name') + expect(metric.namespace).to.be.equal('iast') + expect(metric.type).to.be.equal('count') + expect(metric.tags).to.be.deep.equal(['key:tag1', `version:${process.version}`]) + }) + + it('should register a new count type metric and store it in the map supporting non array tags', () => { + const namespace = new IastNamespace() + + const metric = namespace.getMetric('metric.name', { key: 'tag1' }) + + expect(metric).to.not.be.undefined + expect(metric.metric).to.be.equal('metric.name') + expect(metric.namespace).to.be.equal('iast') + expect(metric.type).to.be.equal('count') + expect(metric.tags).to.be.deep.equal(['key:tag1', `version:${process.version}`]) + }) + + it('should register a new distribution type metric and store it in the map', () => { + const namespace = new IastNamespace() + + const metric = namespace.getMetric('metric.name', ['key:tag1'], 'distribution') + + expect(metric).to.not.be.undefined + expect(metric.metric).to.be.equal('metric.name') + expect(metric.namespace).to.be.equal('iast') + expect(metric.type).to.be.equal('distribution') + expect(metric.tags).to.be.deep.equal(['key:tag1', `version:${process.version}`]) + }) + + it('should not add the version tags to the tags array', () => { + const namespace = new IastNamespace() + + const tags = ['key:tag1'] + const metric = namespace.getMetric('metric.name', tags) + + expect(tags).to.be.deep.equal(['key:tag1']) + expect(metric.tags).to.be.deep.equal(['key:tag1', `version:${process.version}`]) + }) + + it('should not create a previously created metric', () => { + const namespace = new IastNamespace() + + const metric = {} + const count = sinon.stub(Namespace.prototype, 'count').returns(metric) + + const tags = ['key:tag1'] + namespace.getMetric('metric.name', tags) + namespace.getMetric('metric.name', tags) + + expect(count).to.be.calledOnceWith('metric.name', tags) + }) + + it('should reuse a previously created metric', () => { + const namespace = new IastNamespace() + + const metric = namespace.getMetric('metric.name', ['key:tag1']) + + metric.track(42) + + const metric2 = namespace.getMetric('metric.name', ['key:tag1']) + + expect(metric2).to.be.equal(metric) + expect(metric2.points[0][1]).to.be.equal(42) + }) + + it('should not cache more than max tags for same metric', () => { + const namespace = new IastNamespace(1) + + namespace.getMetric('metric.name', ['key:tag1']) + + namespace.getMetric('metric.name', ['key:tag2']) + + namespace.getMetric('metric.name', ['key:tag3']) + + expect(namespace.iastMetrics.size).to.be.equal(1) + expect(namespace.iastMetrics.get('metric.name').size).to.be.equal(1) + }) + }) }) diff --git a/packages/dd-trace/test/appsec/iast/telemetry/span-tags.spec.js b/packages/dd-trace/test/appsec/iast/telemetry/span-tags.spec.js index 796e06985e2..2a7cc484529 100644 --- a/packages/dd-trace/test/appsec/iast/telemetry/span-tags.spec.js +++ b/packages/dd-trace/test/appsec/iast/telemetry/span-tags.spec.js @@ -5,8 +5,7 @@ const { EXECUTED_SINK, EXECUTED_SOURCE, REQUEST_TAINTED } = require('../../../.. const { addMetricsToSpan } = require('../../../../src/appsec/iast/telemetry/span-tags') const { getNamespaceFromContext, - initRequestNamespace, - globalNamespace + initRequestNamespace } = require('../../../../src/appsec/iast/telemetry/namespaces') describe('Telemetry Span tags', () => { @@ -19,15 +18,13 @@ describe('Telemetry Span tags', () => { } context = {} initRequestNamespace(context) - - globalNamespace.metrics.clear() }) afterEach(sinon.restore) it('should add span tags with tag name like \'tagPrefix.metricName.tagKey\' for tagged metrics', () => { - EXECUTED_SOURCE.add(42, 'source.type.1', context) - EXECUTED_SINK.add(3, 'sink_type_1', context) + EXECUTED_SOURCE.inc(context, ['source.type.1'], 42) + EXECUTED_SINK.inc(context, ['sink_type_1'], 3) const { metrics } = getNamespaceFromContext(context).toJSON() @@ -40,10 +37,10 @@ describe('Telemetry Span tags', () => { it('should add span tags with tag name like \'tagPrefix.metricName.tagKey\' for tagged metrics flattened', () => { // a request metric with no context it behaves like a global metric - EXECUTED_SOURCE.add(42, 'source.type.1') - EXECUTED_SOURCE.add(32, 'source.type.1') + EXECUTED_SOURCE.inc(context, ['source.type.1'], 42) + EXECUTED_SOURCE.inc(context, ['source.type.1'], 32) - const { metrics } = globalNamespace.toJSON() + const { metrics } = getNamespaceFromContext(context).toJSON() addMetricsToSpan(rootSpan, metrics.series, tagPrefix) @@ -52,12 +49,12 @@ describe('Telemetry Span tags', () => { it('should add span tags with tag name like \'tagPrefix.metricName.tagKey\' for different tagged metrics', () => { // a request metric with no context it behaves like a global metric - EXECUTED_SOURCE.add(42, 'source.type.1') - EXECUTED_SOURCE.add(32, 'source.type.1') + EXECUTED_SOURCE.inc(context, ['source.type.1'], 42) + EXECUTED_SOURCE.inc(context, ['source.type.1'], 32) - EXECUTED_SOURCE.add(2, 'source.type.2') + EXECUTED_SOURCE.inc(context, ['source.type.2'], 2) - const { metrics } = globalNamespace.toJSON() + const { metrics } = getNamespaceFromContext(context).toJSON() addMetricsToSpan(rootSpan, metrics.series, tagPrefix) @@ -67,7 +64,7 @@ describe('Telemetry Span tags', () => { }) it('should add span tags with tag name like \'tagPrefix.metricName\' for not tagged metrics', () => { - REQUEST_TAINTED.add(42, null, context) + REQUEST_TAINTED.inc(context, 42) const { metrics } = getNamespaceFromContext(context).toJSON() diff --git a/packages/dd-trace/test/appsec/iast/utils.js b/packages/dd-trace/test/appsec/iast/utils.js index b48404e8a93..2ef5a77ee30 100644 --- a/packages/dd-trace/test/appsec/iast/utils.js +++ b/packages/dd-trace/test/appsec/iast/utils.js @@ -4,7 +4,6 @@ const fs = require('fs') const os = require('os') const path = require('path') -const getPort = require('get-port') const agent = require('../../plugins/agent') const axios = require('axios') const iast = require('../../../src/appsec/iast') @@ -17,12 +16,6 @@ function testInRequest (app, tests) { let appListener const config = {} - beforeEach(() => { - return getPort().then(newPort => { - config.port = newPort - }) - }) - beforeEach(() => { listener = (req, res) => { const appResult = app && app(req, res) @@ -48,7 +41,10 @@ function testInRequest (app, tests) { beforeEach(done => { const server = new http.Server(listener) appListener = server - .listen(config.port, 'localhost', () => done()) + .listen(0, 'localhost', () => { + config.port = appListener.address().port + done() + }) }) afterEach(() => { @@ -59,9 +55,9 @@ function testInRequest (app, tests) { tests(config) } -function testOutsideRequestHasVulnerability (fnToTest, vulnerability) { +function testOutsideRequestHasVulnerability (fnToTest, vulnerability, plugins, timeout) { beforeEach(async () => { - await agent.load() + await agent.load(plugins) }) afterEach(() => { return agent.close({ ritmReset: false }) @@ -82,13 +78,17 @@ function testOutsideRequestHasVulnerability (fnToTest, vulnerability) { iast.disable() }) it(`should detect ${vulnerability} vulnerability out of request`, function (done) { + if (timeout) { + this.timeout(timeout) + } agent .use(traces => { expect(traces[0][0].meta['_dd.iast.json']).to.include(`"${vulnerability}"`) expect(traces[0][0].metrics['_dd.iast.enabled']).to.be.equal(1) - }) + }, { timeoutMs: 10000 }) .then(done) .catch(done) + fnToTest() }) } @@ -112,9 +112,7 @@ function beforeEachIastTest (iastConfig) { beforeEach(() => { vulnerabilityReporter.clearCache() iast.enable(new Config({ - experimental: { - iast: iastConfig - } + iast: iastConfig })) }) } @@ -154,7 +152,7 @@ function checkNoVulnerabilityInRequest (vulnerability, config, done, makeRequest function checkVulnerabilityInRequest (vulnerability, occurrencesAndLocation, cb, makeRequest, config, done) { let location let occurrences = occurrencesAndLocation - if (typeof occurrencesAndLocation === 'object') { + if (occurrencesAndLocation !== null && typeof occurrencesAndLocation === 'object') { location = occurrencesAndLocation.location occurrences = occurrencesAndLocation.occurrences } @@ -170,7 +168,7 @@ function checkVulnerabilityInRequest (vulnerability, occurrencesAndLocation, cb, vulnerabilitiesCount.set(v.type, ++count) }) - expect(vulnerabilitiesCount.get(vulnerability)).to.not.be.null + expect(vulnerabilitiesCount.get(vulnerability)).to.be.greaterThan(0) if (occurrences) { expect(vulnerabilitiesCount.get(vulnerability)).to.equal(occurrences) } @@ -215,12 +213,6 @@ function prepareTestServerForIast (description, tests, iastConfig) { let appListener let app - before(() => { - return getPort().then(newPort => { - config.port = newPort - }) - }) - before(() => { listener = (req, res) => { endResponse(res, app && app(req, res)) @@ -237,7 +229,10 @@ function prepareTestServerForIast (description, tests, iastConfig) { before(done => { const server = new http.Server(listener) appListener = server - .listen(config.port, 'localhost', () => done()) + .listen(0, 'localhost', () => { + config.port = appListener.address().port + done() + }) }) beforeEachIastTest(iastConfig) @@ -252,8 +247,8 @@ function prepareTestServerForIast (description, tests, iastConfig) { return agent.close({ ritmReset: false }) }) - function testThatRequestHasVulnerability (fn, vulnerability, occurrences, cb, makeRequest) { - it(`should have ${vulnerability} vulnerability`, function (done) { + function testThatRequestHasVulnerability (fn, vulnerability, occurrences, cb, makeRequest, description) { + it(description || `should have ${vulnerability} vulnerability`, function (done) { this.timeout(5000) app = fn checkVulnerabilityInRequest(vulnerability, occurrences, cb, makeRequest, config, done) @@ -292,14 +287,14 @@ function prepareTestServerForIastInExpress (description, expressVersion, loadMid before((done) => { const express = require(`../../../../../versions/express@${expressVersion}`).get() - const bodyParser = require(`../../../../../versions/body-parser`).get() + const bodyParser = require('../../../../../versions/body-parser').get() const expressApp = express() if (loadMiddlewares) loadMiddlewares(expressApp) expressApp.use(bodyParser.json()) try { - const cookieParser = require(`../../../../../versions/cookie-parser`).get() + const cookieParser = require('../../../../../versions/cookie-parser').get() expressApp.use(cookieParser()) } catch (e) { // do nothing, in some scenarios we don't have cookie-parser dependency available, and we don't need @@ -307,11 +302,10 @@ function prepareTestServerForIastInExpress (description, expressVersion, loadMid } expressApp.all('/', listener) - getPort().then(newPort => { - config.port = newPort - server = expressApp.listen(newPort, () => { - done() - }) + + server = expressApp.listen(0, () => { + config.port = server.address().port + done() }) }) @@ -329,7 +323,7 @@ function prepareTestServerForIastInExpress (description, expressVersion, loadMid function testThatRequestHasVulnerability (fn, vulnerability, occurrencesAndLocation, cb, makeRequest) { let testDescription - if (typeof fn === 'object') { + if (fn !== null && typeof fn === 'object') { const obj = fn fn = obj.fn vulnerability = obj.vulnerability @@ -338,7 +332,9 @@ function prepareTestServerForIastInExpress (description, expressVersion, loadMid makeRequest = obj.makeRequest testDescription = obj.testDescription || testDescription } + testDescription = testDescription || `should have ${vulnerability} vulnerability` + it(testDescription, function (done) { this.timeout(5000) app = fn @@ -348,8 +344,8 @@ function prepareTestServerForIastInExpress (description, expressVersion, loadMid } function testThatRequestHasNoVulnerability (fn, vulnerability, makeRequest) { - let testDescription = `should not have ${vulnerability} vulnerability` - if (typeof fn === 'object') { + let testDescription + if (fn !== null && typeof fn === 'object') { const obj = fn fn = obj.fn vulnerability = obj.vulnerability @@ -357,6 +353,8 @@ function prepareTestServerForIastInExpress (description, expressVersion, loadMid testDescription = obj.testDescription || testDescription } + testDescription = testDescription || `should not have ${vulnerability} vulnerability` + it(testDescription, function (done) { app = fn checkNoVulnerabilityInRequest(vulnerability, config, done, makeRequest) diff --git a/packages/dd-trace/test/appsec/iast/vulnerability-formatter/index.spec.js b/packages/dd-trace/test/appsec/iast/vulnerability-formatter/index.spec.js index f005bdd7306..d77c5fb8e9b 100644 --- a/packages/dd-trace/test/appsec/iast/vulnerability-formatter/index.spec.js +++ b/packages/dd-trace/test/appsec/iast/vulnerability-formatter/index.spec.js @@ -5,6 +5,12 @@ const sensitiveHandler = require('../../../../src/appsec/iast/vulnerabilities-formatter/evidence-redaction/sensitive-handler') const { suite } = require('./resources/evidence-redaction-suite.json') +const excludedVulnerabilityTypes = ['XSS', 'EMAIL_HTML_INJECTION'] +const excludedTests = [ + 'Query with single quoted string literal and null source', // does not apply + 'Redacted source that needs to be truncated', // not implemented yet + 'CODE_INJECTION - Tainted range based redaction - with null source ' // does not apply +] function doTest (testCase, parameters) { let { description, input, expected } = testCase @@ -16,6 +22,14 @@ function doTest (testCase, parameters) { }) } + if (expected.vulnerabilities?.length && excludedVulnerabilityTypes.includes(expected.vulnerabilities[0].type)) { + return + } + + if (excludedTests.includes(description)) { + return + } + it(description, () => { const testInput = input.map(i => ( { @@ -63,7 +77,7 @@ function extractTestParameters (testCase) { describe('Vulnerability formatter', () => { describe('Vulnerability redaction', () => { - suite.filter(testCase => testCase.type === 'VULNERABILITIES' && testCase.input[0]?.type !== 'XSS') + suite.filter(testCase => testCase.type === 'VULNERABILITIES') .forEach((testCase) => { if (!testCase.parameters) { doTest(testCase) diff --git a/packages/dd-trace/test/appsec/iast/vulnerability-formatter/resources/evidence-redaction-suite.json b/packages/dd-trace/test/appsec/iast/vulnerability-formatter/resources/evidence-redaction-suite.json index 4f63b3d1f44..d40546b7328 100644 --- a/packages/dd-trace/test/appsec/iast/vulnerability-formatter/resources/evidence-redaction-suite.json +++ b/packages/dd-trace/test/appsec/iast/vulnerability-formatter/resources/evidence-redaction-suite.json @@ -8,6 +8,7 @@ "$1": [ "access_key_id", "accessKeyId", + "address", "apikey", "api_key", "apiToken", @@ -21,8 +22,11 @@ "consumer_key", "consumerSecret", "consumer_secret", + "email", "expirationToken", "expiration_token", + "lastname", + "mail", "pass", "passwd", "password", @@ -43,7 +47,10 @@ "sign", "signature", "signed", - "token" + "surname", + "token", + "user", + "username" ] }, "input": [ @@ -70,7 +77,8 @@ "glpat-xxxxxxxxxxxxxxxxxxxx", "-----BEGIN RSA PRIVATE KEY----- MIIEpAIBAAKCAQEAkVDOAMenPclQ7z5U3i3QYw4lQuijEyxnEgTXkk88L20moFBU 4vJkSguvUXrGzNiH+WMWWWTAXBTDdtOHApQJSdU0P4lY+0P3Lw3WeZaetPm583ac DlaCk9DaqPQnjpZ/9DLqmx1r5JYAZbCiuXWMA0lzJUOOniwt94BWCnz3+0LbrC7j NsiaC7cRc1kmj/Nmu8ydA4eop44tJMlaXb9nnUIxglUm0yL1NDOTzokTP03Fa7JW t46gMo6co751nYm43MwOb/cY0Uh6+i59czXuCs0hFpWyEkQJDjcQNXgy9ctI0R/J nBbQykSJG8C0cB9nsfwbtuRIQVrgoj65erlXawIDAQABAoIBAByGkTnj93eQilu8 j6phsfOP9k6RHloIMF+AJdUpyrXApoF344H9dSR38L187YOOyfpxshRwS7aHuOsd kPY3my8sNCp4ysfgSqio/b42jAcYsqERWocSAmYD7LiX3SAHeSy1xgoXF3Py4jcU Go1vfsGybHEXNurj304jmkBK0d83rYdYFNa58jY+6fCrt7b7SdxcjImvRbx0ByvB O/igAQxHLYZAVM+9eD8kHRt6nFkdllGkdynMPx82RllpjyZvxBm8hXeRCXvT78Ja 9aOx6YZLND6iLinAh2J+zFKTtl+iX8DD+39DMFEgLjgKJB84phux1h/2PP8RS2tp 5TqWy7ECgYEA+A8HEKKFTaYD4GQaiD+L4gOh2ZcLykdG8IIXRxzCPtv5VWKS2SCz WWyFoVRlV4b6q96PJwdS/6skbbWS98HIg3aqhOVaXyGxZHlzRgopE3OfRiDcf/Xd bO+Y7phH6h+hMBWpAAojJ+lWzGkg2DewCY0NjkUdOrFAZZWWLrQqGGMCgYEAlfe+ S3gXGqVk3ZyS4f8TyWrkKfVaRVa2KT0GGBJ8TNOB7xlf0oVmKCKSGoWbY5znt2e2 OTb6/zL0qzm1R9pNw5tUE5k/cCReZ20TpcHExoc+1prvmoCO8ToYMfGPOTBpRKBo Hdtx4xjBVe9omP6c/U8jfMDUL+cEKgvvjHUXv1kCgYEApTo1RYJLcoYjTOLAvYI+ ZYRv2SSAKPNDME4mvSpNxFr3gEVRdSkP7X+YnvY9LojtDXAIQEHjqgLQF/d69mZw bgir2it+/6DMrRUskDmSVK+OJsMavG0DWV1aq4ppVGxPDF1RHYKjGiGVvEBGLV8i daornlkw9/g64a86ws8kvusCgYBvnRs7//zyD/aqGUYYfUe0uKFnuPueb5LTzl8i u19XrnMeCLyQakhFxrUGmDm2QakTj1TH8GuOU9ZVOXX6LDeERa6lh4D3bZn1T/E3 hKd3OmFCR73cN6IrVxl60lXOMoGmWdwjnJd+dYYu9yfZ9mXRAX1f9AP4Qu+Oe6Ol 3d/2wQKBgQCgdA48bkRGFR/OqcGACNVQFcXQYvSabKOZkg303NH7p4pD8Ng6FDW+ r8r8+M/iMF9q7XvcX5pF8zgGk/MfHOdf9wWv7Uih7CIQzJLEs+OzNqx//Jn1EuV4 GBudByVPLqUDB5nvcDxTTsDP+gPFQtQ1mAWB1r18s9x4OioqvoV/6Q== -----END RSA PRIVATE KEY-----", "-----BEGIN OPENSSH PRIVATE KEY----- MIIEpAIBAAKCAQEAkVDOAMenPclQ7z5U3i3QYw4lQuijEyxnEgTXkk88L20moFBU 4vJkSguvUXrGzNiH+WMWWWTAXBTDdtOHApQJSdU0P4lY+0P3Lw3WeZaetPm583ac DlaCk9DaqPQnjpZ/9DLqmx1r5JYAZbCiuXWMA0lzJUOOniwt94BWCnz3+0LbrC7j NsiaC7cRc1kmj/Nmu8ydA4eop44tJMlaXb9nnUIxglUm0yL1NDOTzokTP03Fa7JW t46gMo6co751nYm43MwOb/cY0Uh6+i59czXuCs0hFpWyEkQJDjcQNXgy9ctI0R/J nBbQykSJG8C0cB9nsfwbtuRIQVrgoj65erlXawIDAQABAoIBAByGkTnj93eQilu8 j6phsfOP9k6RHloIMF+AJdUpyrXApoF344H9dSR38L187YOOyfpxshRwS7aHuOsd kPY3my8sNCp4ysfgSqio/b42jAcYsqERWocSAmYD7LiX3SAHeSy1xgoXF3Py4jcU Go1vfsGybHEXNurj304jmkBK0d83rYdYFNa58jY+6fCrt7b7SdxcjImvRbx0ByvB O/igAQxHLYZAVM+9eD8kHRt6nFkdllGkdynMPx82RllpjyZvxBm8hXeRCXvT78Ja 9aOx6YZLND6iLinAh2J+zFKTtl+iX8DD+39DMFEgLjgKJB84phux1h/2PP8RS2tp 5TqWy7ECgYEA+A8HEKKFTaYD4GQaiD+L4gOh2ZcLykdG8IIXRxzCPtv5VWKS2SCz WWyFoVRlV4b6q96PJwdS/6skbbWS98HIg3aqhOVaXyGxZHlzRgopE3OfRiDcf/Xd bO+Y7phH6h+hMBWpAAojJ+lWzGkg2DewCY0NjkUdOrFAZZWWLrQqGGMCgYEAlfe+ S3gXGqVk3ZyS4f8TyWrkKfVaRVa2KT0GGBJ8TNOB7xlf0oVmKCKSGoWbY5znt2e2 OTb6/zL0qzm1R9pNw5tUE5k/cCReZ20TpcHExoc+1prvmoCO8ToYMfGPOTBpRKBo Hdtx4xjBVe9omP6c/U8jfMDUL+cEKgvvjHUXv1kCgYEApTo1RYJLcoYjTOLAvYI+ ZYRv2SSAKPNDME4mvSpNxFr3gEVRdSkP7X+YnvY9LojtDXAIQEHjqgLQF/d69mZw bgir2it+/6DMrRUskDmSVK+OJsMavG0DWV1aq4ppVGxPDF1RHYKjGiGVvEBGLV8i daornlkw9/g64a86ws8kvusCgYBvnRs7//zyD/aqGUYYfUe0uKFnuPueb5LTzl8i u19XrnMeCLyQakhFxrUGmDm2QakTj1TH8GuOU9ZVOXX6LDeERa6lh4D3bZn1T/E3 hKd3OmFCR73cN6IrVxl60lXOMoGmWdwjnJd+dYYu9yfZ9mXRAX1f9AP4Qu+Oe6Ol 3d/2wQKBgQCgdA48bkRGFR/OqcGACNVQFcXQYvSabKOZkg303NH7p4pD8Ng6FDW+ r8r8+M/iMF9q7XvcX5pF8zgGk/MfHOdf9wWv7Uih7CIQzJLEs+OzNqx//Jn1EuV4 GBudByVPLqUDB5nvcDxTTsDP+gPFQtQ1mAWB1r18s9x4OioqvoV/6Q== -----END OPENSSH PRIVATE KEY-----", - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCRUM4Ax6c9yVDvPlTeLdBjDiVC6KMTLGcSBNeSTzwvbSagUFTi8mRKC69ResbM2If5YxZZZMBcFMN204cClAlJ1TQ/iVj7Q/cvDdZ5lp60+bnzdpwOVoKT0Nqo9CeOln/0MuqbHWvklgBlsKK5dYwDSXMlQ46eLC33gFYKfPf7QtusLuM2yJoLtxFzWSaP82a7zJ0Dh6inji0kyVpdv2edQjGCVSbTIvU0M5POiRM/TcVrsla3jqAyjpyjvnWdibjczA5v9xjRSHr6Ln1zNe4KzSEWlbISRAkONxA1eDL1y0jRH8mcFtDKRIkbwLRwH2ex/Bu25EhBWuCiPrl6uVdr" + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCRUM4Ax6c9yVDvPlTeLdBjDiVC6KMTLGcSBNeSTzwvbSagUFTi8mRKC69ResbM2If5YxZZZMBcFMN204cClAlJ1TQ/iVj7Q/cvDdZ5lp60+bnzdpwOVoKT0Nqo9CeOln/0MuqbHWvklgBlsKK5dYwDSXMlQ46eLC33gFYKfPf7QtusLuM2yJoLtxFzWSaP82a7zJ0Dh6inji0kyVpdv2edQjGCVSbTIvU0M5POiRM/TcVrsla3jqAyjpyjvnWdibjczA5v9xjRSHr6Ln1zNe4KzSEWlbISRAkONxA1eDL1y0jRH8mcFtDKRIkbwLRwH2ex/Bu25EhBWuCiPrl6uVdr", + "mail@to.net" ] }, "input": [ @@ -299,6 +307,54 @@ ] } }, + { + "type": "VULNERABILITIES", + "description": "Query with single quoted string literal and null source", + "input": [ + { + "type": "SQL_INJECTION", + "evidence": { + "value": "select * from users where username = 'user'", + "ranges": [ + { + "start": 38, + "end": 42, + "iinfo": { + "type": "http.request.body" + } + } + ] + } + } + ], + "expected": { + "sources": [ + { + "origin": "http.request.body" + } + ], + "vulnerabilities": [ + { + "type": "SQL_INJECTION", + "evidence": { + "valueParts": [ + { + "value": "select * from users where username = '" + }, + { + "redacted": true, + "source": 0, + "pattern": "****" + }, + { + "value": "'" + } + ] + } + } + ] + } + }, { "type": "VULNERABILITIES", "description": "$1 query with double quoted string literal $2", @@ -2694,7 +2750,7 @@ "end": 47, "iinfo": { "type": "http.request.parameter", - "parameterName": "email", + "parameterName": "param", "parameterValue": "' OR TRUE --" } } @@ -2706,7 +2762,7 @@ "sources": [ { "origin": "http.request.parameter", - "name": "email", + "name": "param", "value": "' OR TRUE --" } ], @@ -2850,10 +2906,17 @@ }, { "type": "VULNERABILITIES", - "description": "Tainted range based redaction ", + "description": "$1 - Tainted range based redaction ", + "parameters": { + "$1": [ + "XSS", + "CODE_INJECTION", + "EMAIL_HTML_INJECTION" + ] + }, "input": [ { - "type": "XSS", + "type": "$1", "evidence": { "value": "this could be a super long text, so we need to reduce it before send it to the backend. This redaction strategy applies to XSS vulnerability but can be extended to future ones", "ranges": [ @@ -2880,7 +2943,7 @@ ], "vulnerabilities": [ { - "type": "XSS", + "type": "$1", "evidence": { "valueParts": [ { @@ -2901,10 +2964,17 @@ }, { "type": "VULNERABILITIES", - "description": "Tainted range based redaction - with redactable source ", + "description": "$1 - Tainted range based redaction - with redactable source ", + "parameters": { + "$1": [ + "XSS", + "CODE_INJECTION", + "EMAIL_HTML_INJECTION" + ] + }, "input": [ { - "type": "XSS", + "type": "$1", "evidence": { "value": "this could be a super long text, so we need to reduce it before send it to the backend. This redaction strategy applies to XSS vulnerability but can be extended to future ones", "ranges": [ @@ -2932,7 +3002,7 @@ ], "vulnerabilities": [ { - "type": "XSS", + "type": "$1", "evidence": { "valueParts": [ { @@ -2954,10 +3024,17 @@ }, { "type": "VULNERABILITIES", - "description": "Tainted range based redaction - with null source ", + "description": "$1 - Tainted range based redaction - with null source ", + "parameters": { + "$1": [ + "XSS", + "CODE_INJECTION", + "EMAIL_HTML_INJECTION" + ] + }, "input": [ { - "type": "XSS", + "type": "$1", "evidence": { "value": "this could be a super long text, so we need to reduce it before send it to the backend. This redaction strategy applies to XSS vulnerability but can be extended to future ones", "ranges": [ @@ -2980,7 +3057,7 @@ ], "vulnerabilities": [ { - "type": "XSS", + "type": "$1", "evidence": { "valueParts": [ { @@ -3001,10 +3078,17 @@ }, { "type": "VULNERABILITIES", - "description": "Tainted range based redaction - multiple ranges", + "description": "$1 - Tainted range based redaction - multiple ranges", + "parameters": { + "$1": [ + "XSS", + "CODE_INJECTION", + "EMAIL_HTML_INJECTION" + ] + }, "input": [ { - "type": "XSS", + "type": "$1", "evidence": { "value": "this could be a super long text, so we need to reduce it before send it to the backend. This redaction strategy applies to XSS vulnerability but can be extended to future ones", "ranges": [ @@ -3045,7 +3129,7 @@ ], "vulnerabilities": [ { - "type": "XSS", + "type": "$1", "evidence": { "valueParts": [ { @@ -3073,10 +3157,17 @@ }, { "type": "VULNERABILITIES", - "description": "Tainted range based redaction - first range at the beginning ", + "description": "$1 - Tainted range based redaction - first range at the beginning ", + "parameters": { + "$1": [ + "XSS", + "CODE_INJECTION", + "EMAIL_HTML_INJECTION" + ] + }, "input": [ { - "type": "XSS", + "type": "$1", "evidence": { "value": "this could be a super long text, so we need to reduce it before send it to the backend. This redaction strategy applies to XSS vulnerability but can be extended to future ones", "ranges": [ @@ -3117,7 +3208,7 @@ ], "vulnerabilities": [ { - "type": "XSS", + "type": "$1", "evidence": { "valueParts": [ { @@ -3142,10 +3233,17 @@ }, { "type": "VULNERABILITIES", - "description": "Tainted range based redaction - last range at the end ", + "description": "$1 - Tainted range based redaction - last range at the end ", + "parameters": { + "$1": [ + "XSS", + "CODE_INJECTION", + "EMAIL_HTML_INJECTION" + ] + }, "input": [ { - "type": "XSS", + "type": "$1", "evidence": { "value": "this could be a super long text, so we need to reduce it before send it to the backend. This redaction strategy applies to XSS", "ranges": [ @@ -3186,7 +3284,7 @@ ], "vulnerabilities": [ { - "type": "XSS", + "type": "$1", "evidence": { "valueParts": [ { @@ -3208,10 +3306,17 @@ }, { "type": "VULNERABILITIES", - "description": "Tainted range based redaction - whole text ", + "description": "$1 - Tainted range based redaction - whole text ", + "parameters": { + "$1": [ + "XSS", + "CODE_INJECTION", + "EMAIL_HTML_INJECTION" + ] + }, "input": [ { - "type": "XSS", + "type": "$1", "evidence": { "value": "this could be a super long text, so we need to reduce it before send it to the backend. This redaction strategy applies to XSS", "ranges": [ @@ -3238,7 +3343,7 @@ ], "vulnerabilities": [ { - "type": "XSS", + "type": "$1", "evidence": { "valueParts": [ { @@ -3403,7 +3508,7 @@ "end": 4, "iinfo": { "type": "http.request.parameter", - "parameterName": "username", + "parameterName": "param", "parameterValue": "PREFIX_user" } } @@ -3415,7 +3520,7 @@ "end": 4, "iinfo": { "type": "http.request.parameter", - "parameterName": "username", + "parameterName": "param", "parameterValue": "PREFIX_user" } } @@ -3428,7 +3533,7 @@ "sources": [ { "origin": "http.request.parameter", - "name": "username", + "name": "param", "redacted": true, "pattern": "abcdefghijk" } @@ -3613,6 +3718,113 @@ ] } }, + { + "type": "VULNERABILITIES", + "description": "Redacted source that needs to be truncated", + "input": [ + { + "type": "SQL_INJECTION", + "evidence": { + "value": "select * from users where username = 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.Sed ut perspiciatis unde omnis iste natus error sit voluptatem ac'", + "ranges": [ + { + "start": 26, + "end": 549, + "iinfo": { + "type": "http.request.parameter", + "parameterName": "clause", + "parameterValue": "username = 'Lorem%20ipsum%20dolor%20sit%20amet,%20consectetur%20adipiscing%20elit,%20sed%20do%20eiusmod%20tempor%20incididunt%20ut%20labore%20et%20dolore%20magna%20aliqua.%20Ut%20enim%20ad%20minim%20veniam,%20quis%20nostrud%20exercitation%20ullamco%20laboris%20nisi%20ut%20aliquip%20ex%20ea%20commodo%20consequat.%20Duis%20aute%20irure%20dolor%20in%20reprehenderit%20in%20voluptate%20velit%20esse%20cillum%20dolore%20eu%20fugiat%20nulla%20pariatur.%20Excepteur%20sint%20occaecat%20cupidatat%20non%20proident,%20sunt%20in%20culpa%20qui%20officia%20deserunt%20mollit%20anim%20id%20est%20laborum.Sed%20ut%20perspiciatis%20unde%20omnis%20iste%20natus%20error%20sit%20voluptatem%20ac'" + } + } + ] + } + } + ], + "expected": { + "sources": [ + { + "origin": "http.request.parameter", + "name": "clause", + "redacted": true, + "pattern": "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789ab", + "truncated": "right" + } + ], + "vulnerabilities": [ + { + "type": "SQL_INJECTION", + "evidence": { + "valueParts": [ + { + "value": "select * from users where " + }, + { + "source": 0, + "value": "username = '" + }, + { + "source": 0, + "redacted": true, + "truncated": "right", + "pattern": "**********************************************************************************************************************************************************************************************************************************************************" + }, + { + "source": 0, + "value": "'" + } + ] + } + } + ] + } + }, + { + "type": "VULNERABILITIES", + "description": "No redacted that needs to be truncated - whole text", + "input": [ + { + "type": "XSS", + "evidence": { + "value": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.Sed ut perspiciatis unde omnis iste natus error sit voluptatem ac", + "ranges": [ + { + "start": 0, + "end": 510, + "iinfo": { + "type": "http.request.parameter", + "parameterName": "text", + "parameterValue": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.Sed ut perspiciatis unde omnis iste natus error sit voluptatem ac" + } + } + ] + } + } + ], + "expected": { + "sources": [ + { + "origin": "http.request.parameter", + "name": "text", + "truncated": "right", + "value": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure do" + } + ], + "vulnerabilities": [ + { + "type": "XSS", + "evidence": { + "valueParts": [ + { + "source": 0, + "truncated": "right", + "value": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure do" + } + ] + } + } + ] + } + }, { "type": "VULNERABILITIES", "description": "Header injection without sensitive data", @@ -3969,6 +4181,54 @@ } ] } + }, + { + "type": "VULNERABILITIES", + "description": "Hardcoded password with sensitive data in the variable name", + "input": [ + { + "type": "HARDCODED_PASSWORD", + "evidence": { + "value": "gho_apasswapasswapasswapasswapasswapassw" + } + } + ], + "expected": { + "vulnerabilities": [ + { + "type": "HARDCODED_PASSWORD", + "evidence": { + "valueParts": [ + { + "redacted": true + } + ] + } + } + ] + } + }, + { + "type": "VULNERABILITIES", + "description": "Hardcoded password without sensitive data in the variable name", + "input": [ + { + "type": "HARDCODED_PASSWORD", + "evidence": { + "value": "this_is_a_password" + } + } + ], + "expected": { + "vulnerabilities": [ + { + "type": "HARDCODED_PASSWORD", + "evidence": { + "value": "this_is_a_password" + } + } + ] + } } ] } diff --git a/packages/dd-trace/test/appsec/iast/vulnerability-formatter/utils.spec.js b/packages/dd-trace/test/appsec/iast/vulnerability-formatter/utils.spec.js index f37696f6d80..19e4f3eb70c 100644 --- a/packages/dd-trace/test/appsec/iast/vulnerability-formatter/utils.spec.js +++ b/packages/dd-trace/test/appsec/iast/vulnerability-formatter/utils.spec.js @@ -81,7 +81,7 @@ describe('test vulnerabiilty-formatter utils', () => { const range = { start: 0, end: 5, iinfo } - const { value, ranges } = stringifyWithRanges(src, { '0': [range] }) + const { value, ranges } = stringifyWithRanges(src, { 0: [range] }) expect(value).to.be.equal(JSON.stringify(src, null, 2)) @@ -108,8 +108,8 @@ describe('test vulnerabiilty-formatter utils', () => { const iinfo = { type: 'TEST' } const objRanges = { '0.key': [{ start: 0, end: 8, iinfo }], - '1': [{ start: 0, end: 8, iinfo }], - '2': [ + 1: [{ start: 0, end: 8, iinfo }], + 2: [ { start: 3, end: 7, iinfo }, { start: 8, end: 10, iinfo } ], @@ -128,6 +128,48 @@ describe('test vulnerabiilty-formatter utils', () => { { start: 110, end: 112, iinfo } ]) }) + + it('Very deep object', () => { + const deepObject = [...Array(100).keys()] + .reduce((obj) => ({ key: 'value', obj: obj || {} }), null) + + const expectedObject = [...Array(10).keys()] + .reduce((obj) => ({ key: 'value', obj: obj || {} }), null) + + const iinfo = { type: 'TEST' } + const deepestKey = '0'.concat('.obj'.repeat(9)).concat('.key') + const objRanges = { + [deepestKey]: [{ start: 0, end: 5, iinfo }] + } + + const { value, ranges } = stringifyWithRanges([deepObject], objRanges) + + expect(value).to.be.equal(JSON.stringify([expectedObject], null, 2)) + + expect(ranges).to.be.deep.equal([ + { start: 477, end: 482, iinfo } + ]) + }) + + it('Circular reference object', () => { + const circularObject = { key: 'value' } + circularObject.obj = circularObject + + const expectedObject = [{ key: 'value' }] + + const iinfo = { type: 'TEST' } + const objRanges = { + '0.key': [{ start: 0, end: 5, iinfo }] + } + + const { value, ranges } = stringifyWithRanges([circularObject], objRanges) + + expect(value).to.be.equal(JSON.stringify(expectedObject, null, 2)) + + expect(ranges).to.be.deep.equal([ + { start: 18, end: 23, iinfo } + ]) + }) }) describe('loadSensitiveRanges = true', () => { @@ -277,7 +319,7 @@ describe('test vulnerabiilty-formatter utils', () => { start: 0, end: 5, iinfo } - const { value, ranges, sensitiveRanges } = stringifyWithRanges(src, { '0': [range] }, true) + const { value, ranges, sensitiveRanges } = stringifyWithRanges(src, { 0: [range] }, true) expect(value).to.be.equal(JSON.stringify(src, null, 2)) @@ -308,8 +350,8 @@ describe('test vulnerabiilty-formatter utils', () => { const iinfo = { type: 'TEST' } const objRanges = { '0.key': [{ start: 0, end: 8, iinfo }], - '1': [{ start: 0, end: 8, iinfo }], - '2': [ + 1: [{ start: 0, end: 8, iinfo }], + 2: [ { start: 3, end: 7, iinfo }, { start: 8, end: 10, iinfo } ], diff --git a/packages/dd-trace/test/appsec/iast/vulnerability-reporter.spec.js b/packages/dd-trace/test/appsec/iast/vulnerability-reporter.spec.js index bab1f9aca53..f498ef6e122 100644 --- a/packages/dd-trace/test/appsec/iast/vulnerability-reporter.spec.js +++ b/packages/dd-trace/test/appsec/iast/vulnerability-reporter.spec.js @@ -1,6 +1,9 @@ const { addVulnerability, sendVulnerabilities, clearCache, start, stop } = require('../../../src/appsec/iast/vulnerability-reporter') const VulnerabilityAnalyzer = require('../../../../dd-trace/src/appsec/iast/analyzers/vulnerability-analyzer') +const appsecStandalone = require('../../../src/appsec/standalone') +const { APPSEC_PROPAGATION_KEY } = require('../../../src/constants') + describe('vulnerability-reporter', () => { let vulnerabilityAnalyzer @@ -9,6 +12,8 @@ describe('vulnerability-reporter', () => { vulnerabilityAnalyzer = new VulnerabilityAnalyzer('ANALYZER_TYPE') }) + afterEach(sinon.restore) + describe('addVulnerability', () => { it('should not break with invalid input', () => { addVulnerability() @@ -77,6 +82,7 @@ describe('vulnerability-reporter', () => { describe('without rootSpan', () => { let fakeTracer let onTheFlySpan + beforeEach(() => { onTheFlySpan = { finish: sinon.spy(), @@ -98,10 +104,12 @@ describe('vulnerability-reporter', () => { } }, fakeTracer) }) + afterEach(() => { sinon.restore() stop() }) + it('should create span on the fly', () => { const vulnerability = vulnerabilityAnalyzer._createVulnerability('INSECURE_HASHING', { value: 'sha1' }, undefined, @@ -118,6 +126,7 @@ describe('vulnerability-reporter', () => { }) expect(onTheFlySpan.finish).to.have.been.calledOnce }) + it('should update spanId in vulnerability\'s location with the new id from created span', () => { const vulnerability = vulnerabilityAnalyzer._createVulnerability('INSECURE_HASHING', { value: 'sha1' }, undefined, @@ -130,10 +139,13 @@ describe('vulnerability-reporter', () => { describe('sendVulnerabilities', () => { let span + let context beforeEach(() => { + context = { _trace: { tags: {} } } span = { - addTags: sinon.stub() + addTags: sinon.stub(), + context: sinon.stub().returns(context) } start({ iast: { @@ -141,6 +153,7 @@ describe('vulnerability-reporter', () => { } }) }) + afterEach(() => { sinon.restore() stop() @@ -378,6 +391,40 @@ describe('vulnerability-reporter', () => { '{"spanId":888,"path":"filename.js","line":88}}]}' }) }) + + it('should add _dd.p.appsec trace tag with standalone enabled', () => { + appsecStandalone.configure({ appsec: { standalone: { enabled: true } } }) + const iastContext = { rootSpan: span } + addVulnerability(iastContext, + vulnerabilityAnalyzer._createVulnerability('INSECURE_HASHING', { value: 'sha1' }, 999)) + + sendVulnerabilities(iastContext.vulnerabilities, span) + + expect(span.addTags).to.have.been.calledOnceWithExactly({ + 'manual.keep': 'true', + '_dd.iast.json': '{"sources":[],"vulnerabilities":[{"type":"INSECURE_HASHING","hash":3254801297,' + + '"evidence":{"value":"sha1"},"location":{"spanId":999}}]}' + }) + + expect(span.context()._trace.tags).to.have.property(APPSEC_PROPAGATION_KEY) + }) + + it('should not add _dd.p.appsec trace tag with standalone disabled', () => { + appsecStandalone.configure({ appsec: {} }) + const iastContext = { rootSpan: span } + addVulnerability(iastContext, + vulnerabilityAnalyzer._createVulnerability('INSECURE_HASHING', { value: 'sha1' }, 999)) + + sendVulnerabilities(iastContext.vulnerabilities, span) + + expect(span.addTags).to.have.been.calledOnceWithExactly({ + 'manual.keep': 'true', + '_dd.iast.json': '{"sources":[],"vulnerabilities":[{"type":"INSECURE_HASHING","hash":3254801297,' + + '"evidence":{"value":"sha1"},"location":{"spanId":999}}]}' + }) + + expect(span.context()._trace.tags).to.not.have.property(APPSEC_PROPAGATION_KEY) + }) }) describe('clearCache', () => { diff --git a/packages/dd-trace/test/appsec/index.body-parser.plugin.spec.js b/packages/dd-trace/test/appsec/index.body-parser.plugin.spec.js index f48d330279c..458a69ee97d 100644 --- a/packages/dd-trace/test/appsec/index.body-parser.plugin.spec.js +++ b/packages/dd-trace/test/appsec/index.body-parser.plugin.spec.js @@ -1,7 +1,6 @@ 'use strict' const axios = require('axios') -const getPort = require('get-port') const path = require('path') const agent = require('../plugins/agent') const appsec = require('../../src/appsec') @@ -27,11 +26,9 @@ withVersions('body-parser', 'body-parser', version => { res.end('DONE') }) - getPort().then(newPort => { - port = newPort - server = app.listen(port, () => { - done() - }) + server = app.listen(port, () => { + port = server.address().port + done() }) }) diff --git a/packages/dd-trace/test/appsec/index.cookie-parser.plugin.spec.js b/packages/dd-trace/test/appsec/index.cookie-parser.plugin.spec.js index fbc49565e2a..fed6bbcbf45 100644 --- a/packages/dd-trace/test/appsec/index.cookie-parser.plugin.spec.js +++ b/packages/dd-trace/test/appsec/index.cookie-parser.plugin.spec.js @@ -2,7 +2,6 @@ const { assert } = require('chai') const axios = require('axios') -const getPort = require('get-port') const path = require('path') const agent = require('../plugins/agent') const appsec = require('../../src/appsec') @@ -28,11 +27,9 @@ withVersions('cookie-parser', 'cookie-parser', version => { res.end('DONE') }) - getPort().then(newPort => { - port = newPort - server = app.listen(port, () => { - done() - }) + server = app.listen(port, () => { + port = server.address().port + done() }) }) diff --git a/packages/dd-trace/test/appsec/index.express.plugin.spec.js b/packages/dd-trace/test/appsec/index.express.plugin.spec.js index 9b1d2ea52f8..c38d496623b 100644 --- a/packages/dd-trace/test/appsec/index.express.plugin.spec.js +++ b/packages/dd-trace/test/appsec/index.express.plugin.spec.js @@ -1,6 +1,7 @@ 'use strict' -const axios = require('axios') +const Axios = require('axios') +const { assert } = require('chai') const getPort = require('get-port') const path = require('path') const agent = require('../plugins/agent') @@ -10,8 +11,8 @@ const { json } = require('../../src/appsec/blocked_templates') const zlib = require('zlib') withVersions('express', 'express', version => { - describe('Suspicious request blocking - query', () => { - let port, server, requestBody + describe('Suspicious request blocking - path parameters', () => { + let server, paramCallbackSpy, axios before(() => { return agent.load(['express', 'http'], { client: false }) @@ -19,23 +20,39 @@ withVersions('express', 'express', version => { before((done) => { const express = require('../../../../versions/express').get() - const bodyParser = require('../../../../versions/body-parser').get() const app = express() - app.use(bodyParser.json()) - app.get('/', (req, res) => { - requestBody() - res.end('DONE') + app.get('/multiple-path-params/:parameter1/:parameter2', (req, res) => { + res.send('DONE') }) - app.post('/', (req, res) => { - res.end('DONE') + const nestedRouter = express.Router({ mergeParams: true }) + nestedRouter.get('/:nestedDuplicatedParameter', (req, res) => { + res.send('DONE') + }) + + app.use('/nested/:nestedDuplicatedParameter', nestedRouter) + + app.get('/callback-path-param/:callbackedParameter', (req, res) => { + res.send('DONE') }) - getPort().then(newPort => { - port = newPort + const paramCallback = (req, res, next) => { + next() + } + + paramCallbackSpy = sinon.spy(paramCallback) + + app.param(() => { + return paramCallbackSpy + }) + + app.param('callbackedParameter') + + getPort().then((port) => { server = app.listen(port, () => { + axios = Axios.create({ baseURL: `http://localhost:${port}` }) done() }) }) @@ -46,86 +63,330 @@ withVersions('express', 'express', version => { return agent.close({ ritmReset: false }) }) - describe('Blocking', () => { - beforeEach(async () => { - requestBody = sinon.stub() - appsec.enable(new Config({ appsec: { enabled: true, rules: path.join(__dirname, 'express-rules.json') } })) + beforeEach(async () => { + appsec.enable(new Config({ + appsec: { + enabled: true, + rules: path.join(__dirname, 'express-rules.json') + } + })) + }) + + afterEach(() => { + appsec.disable() + sinon.reset() + }) + + describe('route with multiple path parameters', () => { + it('should not block the request when attack is not detected', async () => { + const res = await axios.get('/multiple-path-params/safe_param/safe_param') + + assert.equal(res.status, 200) + assert.equal(res.data, 'DONE') }) - afterEach(() => { - appsec.disable() + it('should block the request when attack is detected in both parameters', async () => { + try { + await axios.get('/multiple-path-params/testattack/testattack') + + return Promise.reject(new Error('Request should not return 200')) + } catch (e) { + assert.equal(e.response.status, 403) + assert.deepEqual(e.response.data, JSON.parse(json)) + } }) - it('should not block the request without an attack', async () => { - const res = await axios.get(`http://localhost:${port}/?key=value`) + it('should block the request when attack is detected in the first parameter', async () => { + try { + await axios.get('/multiple-path-params/testattack/safe_param') + + return Promise.reject(new Error('Request should not return 200')) + } catch (e) { + assert.equal(e.response.status, 403) + assert.deepEqual(e.response.data, JSON.parse(json)) + } + }) + + it('should block the request when attack is detected in the second parameter', async () => { + try { + await axios.get('/multiple-path-params/safe_param/testattack') + + return Promise.reject(new Error('Request should not return 200')) + } catch (e) { + assert.equal(e.response.status, 403) + assert.deepEqual(e.response.data, JSON.parse(json)) + } + }) + }) + + describe('nested routers', () => { + it('should not block the request when attack is not detected', async () => { + const res = await axios.get('/nested/safe_param/safe_param') - expect(requestBody).to.be.calledOnce - expect(res.data).to.be.equal('DONE') + assert.equal(res.status, 200) + assert.equal(res.data, 'DONE') + }) + + it('should block the request when attack is detected in the nested paremeter', async () => { + try { + await axios.get('/nested/safe_param/testattack') + + return Promise.reject(new Error('Request should not return 200')) + } catch (e) { + assert.equal(e.response.status, 403) + assert.deepEqual(e.response.data, JSON.parse(json)) + } + }) + + it('should block the request when attack is detected in the parent paremeter', async () => { + try { + await axios.get('/nested/testattack/safe_param') + + return Promise.reject(new Error('Request should not return 200')) + } catch (e) { + assert.equal(e.response.status, 403) + assert.deepEqual(e.response.data, JSON.parse(json)) + } + }) + + it('should block the request when attack is detected both parameters', async () => { + try { + await axios.get('/nested/testattack/testattack') + + return Promise.reject(new Error('Request should not return 200')) + } catch (e) { + assert.equal(e.response.status, 403) + assert.deepEqual(e.response.data, JSON.parse(json)) + } + }) + }) + + describe('path parameter callback', () => { + it('should not block the request when attack is not detected', async () => { + const res = await axios.get('/callback-path-param/safe_param') + assert.equal(res.status, 200) + assert.equal(res.data, 'DONE') + sinon.assert.calledOnce(paramCallbackSpy) }) it('should block the request when attack is detected', async () => { try { - await axios.get(`http://localhost:${port}/?key=testattack`) + await axios.get('/callback-path-param/testattack') return Promise.reject(new Error('Request should not return 200')) } catch (e) { - expect(e.response.status).to.be.equals(403) - expect(e.response.data).to.be.deep.equal(JSON.parse(json)) - expect(requestBody).not.to.be.called + assert.equal(e.response.status, 403) + assert.deepEqual(e.response.data, JSON.parse(json)) + sinon.assert.notCalled(paramCallbackSpy) } }) }) + }) - describe('Api Security', () => { - let config + describe('Suspicious request blocking - query', () => { + let server, requestBody, axios - beforeEach(() => { - config = new Config({ - appsec: { - enabled: true, - rules: path.join(__dirname, 'api_security_rules.json'), - apiSecurity: { - enabled: true, - requestSampling: 1.0 - } - } + before(() => { + return agent.load(['express', 'http'], { client: false }) + }) + + before((done) => { + const express = require('../../../../versions/express').get() + + const app = express() + + app.get('/', (req, res) => { + requestBody() + res.end('DONE') + }) + + getPort().then((port) => { + server = app.listen(port, () => { + axios = Axios.create({ baseURL: `http://localhost:${port}` }) + done() + }) + }) + }) + + after(() => { + server.close() + return agent.close({ ritmReset: false }) + }) + + beforeEach(async () => { + requestBody = sinon.stub() + appsec.enable(new Config({ + appsec: { + enabled: true, + rules: path.join(__dirname, 'express-rules.json') + } + })) + }) + + afterEach(() => { + appsec.disable() + }) + + it('should not block the request without an attack', async () => { + const res = await axios.get('/?key=value') + + assert.equal(res.status, 200) + assert.equal(res.data, 'DONE') + sinon.assert.calledOnce(requestBody) + }) + + it('should block the request when attack is detected', async () => { + try { + await axios.get('/?key=testattack') + + return Promise.reject(new Error('Request should not return 200')) + } catch (e) { + assert.equal(e.response.status, 403) + assert.deepEqual(e.response.data, JSON.parse(json)) + sinon.assert.notCalled(requestBody) + } + }) + }) + + describe('Api Security', () => { + let config, server, axios + + before(() => { + return agent.load(['express', 'http'], { client: false }) + }) + + before((done) => { + const express = require('../../../../versions/express').get() + const bodyParser = require('../../../../versions/body-parser').get() + + const app = express() + app.use(bodyParser.json()) + + app.post('/', (req, res) => { + res.send('DONE') + }) + + app.post('/sendjson', (req, res) => { + res.send({ sendResKey: 'sendResValue' }) + }) + + app.post('/jsonp', (req, res) => { + res.jsonp({ jsonpResKey: 'jsonpResValue' }) + }) + + app.post('/json', (req, res) => { + res.jsonp({ jsonResKey: 'jsonResValue' }) + }) + + getPort().then((port) => { + server = app.listen(port, () => { + axios = Axios.create({ baseURL: `http://localhost:${port}` }) + done() }) }) + }) - afterEach(() => { - appsec.disable() + after(() => { + server.close() + return agent.close({ ritmReset: false }) + }) + + beforeEach(() => { + config = new Config({ + appsec: { + enabled: true, + rules: path.join(__dirname, 'api_security_rules.json'), + apiSecurity: { + enabled: true + } + } }) + }) + + afterEach(() => { + appsec.disable() + }) - it('should get the schema', async () => { + describe('with requestSampling 1.0', () => { + beforeEach(() => { + config.appsec.apiSecurity.requestSampling = 1.0 appsec.enable(config) + }) + + function formatSchema (body) { + return zlib.gzipSync(JSON.stringify(body)).toString('base64') + } + + it('should get the request body schema', async () => { + const expectedRequestBodySchema = formatSchema([{ key: [8] }]) - const expectedSchema = zlib.gzipSync(JSON.stringify([{ 'key': [8] }])).toString('base64') - const res = await axios.post(`http://localhost:${port}/`, { key: 'value' }) + const res = await axios.post('/', { key: 'value' }) await agent.use((traces) => { const span = traces[0][0] - expect(span.meta).to.haveOwnProperty('_dd.appsec.s.req.body') - expect(span.meta['_dd.appsec.s.req.body']).to.be.equal(expectedSchema) + assert.property(span.meta, '_dd.appsec.s.req.body') + assert.notProperty(span.meta, '_dd.appsec.s.res.body') + assert.equal(span.meta['_dd.appsec.s.req.body'], expectedRequestBodySchema) }) - expect(res.status).to.be.equal(200) - expect(res.data).to.be.equal('DONE') + assert.equal(res.status, 200) + assert.equal(res.data, 'DONE') }) - it('should not get the schema', async () => { - config.appsec.apiSecurity.requestSampling = 0 - appsec.enable(config) + it('should get the response body schema with res.send method with object', async () => { + const expectedResponseBodySchema = formatSchema([{ sendResKey: [8] }]) + const res = await axios.post('/sendjson', { key: 'value' }) + + await agent.use((traces) => { + const span = traces[0][0] + assert.equal(span.meta['_dd.appsec.s.res.body'], expectedResponseBodySchema) + }) + + assert.equal(res.status, 200) + assert.deepEqual(res.data, { sendResKey: 'sendResValue' }) + }) + + it('should get the response body schema with res.json method', async () => { + const expectedResponseBodySchema = formatSchema([{ jsonResKey: [8] }]) + const res = await axios.post('/json', { key: 'value' }) + + await agent.use((traces) => { + const span = traces[0][0] + assert.equal(span.meta['_dd.appsec.s.res.body'], expectedResponseBodySchema) + }) - const res = await axios.post(`http://localhost:${port}/`, { key: 'value' }) + assert.equal(res.status, 200) + assert.deepEqual(res.data, { jsonResKey: 'jsonResValue' }) + }) + + it('should get the response body schema with res.jsonp method', async () => { + const expectedResponseBodySchema = formatSchema([{ jsonpResKey: [8] }]) + const res = await axios.post('/jsonp', { key: 'value' }) await agent.use((traces) => { const span = traces[0][0] - expect(span.meta).not.to.haveOwnProperty('_dd.appsec.s.req.body') + assert.equal(span.meta['_dd.appsec.s.res.body'], expectedResponseBodySchema) }) - expect(res.status).to.be.equal(200) - expect(res.data).to.be.equal('DONE') + assert.equal(res.status, 200) + assert.deepEqual(res.data, { jsonpResKey: 'jsonpResValue' }) + }) + }) + + it('should not get the schema', async () => { + config.appsec.apiSecurity.requestSampling = 0 + appsec.enable(config) + + const res = await axios.post('/', { key: 'value' }) + + await agent.use((traces) => { + const span = traces[0][0] + assert.notProperty(span.meta, '_dd.appsec.s.req.body') + assert.notProperty(span.meta, '_dd.appsec.s.res.body') }) + + assert.equal(res.status, 200) + assert.equal(res.data, 'DONE') }) }) }) diff --git a/packages/dd-trace/test/appsec/index.next.plugin.spec.js b/packages/dd-trace/test/appsec/index.next.plugin.spec.js index 560b55eb7c8..38cac8f375c 100644 --- a/packages/dd-trace/test/appsec/index.next.plugin.spec.js +++ b/packages/dd-trace/test/appsec/index.next.plugin.spec.js @@ -8,17 +8,23 @@ const { writeFileSync } = require('fs') const { satisfies } = require('semver') const path = require('path') -const { DD_MAJOR } = require('../../../../version') +const { DD_MAJOR, NODE_MAJOR } = require('../../../../version') const agent = require('../plugins/agent') +const BUILD_COMMAND = NODE_MAJOR < 18 + ? 'yarn exec next build' + : 'NODE_OPTIONS=--openssl-legacy-provider yarn exec next build' +let VERSIONS_TO_TEST = NODE_MAJOR < 18 ? '>=11.1 <13.2' : '>=11.1' +VERSIONS_TO_TEST = DD_MAJOR >= 4 ? VERSIONS_TO_TEST : '>=9.5 <11.1' + describe('test suite', () => { let server let port const satisfiesStandalone = version => satisfies(version, '>=12.0.0') - withVersions('next', 'next', DD_MAJOR >= 4 && '>=11', version => { - const realVersion = require(`${__dirname}/../../../../versions/next@${version}`).version() + withVersions('next', 'next', VERSIONS_TO_TEST, version => { + const realVersion = require(`../../../../versions/next@${version}`).version() function initApp (appName) { const appDir = path.join(__dirname, 'next', appName) @@ -28,7 +34,7 @@ describe('test suite', () => { const cwd = appDir - const pkg = require(`${__dirname}/../../../../versions/next@${version}/package.json`) + const pkg = require(`../../../../versions/next@${version}/package.json`) if (realVersion.startsWith('10')) { return this.skip() // TODO: Figure out why 10.x tests fail. @@ -45,10 +51,14 @@ describe('test suite', () => { // installing here for standalone purposes, copying `nodules` above was not generating the server file properly // if there is a way to re-use nodules from somewhere in the versions folder, this `execSync` will be reverted - execSync('yarn install', { cwd }) + try { + execSync('yarn install', { cwd }) + } catch (e) { // retry in case of error from registry + execSync('yarn install', { cwd }) + } // building in-process makes tests fail for an unknown reason - execSync('yarn exec next build', { + execSync(BUILD_COMMAND, { cwd, env: { ...process.env, @@ -150,6 +160,25 @@ describe('test suite', () => { }) } + function getFindBodyThreatMethod (done) { + return function findBodyThreat (traces) { + let attackFound = false + + traces.forEach(trace => { + trace.forEach(span => { + if (span.meta['_dd.appsec.json']) { + attackFound = true + } + }) + }) + + if (attackFound) { + agent.unsubscribe(findBodyThreat) + done() + } + } + } + tests.forEach(({ appName, serverPath }) => { describe(`should detect threats in ${appName}`, () => { initApp(appName) @@ -159,22 +188,7 @@ describe('test suite', () => { it('in request body', function (done) { this.timeout(5000) - function findBodyThreat (traces) { - let attackFound = false - - traces.forEach(trace => { - trace.forEach(span => { - if (span.meta['_dd.appsec.json']) { - attackFound = true - } - }) - }) - - if (attackFound) { - agent.unsubscribe(findBodyThreat) - done() - } - } + const findBodyThreat = getFindBodyThreatMethod(done) agent.subscribe(findBodyThreat) axios @@ -183,27 +197,26 @@ describe('test suite', () => { }).catch(e => { done(e) }) }) - if (appName === 'app-dir') { - it('in request body with .text() function', function (done) { - this.timeout(5000) + it('in form data body', function (done) { + this.timeout(5000) - function findBodyThreat (traces) { - let attackFound = false + const findBodyThreat = getFindBodyThreatMethod(done) - traces.forEach(trace => { - trace.forEach(span => { - if (span.meta['_dd.appsec.json']) { - attackFound = true - } - }) - }) + agent.subscribe(findBodyThreat) - if (attackFound) { - agent.unsubscribe(findBodyThreat) - done() - } - } + axios + .post(`http://127.0.0.1:${port}/api/test-formdata`, new URLSearchParams({ + key: 'testattack' + })).catch(e => { + done(e) + }) + }) + if (appName === 'app-dir') { + it('in request body with .text() function', function (done) { + this.timeout(5000) + + const findBodyThreat = getFindBodyThreatMethod(done) agent.subscribe(findBodyThreat) axios .post(`http://127.0.0.1:${port}/api/test-text`, { @@ -217,20 +230,7 @@ describe('test suite', () => { it('in request query', function (done) { this.timeout(5000) - function findBodyThreat (traces) { - let attackFound = false - traces.forEach(trace => { - trace.forEach(span => { - if (span.meta['_dd.appsec.json']) { - attackFound = true - } - }) - }) - if (attackFound) { - agent.unsubscribe(findBodyThreat) - done() - } - } + const findBodyThreat = getFindBodyThreatMethod(done) axios .get(`http://127.0.0.1:${port}/api/test?param=testattack`) @@ -242,20 +242,7 @@ describe('test suite', () => { it('in request query with array params, attack in the second item', function (done) { this.timeout(5000) - function findBodyThreat (traces) { - let attackFound = false - traces.forEach(trace => { - trace.forEach(span => { - if (span.meta['_dd.appsec.json']) { - attackFound = true - } - }) - }) - if (attackFound) { - agent.unsubscribe(findBodyThreat) - done() - } - } + const findBodyThreat = getFindBodyThreatMethod(done) axios .get(`http://127.0.0.1:${port}/api/test?param[]=safe¶m[]=testattack`) @@ -267,20 +254,7 @@ describe('test suite', () => { it('in request query with array params, threat in the first item', function (done) { this.timeout(5000) - function findBodyThreat (traces) { - let attackFound = false - traces.forEach(trace => { - trace.forEach(span => { - if (span.meta['_dd.appsec.json']) { - attackFound = true - } - }) - }) - if (attackFound) { - agent.unsubscribe(findBodyThreat) - done() - } - } + const findBodyThreat = getFindBodyThreatMethod(done) axios .get(`http://127.0.0.1:${port}/api/test?param[]=testattack¶m[]=safe`) diff --git a/packages/dd-trace/test/appsec/index.sequelize.plugin.spec.js b/packages/dd-trace/test/appsec/index.sequelize.plugin.spec.js new file mode 100644 index 00000000000..d444b82ec5e --- /dev/null +++ b/packages/dd-trace/test/appsec/index.sequelize.plugin.spec.js @@ -0,0 +1,90 @@ +'use strict' + +const path = require('path') +const axios = require('axios') +const agent = require('../plugins/agent') +const appsec = require('../../src/appsec') +const Config = require('../../src/config') + +describe('sequelize', () => { + withVersions('sequelize', 'sequelize', sequelizeVersion => { + withVersions('mysql2', 'mysql2', () => { + withVersions('sequelize', 'express', (expressVersion) => { + let sequelize, User, server, port + + // init tracer + before(async () => { + await agent.load(['express', 'http'], { client: false }, { flushInterval: 1 }) + appsec.enable(new Config({ + appsec: { + enabled: true, + rules: path.join(__dirname, 'express-rules.json'), + apiSecurity: { + enabled: true, + requestSampling: 1 + } + } + })) + }) + + // close agent + after(() => { + appsec.disable() + return agent.close({ ritmReset: false }) + }) + + // init database + before(async () => { + const { Sequelize, DataTypes } = require(`../../../../versions/sequelize@${sequelizeVersion}`).get() + + sequelize = new Sequelize('db', 'root', '', { + host: '127.0.0.1', + dialect: 'mysql' + }) + User = sequelize.define('User', { + username: DataTypes.STRING, + birthday: DataTypes.DATE + }) + + await sequelize.sync({ force: true }) + await User.create({ + username: 'janedoe', + birthday: new Date(1980, 6, 20) + }) + }) + + // clean database + after(async () => { + await User.drop() + }) + + // init express + before((done) => { + const express = require(`../../../../versions/express@${expressVersion}`).get() + + const app = express() + app.get('/users', async (req, res) => { + const users = await User.findAll() + res.json(users) + }) + + server = app.listen(0, () => { + port = server.address().port + done() + }) + }) + + // stop express + after(() => { + return server.close() + }) + + it('Should complete the request on time', (done) => { + axios.get(`http://localhost:${port}/users`) + .then(() => done()) + .catch(done) + }) + }) + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/index.spec.js b/packages/dd-trace/test/appsec/index.spec.js index 14306f8203a..4b8c6c0438c 100644 --- a/packages/dd-trace/test/appsec/index.spec.js +++ b/packages/dd-trace/test/appsec/index.spec.js @@ -8,23 +8,37 @@ const appsec = require('../../src/appsec') const { bodyParser, cookieParser, - graphqlFinishExecute, incomingHttpRequestStart, incomingHttpRequestEnd, + passportVerify, queryParser, - passportVerify + nextBodyParsed, + nextQueryParsed, + expressProcessParams, + responseBody, + responseWriteHead, + responseSetHeader } = require('../../src/appsec/channels') const Reporter = require('../../src/appsec/reporter') const agent = require('../plugins/agent') const Config = require('../../src/config') const axios = require('axios') -const getPort = require('get-port') const blockedTemplate = require('../../src/appsec/blocked_templates') const { storage } = require('../../../datadog-core') -const addresses = require('../../src/appsec/addresses') const telemetryMetrics = require('../../src/telemetry/metrics') +const addresses = require('../../src/appsec/addresses') + +const resultActions = { + block_request: { + status_code: '401', + type: 'auto', + grpc_status_code: '10' + } +} + +describe('AppSec Index', function () { + this.timeout(5000) -describe('AppSec Index', () => { let config let AppSec let web @@ -32,6 +46,9 @@ describe('AppSec Index', () => { let passport let log let appsecTelemetry + let graphql + let apiSecuritySampler + let rasp const RULES = { rules: [{ a: 1 }] } @@ -53,6 +70,9 @@ describe('AppSec Index', () => { apiSecurity: { enabled: false, requestSampling: 0 + }, + rasp: { + enabled: true } } } @@ -80,12 +100,29 @@ describe('AppSec Index', () => { disable: sinon.stub() } + graphql = { + enable: sinon.stub(), + disable: sinon.stub() + } + + apiSecuritySampler = require('../../src/appsec/api_security_sampler') + sinon.spy(apiSecuritySampler, 'sampleRequest') + sinon.spy(apiSecuritySampler, 'isSampled') + + rasp = { + enable: sinon.stub(), + disable: sinon.stub() + } + AppSec = proxyquire('../../src/appsec', { '../log': log, '../plugins/util/web': web, './blocking': blocking, './passport': passport, - './telemetry': appsecTelemetry + './telemetry': appsecTelemetry, + './graphql': graphql, + './api_security_sampler': apiSecuritySampler, + './rasp': rasp }) sinon.stub(fs, 'readFileSync').returns(JSON.stringify(RULES)) @@ -112,6 +149,7 @@ describe('AppSec Index', () => { expect(incomingHttpRequestStart.subscribe) .to.have.been.calledOnceWithExactly(AppSec.incomingHttpStartTranslator) expect(incomingHttpRequestEnd.subscribe).to.have.been.calledOnceWithExactly(AppSec.incomingHttpEndTranslator) + expect(graphql.enable).to.have.been.calledOnceWithExactly() }) it('should log when enable fails', () => { @@ -132,17 +170,25 @@ describe('AppSec Index', () => { it('should subscribe to blockable channels', () => { expect(bodyParser.hasSubscribers).to.be.false expect(cookieParser.hasSubscribers).to.be.false - expect(queryParser.hasSubscribers).to.be.false expect(passportVerify.hasSubscribers).to.be.false - expect(graphqlFinishExecute.hasSubscribers).to.be.false + expect(queryParser.hasSubscribers).to.be.false + expect(nextBodyParsed.hasSubscribers).to.be.false + expect(nextQueryParsed.hasSubscribers).to.be.false + expect(expressProcessParams.hasSubscribers).to.be.false + expect(responseWriteHead.hasSubscribers).to.be.false + expect(responseSetHeader.hasSubscribers).to.be.false AppSec.enable(config) expect(bodyParser.hasSubscribers).to.be.true expect(cookieParser.hasSubscribers).to.be.true - expect(graphqlFinishExecute.hasSubscribers).to.be.true - expect(queryParser.hasSubscribers).to.be.true expect(passportVerify.hasSubscribers).to.be.true + expect(queryParser.hasSubscribers).to.be.true + expect(nextBodyParsed.hasSubscribers).to.be.true + expect(nextQueryParsed.hasSubscribers).to.be.true + expect(expressProcessParams.hasSubscribers).to.be.true + expect(responseWriteHead.hasSubscribers).to.be.true + expect(responseSetHeader.hasSubscribers).to.be.true }) it('should not subscribe to passportVerify if eventTracking is disabled', () => { @@ -163,6 +209,19 @@ describe('AppSec Index', () => { expect(appsecTelemetry.enable).to.be.calledOnceWithExactly(config.telemetry) }) + + it('should call rasp enable', () => { + AppSec.enable(config) + + expect(rasp.enable).to.be.calledOnceWithExactly(config) + }) + + it('should not call rasp enable when rasp is disabled', () => { + config.appsec.rasp.enabled = false + AppSec.enable(config) + + expect(rasp.enable).to.not.be.called + }) }) describe('disable', () => { @@ -183,6 +242,8 @@ describe('AppSec Index', () => { expect(incomingHttpRequestStart.unsubscribe) .to.have.been.calledOnceWithExactly(AppSec.incomingHttpStartTranslator) expect(incomingHttpRequestEnd.unsubscribe).to.have.been.calledOnceWithExactly(AppSec.incomingHttpEndTranslator) + expect(graphql.disable).to.have.been.calledOnceWithExactly() + expect(rasp.disable).to.have.been.calledOnceWithExactly() }) it('should disable AppSec when DC channels are not active', () => { @@ -202,9 +263,13 @@ describe('AppSec Index', () => { expect(bodyParser.hasSubscribers).to.be.false expect(cookieParser.hasSubscribers).to.be.false - expect(graphqlFinishExecute.hasSubscribers).to.be.false - expect(queryParser.hasSubscribers).to.be.false expect(passportVerify.hasSubscribers).to.be.false + expect(queryParser.hasSubscribers).to.be.false + expect(nextBodyParsed.hasSubscribers).to.be.false + expect(nextQueryParsed.hasSubscribers).to.be.false + expect(expressProcessParams.hasSubscribers).to.be.false + expect(responseWriteHead.hasSubscribers).to.be.false + expect(responseSetHeader.hasSubscribers).to.be.false }) it('should call appsec telemetry disable', () => { @@ -234,7 +299,7 @@ describe('AppSec Index', () => { url: '/path', headers: { 'user-agent': 'Arachni', - 'host': 'localhost', + host: 'localhost', cookie: 'a=1;b=2' }, method: 'POST', @@ -253,10 +318,12 @@ describe('AppSec Index', () => { 'http.client_ip': '127.0.0.1' }) expect(waf.run).to.have.been.calledOnceWithExactly({ - 'server.request.uri.raw': '/path', - 'server.request.headers.no_cookies': { 'user-agent': 'Arachni', host: 'localhost' }, - 'server.request.method': 'POST', - 'http.client_ip': '127.0.0.1' + persistent: { + 'server.request.uri.raw': '/path', + 'server.request.headers.no_cookies': { 'user-agent': 'Arachni', host: 'localhost' }, + 'server.request.method': 'POST', + 'http.client_ip': '127.0.0.1' + } }, req) }) }) @@ -274,12 +341,12 @@ describe('AppSec Index', () => { web.root.returns(rootSpan) }) - it('should propagate incoming http end data', () => { + it('should not propagate incoming http end data without express', () => { const req = { url: '/path', headers: { 'user-agent': 'Arachni', - 'host': 'localhost', + host: 'localhost', cookie: 'a=1;b=2' }, method: 'POST', @@ -302,20 +369,17 @@ describe('AppSec Index', () => { AppSec.incomingHttpEndTranslator({ req, res }) - expect(waf.run).to.have.been.calledOnceWithExactly({ - 'server.response.status': '201', - 'server.response.headers.no_cookies': { 'content-type': 'application/json', 'content-lenght': 42 } - }, req) + expect(waf.run).to.have.not.been.called expect(Reporter.finishRequest).to.have.been.calledOnceWithExactly(req, res) }) - it('should propagate incoming http end data with invalid framework properties', () => { + it('should not propagate incoming http end data with invalid framework properties', () => { const req = { url: '/path', headers: { 'user-agent': 'Arachni', - 'host': 'localhost', + host: 'localhost', cookie: 'a=1;b=2' }, method: 'POST', @@ -343,10 +407,7 @@ describe('AppSec Index', () => { AppSec.incomingHttpEndTranslator({ req, res }) - expect(waf.run).to.have.been.calledOnceWithExactly({ - 'server.response.status': '201', - 'server.response.headers.no_cookies': { 'content-type': 'application/json', 'content-lenght': 42 } - }, req) + expect(waf.run).to.have.not.been.called expect(Reporter.finishRequest).to.have.been.calledOnceWithExactly(req, res) }) @@ -356,7 +417,7 @@ describe('AppSec Index', () => { url: '/path', headers: { 'user-agent': 'Arachni', - 'host': 'localhost', + host: 'localhost', cookie: 'a=1;b=2' }, method: 'POST', @@ -373,9 +434,6 @@ describe('AppSec Index', () => { route: { path: '/path/:c' }, - params: { - c: '3' - }, cookies: { d: '4', e: '5' @@ -395,12 +453,11 @@ describe('AppSec Index', () => { AppSec.incomingHttpEndTranslator({ req, res }) expect(waf.run).to.have.been.calledOnceWithExactly({ - 'server.response.status': '201', - 'server.response.headers.no_cookies': { 'content-type': 'application/json', 'content-lenght': 42 }, - 'server.request.body': { a: '1' }, - 'server.request.path_params': { c: '3' }, - 'server.request.cookies': { d: '4', e: '5' }, - 'server.request.query': { b: '2' } + persistent: { + 'server.request.body': { a: '1' }, + 'server.request.cookies': { d: '4', e: '5' }, + 'server.request.query': { b: '2' } + } }, req) expect(Reporter.finishRequest).to.have.been.calledOnceWithExactly(req, res) }) @@ -429,7 +486,7 @@ describe('AppSec Index', () => { url: '/path', headers: { 'user-agent': 'Arachni', - 'host': 'localhost', + host: 'localhost', cookie: 'a=1;b=2' }, method: 'POST', @@ -443,10 +500,12 @@ describe('AppSec Index', () => { AppSec.incomingHttpStartTranslator({ req, res }) expect(waf.run).to.have.been.calledOnceWithExactly({ - 'server.request.uri.raw': '/path', - 'server.request.headers.no_cookies': { 'user-agent': 'Arachni', host: 'localhost' }, - 'server.request.method': 'POST', - 'http.client_ip': '127.0.0.1' + persistent: { + 'server.request.uri.raw': '/path', + 'server.request.headers.no_cookies': { 'user-agent': 'Arachni', host: 'localhost' }, + 'server.request.method': 'POST', + 'http.client_ip': '127.0.0.1' + } }, req) }) @@ -462,7 +521,7 @@ describe('AppSec Index', () => { url: '/path', headers: { 'user-agent': 'Arachni', - 'host': 'localhost', + host: 'localhost', cookie: 'a=1;b=2' }, method: 'POST', @@ -476,10 +535,12 @@ describe('AppSec Index', () => { AppSec.incomingHttpStartTranslator({ req, res }) expect(waf.run).to.have.been.calledOnceWithExactly({ - 'server.request.uri.raw': '/path', - 'server.request.headers.no_cookies': { 'user-agent': 'Arachni', host: 'localhost' }, - 'server.request.method': 'POST', - 'http.client_ip': '127.0.0.1' + persistent: { + 'server.request.uri.raw': '/path', + 'server.request.headers.no_cookies': { 'user-agent': 'Arachni', host: 'localhost' }, + 'server.request.method': 'POST', + 'http.client_ip': '127.0.0.1' + } }, req) }) @@ -495,7 +556,7 @@ describe('AppSec Index', () => { url: '/path', headers: { 'user-agent': 'Arachni', - 'host': 'localhost', + host: 'localhost', cookie: 'a=1;b=2' }, method: 'POST', @@ -509,13 +570,62 @@ describe('AppSec Index', () => { AppSec.incomingHttpStartTranslator({ req, res }) expect(waf.run).to.have.been.calledOnceWithExactly({ - 'server.request.uri.raw': '/path', - 'server.request.headers.no_cookies': { 'user-agent': 'Arachni', host: 'localhost' }, - 'server.request.method': 'POST', - 'http.client_ip': '127.0.0.1', - 'waf.context.processor': { 'extract-schema': true } + persistent: { + 'server.request.uri.raw': '/path', + 'server.request.headers.no_cookies': { 'user-agent': 'Arachni', host: 'localhost' }, + 'server.request.method': 'POST', + 'http.client_ip': '127.0.0.1', + 'waf.context.processor': { 'extract-schema': true } + } }, req) }) + + describe('onResponseBody', () => { + beforeEach(() => { + config.appsec.apiSecurity = { + enabled: true, + requestSampling: 1 + } + AppSec.enable(config) + }) + + afterEach(() => { + AppSec.disable() + }) + + it('should not do anything if body is not an object', () => { + responseBody.publish({ req: {}, body: 'string' }) + responseBody.publish({ req: {}, body: null }) + + expect(apiSecuritySampler.isSampled).to.not.been.called + expect(waf.run).to.not.been.called + }) + + it('should not call to the waf if it is not a sampled request', () => { + apiSecuritySampler.isSampled = apiSecuritySampler.isSampled.instantiateFake(() => false) + const req = {} + + responseBody.publish({ req, body: {} }) + + expect(apiSecuritySampler.isSampled).to.have.been.calledOnceWith(req) + expect(waf.run).to.not.been.called + }) + + it('should call to the waf if it is a sampled request', () => { + apiSecuritySampler.isSampled = apiSecuritySampler.isSampled.instantiateFake(() => true) + const req = {} + const body = {} + + responseBody.publish({ req, body }) + + expect(apiSecuritySampler.isSampled).to.have.been.calledOnceWith(req) + expect(waf.run).to.been.calledOnceWith({ + persistent: { + [addresses.HTTP_INCOMING_RESPONSE_BODY]: body + } + }, req) + }) + }) }) describe('Channel handlers', () => { @@ -533,7 +643,7 @@ describe('AppSec Index', () => { url: '/path', headers: { 'user-agent': 'Arachni', - 'host': 'localhost' + host: 'localhost' }, method: 'POST', socket: { @@ -546,18 +656,16 @@ describe('AppSec Index', () => { 'content-type': 'application/json', 'content-lenght': 42 }), - setHeader: sinon.stub(), - end: sinon.stub() + writeHead: sinon.stub(), + end: sinon.stub(), + getHeaderNames: sinon.stub().returns([]) } + res.writeHead.returns(res) AppSec.enable(config) AppSec.incomingHttpStartTranslator({ req, res }) }) - afterEach(() => { - AppSec.disable() - }) - describe('onRequestBodyParsed', () => { it('Should not block without body', () => { sinon.stub(waf, 'run') @@ -577,7 +685,9 @@ describe('AppSec Index', () => { bodyParser.publish({ req, res, body, abortController }) expect(waf.run).to.have.been.calledOnceWith({ - 'server.request.body': { key: 'value' } + persistent: { + 'server.request.body': { key: 'value' } + } }) expect(abortController.abort).not.to.have.been.called expect(res.end).not.to.have.been.called @@ -586,12 +696,14 @@ describe('AppSec Index', () => { it('Should block when it is detected as attack', () => { const body = { key: 'value' } req.body = body - sinon.stub(waf, 'run').returns(['block']) + sinon.stub(waf, 'run').returns(resultActions) bodyParser.publish({ req, res, body, abortController }) expect(waf.run).to.have.been.calledOnceWith({ - 'server.request.body': { key: 'value' } + persistent: { + 'server.request.body': { key: 'value' } + } }) expect(abortController.abort).to.have.been.called expect(res.end).to.have.been.called @@ -616,7 +728,9 @@ describe('AppSec Index', () => { cookieParser.publish({ req, res, abortController, cookies }) expect(waf.run).to.have.been.calledOnceWith({ - 'server.request.cookies': { key: 'value' } + persistent: { + 'server.request.cookies': { key: 'value' } + } }) expect(abortController.abort).not.to.have.been.called expect(res.end).not.to.have.been.called @@ -624,12 +738,14 @@ describe('AppSec Index', () => { it('Should block when it is detected as attack', () => { const cookies = { key: 'value' } - sinon.stub(waf, 'run').returns(['block']) + sinon.stub(waf, 'run').returns(resultActions) cookieParser.publish({ req, res, abortController, cookies }) expect(waf.run).to.have.been.calledOnceWith({ - 'server.request.cookies': { key: 'value' } + persistent: { + 'server.request.cookies': { key: 'value' } + } }) expect(abortController.abort).to.have.been.called expect(res.end).to.have.been.called @@ -655,7 +771,9 @@ describe('AppSec Index', () => { queryParser.publish({ req, res, query, abortController }) expect(waf.run).to.have.been.calledOnceWith({ - 'server.request.query': { key: 'value' } + persistent: { + 'server.request.query': { key: 'value' } + } }) expect(abortController.abort).not.to.have.been.called expect(res.end).not.to.have.been.called @@ -664,12 +782,14 @@ describe('AppSec Index', () => { it('Should block when it is detected as attack', () => { const query = { key: 'value' } req.query = query - sinon.stub(waf, 'run').returns(['block']) + sinon.stub(waf, 'run').returns(resultActions) queryParser.publish({ req, res, query, abortController }) expect(waf.run).to.have.been.calledOnceWith({ - 'server.request.query': { key: 'value' } + persistent: { + 'server.request.query': { key: 'value' } + } }) expect(abortController.abort).to.have.been.called expect(res.end).to.have.been.called @@ -705,63 +825,136 @@ describe('AppSec Index', () => { }) }) - describe('onGraphqlQueryParse', () => { - it('Should not call waf if resolvers is undefined', () => { - const resolvers = undefined - const rootSpan = {} + describe('onResponseWriteHead', () => { + it('should call abortController if response was already blocked', () => { + sinon.stub(waf, 'run').returns(resultActions) - sinon.stub(waf, 'run') - sinon.stub(storage, 'getStore').returns({ req: {} }) - web.root.returns(rootSpan) + const responseHeaders = { + 'content-type': 'application/json', + 'content-lenght': 42, + 'set-cookie': 'a=1;b=2' + } - graphqlFinishExecute.publish({ resolvers }) + responseWriteHead.publish({ req, res, abortController, statusCode: 404, responseHeaders }) - expect(waf.run).not.to.have.been.called + expect(waf.run).to.have.been.calledOnceWithExactly({ + persistent: { + 'server.response.status': '404', + 'server.response.headers.no_cookies': { + 'content-type': 'application/json', + 'content-lenght': 42 + } + } + }, req) + expect(abortController.abort).to.have.been.calledOnce + expect(res.end).to.have.been.calledOnce + + abortController.abort.resetHistory() + + responseWriteHead.publish({ req, res, abortController, statusCode: 404, responseHeaders }) + + expect(waf.run).to.have.been.calledOnce + expect(abortController.abort).to.have.been.calledOnce + expect(res.end).to.have.been.calledOnce }) - it('Should not call waf if resolvers is not an object', () => { - const resolvers = '' - const rootSpan = {} + it('should not call the WAF if response was already analyzed', () => { + sinon.stub(waf, 'run').returns(null) - sinon.stub(waf, 'run') - sinon.stub(storage, 'getStore').returns({ req: {} }) - web.root.returns(rootSpan) + const responseHeaders = { + 'content-type': 'application/json', + 'content-lenght': 42, + 'set-cookie': 'a=1;b=2' + } - graphqlFinishExecute.publish({ resolvers }) + responseWriteHead.publish({ req, res, abortController, statusCode: 404, responseHeaders }) - expect(waf.run).not.to.have.been.called + expect(waf.run).to.have.been.calledOnceWithExactly({ + persistent: { + 'server.response.status': '404', + 'server.response.headers.no_cookies': { + 'content-type': 'application/json', + 'content-lenght': 42 + } + } + }, req) + expect(abortController.abort).to.have.not.been.called + expect(res.end).to.have.not.been.called + + responseWriteHead.publish({ req, res, abortController, statusCode: 404, responseHeaders }) + + expect(waf.run).to.have.been.calledOnce + expect(abortController.abort).to.have.not.been.called + expect(res.end).to.have.not.been.called }) - it('Should not call waf if req is unavailable', () => { - const resolvers = { user: [ { id: '1234' } ] } - sinon.stub(waf, 'run') - sinon.stub(storage, 'getStore').returns({}) + it('should not do anything without a root span', () => { + web.root.returns(null) + sinon.stub(waf, 'run').returns(null) - graphqlFinishExecute.publish({ resolvers }) + const responseHeaders = { + 'content-type': 'application/json', + 'content-lenght': 42, + 'set-cookie': 'a=1;b=2' + } - expect(waf.run).not.to.have.been.called + responseWriteHead.publish({ req, res, abortController, statusCode: 404, responseHeaders }) + + expect(waf.run).to.have.not.been.called + expect(abortController.abort).to.have.not.been.called + expect(res.end).to.have.not.been.called }) - it('Should call waf if resolvers is well formatted', () => { - const context = { - resolvers: { - user: [ { id: '1234' } ] + it('should call the WAF with responde code and headers', () => { + sinon.stub(waf, 'run').returns(resultActions) + + const responseHeaders = { + 'content-type': 'application/json', + 'content-lenght': 42, + 'set-cookie': 'a=1;b=2' + } + + responseWriteHead.publish({ req, res, abortController, statusCode: 404, responseHeaders }) + + expect(waf.run).to.have.been.calledOnceWithExactly({ + persistent: { + 'server.response.status': '404', + 'server.response.headers.no_cookies': { + 'content-type': 'application/json', + 'content-lenght': 42 + } } + }, req) + expect(abortController.abort).to.have.been.calledOnce + expect(res.end).to.have.been.calledOnce + }) + }) + + describe('onResponseSetHeader', () => { + it('should call abortController if response was already blocked', () => { + // First block the request + sinon.stub(waf, 'run').returns(resultActions) + + const responseHeaders = { + 'content-type': 'application/json', + 'content-lenght': 42, + 'set-cookie': 'a=1;b=2' } - const rootSpan = {} + responseWriteHead.publish({ req, res, abortController, statusCode: 404, responseHeaders }) - sinon.stub(waf, 'run') - sinon.stub(storage, 'getStore').returns({ req: {} }) - web.root.returns(rootSpan) + expect(abortController.abort).to.have.been.calledOnce - graphqlFinishExecute.publish({ context }) + abortController.abort.reset() - expect(waf.run).to.have.been.calledOnceWithExactly( - { - [addresses.HTTP_INCOMING_GRAPHQL_RESOLVERS]: context.resolvers - }, - {} - ) + responseSetHeader.publish({ res, abortController }) + + expect(abortController.abort).to.have.been.calledOnce + }) + + it('should not call abortController if response was not blocked', () => { + responseSetHeader.publish({ res, abortController }) + + expect(abortController.abort).to.have.not.been.calledOnce }) }) }) @@ -826,7 +1019,9 @@ describe('AppSec Index', () => { }) }) -describe('IP blocking', () => { +describe('IP blocking', function () { + this.timeout(5000) + const invalidIp = '1.2.3.4' const validIp = '4.3.2.1' const ruleData = { @@ -848,30 +1043,33 @@ describe('IP blocking', () => { const jsonDefaultContent = JSON.parse(blockedTemplate.json) let http, appListener, port - before(() => { - return getPort().then(newPort => { - port = newPort - }) - }) + before(() => { return agent.load('http') .then(() => { http = require('http') }) }) + before(done => { const server = new http.Server((req, res) => { res.writeHead(200) res.end(JSON.stringify({ message: 'OK' })) }) appListener = server - .listen(port, 'localhost', () => done()) + .listen(0, 'localhost', () => { + port = appListener.address().port + done() + }) }) beforeEach(() => { appsec.enable(new Config({ appsec: { - enabled: true + enabled: true, + rasp: { + enabled: false // disable rasp to not trigger lfi + } } })) @@ -934,7 +1132,7 @@ describe('IP blocking', () => { await axios.get(`http://localhost:${port}/`, { headers: { [ipHeader]: invalidIp, - 'Accept': '*/*' + Accept: '*/*' } }).catch((err) => { expect(err.response.status).to.be.equal(403) @@ -946,7 +1144,7 @@ describe('IP blocking', () => { await axios.get(`http://localhost:${port}/`, { headers: { [ipHeader]: invalidIp, - 'Accept': 'text/html' + Accept: 'text/html' } }).catch((err) => { expect(err.response.status).to.be.equal(403) @@ -995,6 +1193,7 @@ describe('IP blocking', () => { }).then(() => { throw new Error('Not expected') }).catch((err) => { + expect(err.message).to.not.equal('Not expected') expect(err.response.status).to.be.equal(500) expect(err.response.data).to.deep.equal(jsonDefaultContent) }) @@ -1004,11 +1203,12 @@ describe('IP blocking', () => { return axios.get(`http://localhost:${port}/`, { headers: { 'x-forwarded-for': invalidIp, - 'Accept': 'text/html' + Accept: 'text/html' } }).then(() => { throw new Error('Not expected') }).catch((err) => { + expect(err.message).to.not.equal('Not expected') expect(err.response.status).to.be.equal(500) expect(err.response.data).to.deep.equal(htmlDefaultContent) }) @@ -1054,6 +1254,7 @@ describe('IP blocking', () => { }).then(() => { throw new Error('Not resolve expected') }).catch((err) => { + expect(err.message).to.not.equal('Not resolve expected') expect(err.response.status).to.be.equal(301) expect(err.response.headers.location).to.be.equal('/error') }) diff --git a/packages/dd-trace/test/appsec/next/app-dir/app/api/test-formdata/route.js b/packages/dd-trace/test/appsec/next/app-dir/app/api/test-formdata/route.js new file mode 100644 index 00000000000..69109a530e6 --- /dev/null +++ b/packages/dd-trace/test/appsec/next/app-dir/app/api/test-formdata/route.js @@ -0,0 +1,18 @@ +import { NextResponse } from 'next/server' +export async function POST (request) { + const body = await request.formData() + + if (!body.entries) { + return NextResponse.json({ + message: 'Instrumentation modified form data' + }, { + status: 500 + }) + } + + return NextResponse.json({ + now: Date.now(), + cache: 'no-store', + data: body + }) +} diff --git a/packages/dd-trace/test/appsec/next/pages-dir/pages/api/test-formdata/index.js b/packages/dd-trace/test/appsec/next/pages-dir/pages/api/test-formdata/index.js new file mode 100644 index 00000000000..538520f5eaf --- /dev/null +++ b/packages/dd-trace/test/appsec/next/pages-dir/pages/api/test-formdata/index.js @@ -0,0 +1,10 @@ +// Next.js API route support: https://nextjs.org/docs/api-routes/introduction + +export default async function POST (req, res) { + const body = req.body + res.status(200).json({ + cache: 'no-store', + data: body, + query: req.query + }) +} diff --git a/packages/dd-trace/test/appsec/next/pages-dir/server.js b/packages/dd-trace/test/appsec/next/pages-dir/server.js index 673974ac988..c7cfda1abff 100644 --- a/packages/dd-trace/test/appsec/next/pages-dir/server.js +++ b/packages/dd-trace/test/appsec/next/pages-dir/server.js @@ -3,6 +3,7 @@ const { PORT, HOSTNAME } = process.env const { createServer } = require('http') +// eslint-disable-next-line n/no-deprecated-api const { parse } = require('url') const next = require('next') // eslint-disable-line import/no-extraneous-dependencies diff --git a/packages/dd-trace/test/appsec/passport.spec.js b/packages/dd-trace/test/appsec/passport.spec.js index e192d1e150e..7a3db36798c 100644 --- a/packages/dd-trace/test/appsec/passport.spec.js +++ b/packages/dd-trace/test/appsec/passport.spec.js @@ -14,6 +14,7 @@ describe('Passport', () => { } let passportModule, log, events, setUser + beforeEach(() => { rootSpan.context = () => { return {} } diff --git a/packages/dd-trace/test/appsec/rasp/fs-plugin.spec.js b/packages/dd-trace/test/appsec/rasp/fs-plugin.spec.js new file mode 100644 index 00000000000..03b2a0acdd0 --- /dev/null +++ b/packages/dd-trace/test/appsec/rasp/fs-plugin.spec.js @@ -0,0 +1,251 @@ +'use strict' + +const proxyquire = require('proxyquire') +const { assert } = require('chai') +const path = require('path') +const dc = require('dc-polyfill') +const { storage } = require('../../../../datadog-core') +const { AppsecFsPlugin } = require('../../../src/appsec/rasp/fs-plugin') +const agent = require('../../plugins/agent') + +const opStartCh = dc.channel('apm:fs:operation:start') +const opFinishCh = dc.channel('apm:fs:operation:finish') + +describe('AppsecFsPlugin', () => { + let appsecFsPlugin + + beforeEach(() => { + appsecFsPlugin = new AppsecFsPlugin() + appsecFsPlugin.enable() + }) + + afterEach(() => { appsecFsPlugin.disable() }) + + describe('enable/disable', () => { + let fsPlugin, configure + + beforeEach(() => { + configure = sinon.stub() + class PluginClass { + addSub (channelName, handler) {} + + configure (config) { + configure(config) + } + } + + fsPlugin = proxyquire('../../../src/appsec/rasp/fs-plugin', { + '../../plugins/plugin': PluginClass + }) + }) + + afterEach(() => { sinon.restore() }) + + it('should require valid mod when calling enable', () => { + fsPlugin.enable('iast') + + sinon.assert.calledOnceWithExactly(configure, true) + }) + + it('should create only one instance', () => { + fsPlugin.enable('iast') + fsPlugin.enable('iast') + fsPlugin.enable('rasp') + + sinon.assert.calledOnceWithExactly(configure, true) + }) + + it('should discard unknown mods when enabled', () => { + fsPlugin.enable('unknown') + sinon.assert.notCalled(configure) + + fsPlugin.enable() + sinon.assert.notCalled(configure) + }) + + it('should not disable if there are still modules using the plugin', () => { + fsPlugin.enable('iast') + fsPlugin.enable('rasp') + + fsPlugin.disable('rasp') + + sinon.assert.calledOnce(configure) + }) + + it('should disable only if there are no more modules using the plugin', () => { + fsPlugin.enable('iast') + fsPlugin.enable('rasp') + + fsPlugin.disable('rasp') + fsPlugin.disable('iast') + + sinon.assert.calledTwice(configure) + assert.strictEqual(configure.secondCall.args[0], false) + }) + + it('should discard unknown mods when disabling', () => { + fsPlugin.disable('unknown') + sinon.assert.notCalled(configure) + + fsPlugin.disable() + sinon.assert.notCalled(configure) + }) + }) + + describe('_onFsOperationStart', () => { + it('should mark fs root', () => { + const origStore = {} + storage.enterWith(origStore) + + appsecFsPlugin._onFsOperationStart() + + let store = storage.getStore() + assert.property(store, 'fs') + assert.propertyVal(store.fs, 'parentStore', origStore) + assert.propertyVal(store.fs, 'root', true) + + appsecFsPlugin._onFsOperationFinishOrRenderEnd() + + store = storage.getStore() + assert.equal(store, origStore) + assert.notProperty(store, 'fs') + }) + + it('should mark fs children', () => { + const origStore = { orig: true } + storage.enterWith(origStore) + + appsecFsPlugin._onFsOperationStart() + + const rootStore = storage.getStore() + assert.property(rootStore, 'fs') + assert.propertyVal(rootStore.fs, 'parentStore', origStore) + assert.propertyVal(rootStore.fs, 'root', true) + + appsecFsPlugin._onFsOperationStart() + + let store = storage.getStore() + assert.property(store, 'fs') + assert.propertyVal(store.fs, 'parentStore', rootStore) + assert.propertyVal(store.fs, 'root', false) + assert.propertyVal(store, 'orig', true) + + appsecFsPlugin._onFsOperationFinishOrRenderEnd() + + store = storage.getStore() + assert.equal(store, rootStore) + + appsecFsPlugin._onFsOperationFinishOrRenderEnd() + store = storage.getStore() + assert.equal(store, origStore) + }) + }) + + describe('_onResponseRenderStart', () => { + it('should mark fs ops as excluded while response rendering', () => { + appsecFsPlugin.enable() + + const origStore = {} + storage.enterWith(origStore) + + appsecFsPlugin._onResponseRenderStart() + + let store = storage.getStore() + assert.property(store, 'fs') + assert.propertyVal(store.fs, 'parentStore', origStore) + assert.propertyVal(store.fs, 'opExcluded', true) + + appsecFsPlugin._onFsOperationFinishOrRenderEnd() + + store = storage.getStore() + assert.equal(store, origStore) + assert.notProperty(store, 'fs') + }) + }) + + describe('integration', () => { + describe('apm:fs:operation', () => { + let fs + + afterEach(() => agent.close({ ritmReset: false })) + + beforeEach(() => agent.load('fs', undefined, { flushInterval: 1 }).then(() => { + fs = require('fs') + })) + + it('should mark root operations', () => { + let count = 0 + const onStart = () => { + const store = storage.getStore() + assert.isNotNull(store.fs) + + count++ + assert.strictEqual(count === 1, store.fs.root) + } + + try { + const origStore = {} + storage.enterWith(origStore) + + opStartCh.subscribe(onStart) + + fs.readFileSync(path.join(__dirname, 'fs-plugin.spec.js')) + + assert.strictEqual(count, 4) + } finally { + opStartCh.unsubscribe(onStart) + } + }) + + it('should mark root even if op is excluded', () => { + let count = 0 + const onStart = () => { + const store = storage.getStore() + assert.isNotNull(store.fs) + + count++ + assert.isUndefined(store.fs.root) + } + + try { + const origStore = { + fs: { opExcluded: true } + } + storage.enterWith(origStore) + + opStartCh.subscribe(onStart) + + fs.readFileSync(path.join(__dirname, 'fs-plugin.spec.js')) + + assert.strictEqual(count, 4) + } finally { + opStartCh.unsubscribe(onStart) + } + }) + + it('should clean up store when finishing op', () => { + let count = 4 + const onFinish = () => { + const store = storage.getStore() + count-- + + if (count === 0) { + assert.isUndefined(store.fs) + } + } + try { + const origStore = {} + storage.enterWith(origStore) + + opFinishCh.subscribe(onFinish) + + fs.readFileSync(path.join(__dirname, 'fs-plugin.spec.js')) + + assert.strictEqual(count, 0) + } finally { + opFinishCh.unsubscribe(onFinish) + } + }) + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/rasp/index.spec.js b/packages/dd-trace/test/appsec/rasp/index.spec.js new file mode 100644 index 00000000000..be6c602780a --- /dev/null +++ b/packages/dd-trace/test/appsec/rasp/index.spec.js @@ -0,0 +1,105 @@ +'use strict' + +const proxyquire = require('proxyquire') +const { handleUncaughtExceptionMonitor } = require('../../../src/appsec/rasp') +const { DatadogRaspAbortError } = require('../../../src/appsec/rasp/utils') + +describe('RASP', () => { + let rasp, subscribe, unsubscribe, block, blocked + + beforeEach(() => { + const config = { + appsec: { + stackTrace: { + enabled: true, + maxStackTraces: 2, + maxDepth: 42 + } + } + } + + subscribe = sinon.stub() + unsubscribe = sinon.stub() + + block = sinon.stub() + + rasp = proxyquire('../../../src/appsec/rasp', { + '../blocking': { + block, + isBlocked: sinon.stub().callsFake(() => blocked) + }, + '../channels': { + expressMiddlewareError: { + subscribe, + unsubscribe, + hasSubscribers: true + } + } + }) + + rasp.enable(config) + }) + + afterEach(() => { + sinon.restore() + rasp.disable() + }) + + describe('handleUncaughtExceptionMonitor', () => { + it('should not break with infinite loop of cause', () => { + const err = new Error() + err.cause = err + + handleUncaughtExceptionMonitor(err) + }) + }) + + describe('enable/disable', () => { + it('should subscribe to apm:express:middleware:error', () => { + sinon.assert.calledOnce(subscribe) + }) + + it('should unsubscribe to apm:express:middleware:error', () => { + rasp.disable() + + sinon.assert.calledOnce(unsubscribe) + }) + }) + + describe('blockOnDatadogRaspAbortError', () => { + let req, res, blockingAction + + beforeEach(() => { + req = {} + res = {} + blockingAction = {} + }) + + afterEach(() => { + sinon.restore() + }) + + it('should skip non DatadogRaspAbortError', () => { + rasp.blockOnDatadogRaspAbortError({ error: new Error() }) + + sinon.assert.notCalled(block) + }) + + it('should block DatadogRaspAbortError first time', () => { + rasp.blockOnDatadogRaspAbortError({ error: new DatadogRaspAbortError(req, res, blockingAction) }) + + sinon.assert.calledOnce(block) + }) + + it('should skip calling block if blocked before', () => { + rasp.blockOnDatadogRaspAbortError({ error: new DatadogRaspAbortError(req, res, blockingAction) }) + + blocked = true + + rasp.blockOnDatadogRaspAbortError({ error: new DatadogRaspAbortError(req, res, blockingAction) }) + rasp.blockOnDatadogRaspAbortError({ error: new DatadogRaspAbortError(req, res, blockingAction) }) + + sinon.assert.calledOnce(block) + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/rasp/lfi.express.plugin.spec.js b/packages/dd-trace/test/appsec/rasp/lfi.express.plugin.spec.js new file mode 100644 index 00000000000..b5b825cc628 --- /dev/null +++ b/packages/dd-trace/test/appsec/rasp/lfi.express.plugin.spec.js @@ -0,0 +1,469 @@ +'use strict' + +const Axios = require('axios') +const os = require('os') +const fs = require('fs') +const agent = require('../../plugins/agent') +const appsec = require('../../../src/appsec') +const Config = require('../../../src/config') +const path = require('path') +const { assert } = require('chai') +const { checkRaspExecutedAndNotThreat, checkRaspExecutedAndHasThreat } = require('./utils') + +describe('RASP - lfi', () => { + let axios + + async function testBlockingRequest (url = '/?file=/test.file', config = undefined, ruleEvalCount = 1) { + try { + await axios.get(url, config) + } catch (e) { + if (!e.response) { + throw e + } + + assert.strictEqual(e.response.status, 418) // a teapot + + return checkRaspExecutedAndHasThreat(agent, 'rasp-lfi-rule-id-1', ruleEvalCount) + } + + assert.fail('Request should be blocked') + } + + withVersions('express', 'express', expressVersion => { + let app, server + + before(() => { + return agent.load(['http', 'express'], { client: false }) + }) + + before((done) => { + const express = require(`../../../../../versions/express@${expressVersion}`).get() + const expressApp = express() + + expressApp.get('/', (req, res) => { + app(req, res) + }) + + appsec.enable(new Config({ + appsec: { + enabled: true, + rules: path.join(__dirname, 'resources', 'lfi_rasp_rules.json'), + rasp: { enabled: true } + } + })) + + server = expressApp.listen(0, () => { + const port = server.address().port + axios = Axios.create({ + baseURL: `http://localhost:${port}` + }) + done() + }) + }) + + after(() => { + appsec.disable() + server.close() + return agent.close({ ritmReset: false }) + }) + + describe('lfi', () => { + function getApp (fn, args, options) { + return async (req, res) => { + try { + const result = await fn(args) + options.onfinish?.(result) + } catch (e) { + if (e.message === 'DatadogRaspAbortError') { + res.writeHead(418) + } + } + res.end('end') + } + } + + function getAppSync (fn, args, options) { + return (req, res) => { + try { + const result = fn(args) + options.onfinish?.(result) + } catch (e) { + if (e.message === 'DatadogRaspAbortError') { + res.writeHead(418) + } + } + res.end('end') + } + } + + function runFsMethodTest (description, options, fn, ...args) { + const { vulnerableIndex = 0, ruleEvalCount } = options + + describe(description, () => { + const getAppFn = options.getAppFn ?? getApp + + it('should block param from the request', async () => { + app = getAppFn(fn, args, options) + + const file = args[vulnerableIndex] + return testBlockingRequest(`/?file=${file}`, undefined, ruleEvalCount) + .then(span => { + assert(span.meta['_dd.appsec.json'].includes(file)) + }) + }) + + it('should not block if param not found in the request', async () => { + app = getAppFn(fn, args, options) + + await axios.get('/?file=/test.file') + + return checkRaspExecutedAndNotThreat(agent, false) + }) + }) + } + + function runFsMethodTestThreeWay (methodName, options = {}, ...args) { + let desc = `test ${methodName} ${options.desc ?? ''}` + const { vulnerableIndex = 0 } = options + if (vulnerableIndex !== 0) { + desc += ` with vulnerable index ${vulnerableIndex}` + } + describe(desc, () => { + runFsMethodTest(`test fs.${methodName}Sync method`, { ...options, getAppFn: getAppSync }, (args) => { + return require('fs')[`${methodName}Sync`](...args) + }, ...args) + + runFsMethodTest(`test fs.${methodName} method`, options, (args) => { + return new Promise((resolve, reject) => { + require('fs')[methodName](...args, (err, res) => { + if (err) reject(err) + else resolve(res) + }) + }) + }, ...args) + + runFsMethodTest(`test fs.promises.${methodName} method`, options, async (args) => { + return require('fs').promises[methodName](...args) + }, ...args) + }) + } + + function unlink (...args) { + args.forEach(arg => { + try { + fs.unlinkSync(arg) + } catch (e) { + + } + }) + } + + describe('test access', () => { + runFsMethodTestThreeWay('access', undefined, __filename) + runFsMethodTestThreeWay('access', { desc: 'Buffer' }, Buffer.from(__filename)) + + // not supported by waf yet + // runFsMethodTestThreeWay('access', { desc: 'URL' }, new URL(`file://${__filename}`)) + }) + + describe('test appendFile', () => { + const filename = path.join(os.tmpdir(), 'test-appendfile') + + beforeEach(() => { + fs.writeFileSync(filename, '') + }) + + afterEach(() => { + fs.unlinkSync(filename) + }) + + runFsMethodTestThreeWay('appendFile', undefined, filename, 'test-content') + }) + + describe('test chmod', () => { + const filename = path.join(os.tmpdir(), 'test-chmod') + + beforeEach(() => { + fs.writeFileSync(filename, '') + }) + + afterEach(() => { + fs.unlinkSync(filename) + }) + runFsMethodTestThreeWay('chmod', undefined, filename, '666') + }) + + describe('test copyFile', () => { + const src = path.join(os.tmpdir(), 'test-copyFile-src') + const dest = path.join(os.tmpdir(), 'test-copyFile-dst') + + beforeEach(() => { + fs.writeFileSync(src, '') + }) + + afterEach(() => unlink(src, dest)) + + runFsMethodTestThreeWay('copyFile', { vulnerableIndex: 0, ruleEvalCount: 2 }, src, dest) + runFsMethodTestThreeWay('copyFile', { vulnerableIndex: 1, ruleEvalCount: 2 }, src, dest) + }) + + describe('test link', () => { + const src = path.join(os.tmpdir(), 'test-link-src') + const dest = path.join(os.tmpdir(), 'test-link-dst') + + beforeEach(() => { + fs.writeFileSync(src, '') + }) + + afterEach(() => unlink(src, dest)) + + runFsMethodTestThreeWay('copyFile', { vulnerableIndex: 0, ruleEvalCount: 2 }, src, dest) + runFsMethodTestThreeWay('copyFile', { vulnerableIndex: 1, ruleEvalCount: 2 }, src, dest) + }) + + describe('test lstat', () => { + runFsMethodTestThreeWay('lstat', undefined, __filename) + }) + + describe('test mkdir', () => { + const dirname = path.join(os.tmpdir(), 'test-mkdir') + + afterEach(() => { + try { + fs.rmdirSync(dirname) + } catch (e) { + // some ops are blocked + } + }) + runFsMethodTestThreeWay('mkdir', undefined, dirname) + }) + + describe('test mkdtemp', () => { + const dirname = path.join(os.tmpdir(), 'test-mkdtemp') + + runFsMethodTestThreeWay('mkdtemp', { + onfinish: (todelete) => { + try { + fs.rmdirSync(todelete) + } catch (e) { + // some ops are blocked + } + } + }, dirname) + }) + + describe('test open', () => { + runFsMethodTestThreeWay('open', { + onfinish: (fd) => { + if (fd && fd.close) { + fd.close() + } else { + fs.close(fd, () => {}) + } + } + }, __filename, 'r') + }) + + describe('test opendir', () => { + const dirname = path.join(os.tmpdir(), 'test-opendir') + + beforeEach(() => { + fs.mkdirSync(dirname) + }) + + afterEach(() => { + fs.rmdirSync(dirname) + }) + runFsMethodTestThreeWay('opendir', { + onfinish: (dir) => { + dir.close() + } + }, dirname) + }) + + describe('test readdir', () => { + const dirname = path.join(os.tmpdir(), 'test-opendir') + + beforeEach(() => { + fs.mkdirSync(dirname) + }) + + afterEach(() => { + fs.rmdirSync(dirname) + }) + runFsMethodTestThreeWay('readdir', undefined, dirname) + }) + + describe('test readFile', () => { + runFsMethodTestThreeWay('readFile', undefined, __filename) + }) + + describe('test readlink', () => { + const src = path.join(os.tmpdir(), 'test-readlink-src') + const dest = path.join(os.tmpdir(), 'test-readlink-dst') + + beforeEach(() => { + fs.writeFileSync(src, '') + fs.linkSync(src, dest) + }) + + afterEach(() => unlink(src, dest)) + + runFsMethodTestThreeWay('readlink', undefined, dest) + }) + + describe('test realpath', () => { + runFsMethodTestThreeWay('realpath', undefined, __filename) + + runFsMethodTest('test fs.realpath.native method', {}, (args) => { + return new Promise((resolve, reject) => { + require('fs').realpath.native(...args, (err, result) => { + if (err) reject(err) + else resolve(result) + }) + }) + }, __filename) + }) + + describe('test rename', () => { + const src = path.join(os.tmpdir(), 'test-rename-src') + const dest = path.join(os.tmpdir(), 'test-rename-dst') + + beforeEach(() => { + fs.writeFileSync(src, '') + }) + + afterEach(() => unlink(dest)) + + runFsMethodTestThreeWay('rename', { vulnerableIndex: 0, ruleEvalCount: 2 }, src, dest) + runFsMethodTestThreeWay('rename', { vulnerableIndex: 1, ruleEvalCount: 2 }, src, dest) + }) + + describe('test rmdir', () => { + const dirname = path.join(os.tmpdir(), 'test-rmdir') + + beforeEach(() => { + fs.mkdirSync(dirname) + }) + + afterEach(() => { + try { fs.rmdirSync(dirname) } catch (e) {} + }) + + runFsMethodTestThreeWay('rmdir', undefined, dirname) + }) + + describe('test stat', () => { + runFsMethodTestThreeWay('stat', undefined, __filename) + }) + + describe('test symlink', () => { + const src = path.join(os.tmpdir(), 'test-symlink-src') + const dest = path.join(os.tmpdir(), 'test-symlink-dst') + + beforeEach(() => { + fs.writeFileSync(src, '') + }) + + afterEach(() => { + unlink(src, dest) + }) + + runFsMethodTestThreeWay('symlink', { vulnerableIndex: 0, ruleEvalCount: 2 }, src, dest) + runFsMethodTestThreeWay('symlink', { vulnerableIndex: 1, ruleEvalCount: 2 }, src, dest) + }) + + describe('test truncate', () => { + const src = path.join(os.tmpdir(), 'test-truncate-src') + + beforeEach(() => { + fs.writeFileSync(src, 'aaaaaa') + }) + + afterEach(() => unlink(src)) + + runFsMethodTestThreeWay('truncate', undefined, src) + }) + + describe('test unlink', () => { + const src = path.join(os.tmpdir(), 'test-unlink-src') + + beforeEach(() => { + fs.writeFileSync(src, '') + }) + runFsMethodTestThreeWay('unlink', undefined, src) + }) + + describe('test writeFile', () => { + const src = path.join(os.tmpdir(), 'test-writeFile-src') + + afterEach(() => unlink(src)) + + runFsMethodTestThreeWay('writeFile', undefined, src, 'content') + }) + }) + }) + + describe('without express', () => { + let app, server + + before(() => { + return agent.load(['http'], { client: false }) + }) + + before((done) => { + const http = require('http') + server = http.createServer((req, res) => { + if (app) { + app(req, res) + } else { + res.end('end') + } + }) + + appsec.enable(new Config({ + appsec: { + enabled: true, + rules: path.join(__dirname, 'resources', 'lfi_rasp_rules.json'), + rasp: { enabled: true } + } + })) + + server.listen(0, () => { + const port = server.address().port + axios = Axios.create({ + baseURL: `http://localhost:${port}` + }) + + done() + }) + }) + + after(() => { + appsec.disable() + server.close() + return agent.close({ ritmReset: false }) + }) + + it('Should detect threat but not block', async () => { + app = (req, res) => { + try { + require('fs').statSync(req.headers.file) + } catch (e) { + if (e.message === 'DatadogRaspAbortError') { + res.writeHead(500) + } else { + res.writeHead(418) + } + } + res.end('end') + } + + return testBlockingRequest('/', { + headers: { + file: '/test.file' + } + }) + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/rasp/lfi.integration.express.plugin.spec.js b/packages/dd-trace/test/appsec/rasp/lfi.integration.express.plugin.spec.js new file mode 100644 index 00000000000..45dc1cac46f --- /dev/null +++ b/packages/dd-trace/test/appsec/rasp/lfi.integration.express.plugin.spec.js @@ -0,0 +1,69 @@ +'use strict' + +const { createSandbox, FakeAgent, spawnProc } = require('../../../../../integration-tests/helpers') +const getPort = require('get-port') +const path = require('path') +const Axios = require('axios') +const { assert } = require('chai') + +describe('RASP - lfi - integration - sync', () => { + let axios, sandbox, cwd, appPort, appFile, agent, proc + + before(async function () { + this.timeout(60000) + sandbox = await createSandbox( + ['express', 'fs'], + false, + [path.join(__dirname, 'resources')]) + + appPort = await getPort() + cwd = sandbox.folder + appFile = path.join(cwd, 'resources', 'lfi-app', 'index.js') + + axios = Axios.create({ + baseURL: `http://localhost:${appPort}` + }) + }) + + after(async function () { + this.timeout(60000) + await sandbox.remove() + }) + + beforeEach(async () => { + agent = await new FakeAgent().start() + proc = await spawnProc(appFile, { + cwd, + env: { + DD_TRACE_AGENT_PORT: agent.port, + APP_PORT: appPort, + DD_APPSEC_ENABLED: true, + DD_APPSEC_RASP_ENABLED: true, + DD_APPSEC_RULES: path.join(cwd, 'resources', 'lfi_rasp_rules.json') + } + }) + }) + + afterEach(async () => { + proc.kill() + await agent.stop() + }) + + it('should block a sync endpoint getting the error from apm:express:middleware:error', async () => { + try { + await axios.get('/lfi/sync?file=/etc/passwd') + } catch (e) { + if (!e.response) { + throw e + } + + assert.strictEqual(e.response.status, 403) + return await agent.assertMessageReceived(({ headers, payload }) => { + assert.property(payload[0][0].meta, '_dd.appsec.json') + assert.include(payload[0][0].meta['_dd.appsec.json'], '"rasp-lfi-rule-id-1"') + }) + } + + throw new Error('Request should be blocked') + }) +}) diff --git a/packages/dd-trace/test/appsec/rasp/lfi.spec.js b/packages/dd-trace/test/appsec/rasp/lfi.spec.js new file mode 100644 index 00000000000..405311ae0d3 --- /dev/null +++ b/packages/dd-trace/test/appsec/rasp/lfi.spec.js @@ -0,0 +1,144 @@ +'use strict' + +const proxyquire = require('proxyquire') +const { assert } = require('chai') +const { fsOperationStart, incomingHttpRequestStart } = require('../../../src/appsec/channels') +const { FS_OPERATION_PATH } = require('../../../src/appsec/addresses') +const { RASP_MODULE } = require('../../../src/appsec/rasp/fs-plugin') + +describe('RASP - lfi.js', () => { + let waf, datadogCore, lfi, web, blocking, appsecFsPlugin, config + + beforeEach(() => { + datadogCore = { + storage: { + getStore: sinon.stub() + } + } + + waf = { + run: sinon.stub() + } + + web = { + root: sinon.stub() + } + + blocking = { + block: sinon.stub() + } + + appsecFsPlugin = { + enable: sinon.stub(), + disable: sinon.stub() + } + + lfi = proxyquire('../../../src/appsec/rasp/lfi', { + '../../../../datadog-core': datadogCore, + '../waf': waf, + '../../plugins/util/web': web, + '../blocking': blocking, + './fs-plugin': appsecFsPlugin + }) + + config = { + appsec: { + stackTrace: { + enabled: true, + maxStackTraces: 2, + maxDepth: 42 + } + } + } + }) + + afterEach(() => { + sinon.restore() + lfi.disable() + }) + + describe('enable', () => { + it('should subscribe to first http req', () => { + const subscribe = sinon.stub(incomingHttpRequestStart, 'subscribe') + + lfi.enable(config) + + sinon.assert.calledOnce(subscribe) + }) + + it('should enable AppsecFsPlugin after the first request', () => { + const unsubscribe = sinon.stub(incomingHttpRequestStart, 'unsubscribe') + const fsOpSubscribe = sinon.stub(fsOperationStart, 'subscribe') + + lfi.enable(config) + + incomingHttpRequestStart.publish({}) + + sinon.assert.calledOnceWithExactly(appsecFsPlugin.enable, RASP_MODULE) + + assert(fsOpSubscribe.calledAfter(appsecFsPlugin.enable)) + + process.nextTick(() => { + sinon.assert.calledOnce(unsubscribe) + }) + }) + }) + + describe('disable', () => { + it('should disable AppsecFsPlugin', () => { + lfi.enable(config) + + lfi.disable() + sinon.assert.calledOnceWithExactly(appsecFsPlugin.disable, RASP_MODULE) + }) + }) + + describe('analyzeLfi', () => { + const path = '/etc/passwd' + const ctx = { path } + const req = {} + + beforeEach(() => { + lfi.enable(config) + + incomingHttpRequestStart.publish({}) + }) + + it('should analyze lfi for root fs operations', () => { + const fs = { root: true } + datadogCore.storage.getStore.returns({ req, fs }) + + fsOperationStart.publish(ctx) + + const persistent = { [FS_OPERATION_PATH]: path } + sinon.assert.calledOnceWithExactly(waf.run, { persistent }, req, 'lfi') + }) + + it('should NOT analyze lfi for child fs operations', () => { + const fs = {} + datadogCore.storage.getStore.returns({ req, fs }) + + fsOperationStart.publish(ctx) + + sinon.assert.notCalled(waf.run) + }) + + it('should NOT analyze lfi for undefined fs (AppsecFsPlugin disabled)', () => { + const fs = undefined + datadogCore.storage.getStore.returns({ req, fs }) + + fsOperationStart.publish(ctx) + + sinon.assert.notCalled(waf.run) + }) + + it('should NOT analyze lfi for excluded operations', () => { + const fs = { opExcluded: true, root: true } + datadogCore.storage.getStore.returns({ req, fs }) + + fsOperationStart.publish(ctx) + + sinon.assert.notCalled(waf.run) + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/rasp/resources/lfi-app/index.js b/packages/dd-trace/test/appsec/rasp/resources/lfi-app/index.js new file mode 100644 index 00000000000..1beb4d977cb --- /dev/null +++ b/packages/dd-trace/test/appsec/rasp/resources/lfi-app/index.js @@ -0,0 +1,28 @@ +'use strict' + +const tracer = require('dd-trace') +tracer.init({ + flushInterval: 0 +}) + +const express = require('express') +const { readFileSync } = require('fs') + +const app = express() +const port = process.env.APP_PORT || 3000 + +app.get('/lfi/sync', (req, res) => { + let result + try { + result = readFileSync(req.query.file) + } catch (e) { + if (e.message === 'DatadogRaspAbortError') { + throw e + } + } + res.send(result) +}) + +app.listen(port, () => { + process.send({ port }) +}) diff --git a/packages/dd-trace/test/appsec/rasp/resources/lfi_rasp_rules.json b/packages/dd-trace/test/appsec/rasp/resources/lfi_rasp_rules.json new file mode 100644 index 00000000000..814f6c72236 --- /dev/null +++ b/packages/dd-trace/test/appsec/rasp/resources/lfi_rasp_rules.json @@ -0,0 +1,61 @@ +{ + "version": "2.2", + "metadata": { + "rules_version": "1.99.0" + }, + "rules": [ + { + "id": "rasp-lfi-rule-id-1", + "name": "Local file inclusion exploit", + "enabled": true, + "tags": { + "type": "lfi", + "category": "vulnerability_trigger", + "cwe": "22", + "capec": "1000/255/153/126", + "confidence": "0", + "module": "rasp" + }, + "conditions": [ + { + "parameters": { + "resource": [ + { + "address": "server.io.fs.file" + } + ], + "params": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + }, + { + "address": "server.request.headers.no_cookies" + }, + { + "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" + } + ] + }, + "operator": "lfi_detector" + } + ], + "transformers": [], + "on_match": [ + "block", + "stack_trace" + ] + } + ] +} diff --git a/packages/dd-trace/test/appsec/rasp/resources/postgress-app/index.js b/packages/dd-trace/test/appsec/rasp/resources/postgress-app/index.js new file mode 100644 index 00000000000..e60041bfe7c --- /dev/null +++ b/packages/dd-trace/test/appsec/rasp/resources/postgress-app/index.js @@ -0,0 +1,55 @@ +'use strict' + +const tracer = require('dd-trace') +tracer.init({ + flushInterval: 0 +}) + +const express = require('express') +const pg = require('pg') + +const connectionData = { + host: '127.0.0.1', + user: 'postgres', + password: 'postgres', + database: 'postgres', + application_name: 'test' +} + +const pool = new pg.Pool(connectionData) + +const app = express() +const port = process.env.APP_PORT || 3000 + +app.get('/sqli/client/uncaught-promise', async (req, res) => { + const client = new pg.Client(connectionData) + await client.connect() + + try { + await client.query(`SELECT * FROM users WHERE id = '${req.query.param}'`) + } finally { + client.end() + } + + res.end('OK') +}) + +app.get('/sqli/client/uncaught-query-error', async (req, res) => { + const client = new pg.Client(connectionData) + await client.connect() + const query = new pg.Query(`SELECT * FROM users WHERE id = '${req.query.param}'`) + client.query(query) + + query.on('end', () => { + res.end('OK') + }) +}) + +app.get('/sqli/pool/uncaught-promise', async (req, res) => { + await pool.query(`SELECT * FROM users WHERE id = '${req.query.param}'`) + res.end('OK') +}) + +app.listen(port, () => { + process.send({ port }) +}) diff --git a/packages/dd-trace/test/appsec/rasp/resources/rasp_rules.json b/packages/dd-trace/test/appsec/rasp/resources/rasp_rules.json new file mode 100644 index 00000000000..778e4821e73 --- /dev/null +++ b/packages/dd-trace/test/appsec/rasp/resources/rasp_rules.json @@ -0,0 +1,112 @@ +{ + "version": "2.2", + "metadata": { + "rules_version": "1.99.0" + }, + "rules": [ + { + "id": "rasp-ssrf-rule-id-1", + "name": "Server-side request forgery exploit", + "enabled": true, + "tags": { + "type": "ssrf", + "category": "vulnerability_trigger", + "cwe": "918", + "capec": "1000/225/115/664", + "confidence": "0", + "module": "rasp" + }, + "conditions": [ + { + "parameters": { + "resource": [ + { + "address": "server.io.net.url" + } + ], + "params": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "server.request.headers.no_cookies" + }, + { + "address": "grpc.server.request.message" + }, + { + "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" + } + ] + }, + "operator": "ssrf_detector" + } + ], + "transformers": [], + "on_match": [ + "block", + "stack_trace" + ] + }, + { + "id": "rasp-sqli-rule-id-2", + "name": "SQL injection exploit", + "tags": { + "type": "sql_injection", + "category": "vulnerability_trigger", + "cwe": "89", + "capec": "1000/152/248/66", + "confidence": "0", + "module": "rasp" + }, + "conditions": [ + { + "parameters": { + "resource": [ + { + "address": "server.db.statement" + } + ], + "params": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" + } + ], + "db_type": [ + { + "address": "server.db.system" + } + ] + }, + "operator": "sqli_detector" + } + ], + "transformers": [], + "on_match": [ + "block", + "stack_trace" + ] + } + ] +} diff --git a/packages/dd-trace/test/appsec/rasp/sql_injection.integration.pg.plugin.spec.js b/packages/dd-trace/test/appsec/rasp/sql_injection.integration.pg.plugin.spec.js new file mode 100644 index 00000000000..c4b92b3a2f3 --- /dev/null +++ b/packages/dd-trace/test/appsec/rasp/sql_injection.integration.pg.plugin.spec.js @@ -0,0 +1,107 @@ +'use strict' + +const { createSandbox, FakeAgent, spawnProc } = require('../../../../../integration-tests/helpers') +const getPort = require('get-port') +const path = require('path') +const Axios = require('axios') +const { assert } = require('chai') + +// These test are here and not in the integration tests +// because they require postgres instance +describe('RASP - sql_injection - integration', () => { + let axios, sandbox, cwd, appPort, appFile, agent, proc + + before(async function () { + this.timeout(60000) + sandbox = await createSandbox( + ['express', 'pg'], + false, + [path.join(__dirname, 'resources')]) + + appPort = await getPort() + cwd = sandbox.folder + appFile = path.join(cwd, 'resources', 'postgress-app', 'index.js') + + axios = Axios.create({ + baseURL: `http://localhost:${appPort}` + }) + }) + + after(async function () { + this.timeout(60000) + await sandbox.remove() + }) + + beforeEach(async () => { + agent = await new FakeAgent().start() + proc = await spawnProc(appFile, { + cwd, + env: { + DD_TRACE_AGENT_PORT: agent.port, + APP_PORT: appPort, + DD_APPSEC_ENABLED: true, + DD_APPSEC_RASP_ENABLED: true, + DD_APPSEC_RULES: path.join(cwd, 'resources', 'rasp_rules.json') + } + }) + }) + + afterEach(async () => { + proc.kill() + await agent.stop() + }) + + it('should block using pg.Client and unhandled promise', async () => { + try { + await axios.get('/sqli/client/uncaught-promise?param=\' OR 1 = 1 --') + } catch (e) { + if (!e.response) { + throw e + } + + assert.strictEqual(e.response.status, 403) + return await agent.assertMessageReceived(({ headers, payload }) => { + assert.property(payload[0][0].meta, '_dd.appsec.json') + assert.include(payload[0][0].meta['_dd.appsec.json'], '"rasp-sqli-rule-id-2"') + }) + } + + throw new Error('Request should be blocked') + }) + + it('should block using pg.Client and unhandled query object', async () => { + try { + await axios.get('/sqli/client/uncaught-query-error?param=\' OR 1 = 1 --') + } catch (e) { + if (!e.response) { + throw e + } + + assert.strictEqual(e.response.status, 403) + return await agent.assertMessageReceived(({ headers, payload }) => { + assert.property(payload[0][0].meta, '_dd.appsec.json') + assert.include(payload[0][0].meta['_dd.appsec.json'], '"rasp-sqli-rule-id-2"') + }) + } + + throw new Error('Request should be blocked') + }) + + it('should block using pg.Pool and unhandled promise', async () => { + try { + await axios.get('/sqli/pool/uncaught-promise?param=\' OR 1 = 1 --') + } catch (e) { + if (!e.response) { + throw e + } + + assert.strictEqual(e.response.status, 403) + return await agent.assertMessageReceived(({ headers, payload }) => { + assert.property(payload[0][0].meta, '_dd.appsec.json') + assert.include(payload[0][0].meta['_dd.appsec.json'], '"rasp-sqli-rule-id-2"') + }) + } + + throw new Error('Request should be blocked') + }) +}) diff --git a/packages/dd-trace/test/appsec/rasp/sql_injection.mysql2.plugin.spec.js b/packages/dd-trace/test/appsec/rasp/sql_injection.mysql2.plugin.spec.js new file mode 100644 index 00000000000..2fe74e9f262 --- /dev/null +++ b/packages/dd-trace/test/appsec/rasp/sql_injection.mysql2.plugin.spec.js @@ -0,0 +1,229 @@ +'use strict' + +const agent = require('../../plugins/agent') +const appsec = require('../../../src/appsec') +const Config = require('../../../src/config') +const path = require('path') +const Axios = require('axios') +const { assert } = require('chai') +const { checkRaspExecutedAndNotThreat, checkRaspExecutedAndHasThreat } = require('./utils') + +describe('RASP - sql_injection', () => { + withVersions('mysql2', 'express', expressVersion => { + withVersions('mysql2', 'mysql2', mysql2Version => { + describe('sql injection with mysql2', () => { + const connectionData = { + host: '127.0.0.1', + user: 'root', + database: 'db' + } + let server, axios, app, mysql2 + + before(() => { + return agent.load(['express', 'http', 'mysql2'], { client: false }) + }) + + before(done => { + const express = require(`../../../../../versions/express@${expressVersion}`).get() + mysql2 = require(`../../../../../versions/mysql2@${mysql2Version}`).get() + const expressApp = express() + + expressApp.get('/', (req, res) => { + app(req, res) + }) + + appsec.enable(new Config({ + appsec: { + enabled: true, + rules: path.join(__dirname, 'resources', 'rasp_rules.json'), + rasp: { enabled: true } + } + })) + + server = expressApp.listen(0, () => { + const port = server.address().port + axios = Axios.create({ + baseURL: `http://localhost:${port}` + }) + done() + }) + }) + + after(() => { + appsec.disable() + server.close() + return agent.close({ ritmReset: false }) + }) + + describe('Test using Connection', () => { + let connection + + beforeEach(() => { + connection = mysql2.createConnection(connectionData) + connection.connect() + }) + + afterEach((done) => { + connection.end(() => done()) + }) + + describe('query', () => { + it('Should not detect threat', async () => { + app = (req, res) => { + connection.query('SELECT ' + req.query.param, (err) => { + if (err) { + res.statusCode = 500 + } + + res.end() + }) + } + + axios.get('/?param=1') + + await checkRaspExecutedAndNotThreat(agent) + }) + + it('Should block query with callback', async () => { + app = (req, res) => { + connection.query(`SELECT * FROM users WHERE id='${req.query.param}'`, (err) => { + if (err?.name === 'DatadogRaspAbortError') { + res.statusCode = 500 + } + res.end() + }) + } + + try { + await axios.get('/?param=\' OR 1 = 1 --') + } catch (e) { + return await checkRaspExecutedAndHasThreat(agent, 'rasp-sqli-rule-id-2') + } + + assert.fail('Request should be blocked') + }) + }) + + describe('execute', () => { + it('Should not detect threat', async () => { + app = (req, res) => { + connection.execute('SELECT ' + req.query.param, (err) => { + if (err) { + res.statusCode = 500 + } + + res.end() + }) + } + + axios.get('/?param=1') + + await checkRaspExecutedAndNotThreat(agent) + }) + + it('Should block query with callback', async () => { + app = (req, res) => { + connection.execute(`SELECT * FROM users WHERE id='${req.query.param}'`, (err) => { + if (err?.name === 'DatadogRaspAbortError') { + res.statusCode = 500 + } + res.end() + }) + } + + try { + await axios.get('/?param=\' OR 1 = 1 --') + } catch (e) { + return await checkRaspExecutedAndHasThreat(agent, 'rasp-sqli-rule-id-2') + } + + assert.fail('Request should be blocked') + }) + }) + }) + + describe('Test using Pool', () => { + let pool + + beforeEach(() => { + pool = mysql2.createPool(connectionData) + }) + + describe('query', () => { + it('Should not detect threat', async () => { + app = (req, res) => { + pool.query('SELECT ' + req.query.param, (err) => { + if (err) { + res.statusCode = 500 + } + + res.end() + }) + } + + axios.get('/?param=1') + + await checkRaspExecutedAndNotThreat(agent) + }) + + it('Should block query with callback', async () => { + app = (req, res) => { + pool.query(`SELECT * FROM users WHERE id='${req.query.param}'`, (err) => { + if (err?.name === 'DatadogRaspAbortError') { + res.statusCode = 500 + } + res.end() + }) + } + + try { + await axios.get('/?param=\' OR 1 = 1 --') + } catch (e) { + return await checkRaspExecutedAndHasThreat(agent, 'rasp-sqli-rule-id-2') + } + + assert.fail('Request should be blocked') + }) + }) + + describe('execute', () => { + it('Should not detect threat', async () => { + app = (req, res) => { + pool.execute('SELECT ' + req.query.param, (err) => { + if (err) { + res.statusCode = 500 + } + + res.end() + }) + } + + axios.get('/?param=1') + + await checkRaspExecutedAndNotThreat(agent) + }) + + it('Should block query with callback', async () => { + app = (req, res) => { + pool.execute(`SELECT * FROM users WHERE id='${req.query.param}'`, (err) => { + if (err?.name === 'DatadogRaspAbortError') { + res.statusCode = 500 + } + res.end() + }) + } + + try { + await axios.get('/?param=\' OR 1 = 1 --') + } catch (e) { + return await checkRaspExecutedAndHasThreat(agent, 'rasp-sqli-rule-id-2') + } + + assert.fail('Request should be blocked') + }) + }) + }) + }) + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/rasp/sql_injection.pg.plugin.spec.js b/packages/dd-trace/test/appsec/rasp/sql_injection.pg.plugin.spec.js new file mode 100644 index 00000000000..8f05158c22d --- /dev/null +++ b/packages/dd-trace/test/appsec/rasp/sql_injection.pg.plugin.spec.js @@ -0,0 +1,286 @@ +'use strict' + +const agent = require('../../plugins/agent') +const appsec = require('../../../src/appsec') +const { wafRunFinished } = require('../../../src/appsec/channels') +const addresses = require('../../../src/appsec/addresses') +const Config = require('../../../src/config') +const path = require('path') +const Axios = require('axios') +const { assert } = require('chai') +const { checkRaspExecutedAndNotThreat, checkRaspExecutedAndHasThreat } = require('./utils') + +describe('RASP - sql_injection', () => { + withVersions('pg', 'express', expressVersion => { + withVersions('pg', 'pg', pgVersion => { + describe('sql injection with pg', () => { + const connectionData = { + host: '127.0.0.1', + user: 'postgres', + password: 'postgres', + database: 'postgres', + application_name: 'test' + } + let server, axios, app, pg + + before(() => { + return agent.load(['express', 'http', 'pg'], { client: false }) + }) + + before(done => { + const express = require(`../../../../../versions/express@${expressVersion}`).get() + pg = require(`../../../../../versions/pg@${pgVersion}`).get() + const expressApp = express() + + expressApp.get('/', (req, res) => { + app(req, res) + }) + + appsec.enable(new Config({ + appsec: { + enabled: true, + rules: path.join(__dirname, 'resources', 'rasp_rules.json'), + rasp: { enabled: true } + } + })) + + server = expressApp.listen(0, () => { + const port = server.address().port + axios = Axios.create({ + baseURL: `http://localhost:${port}` + }) + done() + }) + }) + + after(() => { + appsec.disable() + server.close() + return agent.close({ ritmReset: false }) + }) + + describe('Test using pg.Client', () => { + let client + + beforeEach((done) => { + client = new pg.Client(connectionData) + client.connect(err => done(err)) + }) + + afterEach(() => { + client.end() + }) + + it('Should not detect threat', async () => { + app = (req, res) => { + client.query('SELECT ' + req.query.param, (err) => { + if (err) { + res.statusCode = 500 + } + + res.end() + }) + } + + axios.get('/?param=1') + + await checkRaspExecutedAndNotThreat(agent) + }) + + it('Should block query with callback', async () => { + app = (req, res) => { + client.query(`SELECT * FROM users WHERE id='${req.query.param}'`, (err) => { + if (err?.name === 'DatadogRaspAbortError') { + res.statusCode = 500 + } + res.end() + }) + } + + try { + await axios.get('/?param=\' OR 1 = 1 --') + } catch (e) { + return await checkRaspExecutedAndHasThreat(agent, 'rasp-sqli-rule-id-2') + } + + assert.fail('Request should be blocked') + }) + + it('Should block query with promise', async () => { + app = async (req, res) => { + try { + await client.query(`SELECT * FROM users WHERE id = '${req.query.param}'`) + } catch (err) { + if (err?.name === 'DatadogRaspAbortError') { + res.statusCode = 500 + } + res.end() + } + } + + try { + await axios.get('/?param=\' OR 1 = 1 --') + } catch (e) { + return checkRaspExecutedAndHasThreat(agent, 'rasp-sqli-rule-id-2') + } + + assert.fail('Request should be blocked') + }) + }) + + describe('Test using pg.Pool', () => { + let pool + + beforeEach(() => { + pool = new pg.Pool(connectionData) + }) + + it('Should not detect threat', async () => { + app = (req, res) => { + pool.query('SELECT ' + req.query.param, (err) => { + if (err) { + res.statusCode = 500 + } + + res.end() + }) + } + + axios.get('/?param=1') + + await checkRaspExecutedAndNotThreat(agent) + }) + + it('Should block query with callback', async () => { + app = (req, res) => { + pool.query(`SELECT * FROM users WHERE id='${req.query.param}'`, (err) => { + if (err?.name === 'DatadogRaspAbortError') { + res.statusCode = 500 + } + res.end() + }) + } + + try { + await axios.get('/?param=\' OR 1 = 1 --') + } catch (e) { + return checkRaspExecutedAndHasThreat(agent, 'rasp-sqli-rule-id-2') + } + + assert.fail('Request should be blocked') + }) + + it('Should block query with promise', async () => { + app = async (req, res) => { + try { + await pool.query(`SELECT * FROM users WHERE id = '${req.query.param}'`) + } catch (err) { + if (err?.name === 'DatadogRaspAbortError') { + res.statusCode = 500 + } + res.end() + } + } + + try { + await axios.get('/?param=\' OR 1 = 1 --') + } catch (e) { + return checkRaspExecutedAndHasThreat(agent, 'rasp-sqli-rule-id-2') + } + + assert.fail('Request should be blocked') + }) + + describe('double calls', () => { + const WAFContextWrapper = require('../../../src/appsec/waf/waf_context_wrapper') + let run + + beforeEach(() => { + run = sinon.spy(WAFContextWrapper.prototype, 'run') + }) + + afterEach(() => { + sinon.restore() + }) + + async function runQueryAndIgnoreError (query) { + try { + await pool.query(query) + } catch (err) { + // do nothing + } + } + + it('should call to waf only once for sql injection using pg Pool', async () => { + app = async (req, res) => { + await runQueryAndIgnoreError('SELECT 1') + res.end() + } + + await axios.get('/') + + assert.equal(run.args.filter(arg => arg[1] === 'sql_injection').length, 1) + }) + + it('should call to waf twice for sql injection with two different queries in pg Pool', async () => { + app = async (req, res) => { + await runQueryAndIgnoreError('SELECT 1') + await runQueryAndIgnoreError('SELECT 2') + + res.end() + } + + await axios.get('/') + + assert.equal(run.args.filter(arg => arg[1] === 'sql_injection').length, 2) + }) + + it('should call to waf twice for sql injection and same query when input address is updated', async () => { + app = async (req, res) => { + await runQueryAndIgnoreError('SELECT 1') + + wafRunFinished.publish({ + payload: { + persistent: { + [addresses.HTTP_INCOMING_URL]: 'test' + } + } + }) + + await runQueryAndIgnoreError('SELECT 1') + + res.end() + } + + await axios.get('/') + + assert.equal(run.args.filter(arg => arg[1] === 'sql_injection').length, 2) + }) + + it('should call to waf once for sql injection and same query when input address is updated', async () => { + app = async (req, res) => { + await runQueryAndIgnoreError('SELECT 1') + + wafRunFinished.publish({ + payload: { + persistent: { + 'not-an-input': 'test' + } + } + }) + + await runQueryAndIgnoreError('SELECT 1') + + res.end() + } + + await axios.get('/') + + assert.equal(run.args.filter(arg => arg[1] === 'sql_injection').length, 1) + }) + }) + }) + }) + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/rasp/sql_injection.spec.js b/packages/dd-trace/test/appsec/rasp/sql_injection.spec.js new file mode 100644 index 00000000000..d713521e986 --- /dev/null +++ b/packages/dd-trace/test/appsec/rasp/sql_injection.spec.js @@ -0,0 +1,181 @@ +'use strict' + +const { pgQueryStart, mysql2OuterQueryStart } = require('../../../src/appsec/channels') +const addresses = require('../../../src/appsec/addresses') +const proxyquire = require('proxyquire') + +describe('RASP - sql_injection', () => { + let waf, datadogCore, sqli + + beforeEach(() => { + datadogCore = { + storage: { + getStore: sinon.stub() + } + } + + waf = { + run: sinon.stub() + } + + sqli = proxyquire('../../../src/appsec/rasp/sql_injection', { + '../../../../datadog-core': datadogCore, + '../waf': waf + }) + + const config = { + appsec: { + stackTrace: { + enabled: true, + maxStackTraces: 2, + maxDepth: 42 + } + } + } + + sqli.enable(config) + }) + + afterEach(() => { + sinon.restore() + sqli.disable() + }) + + describe('analyzePgSqlInjection', () => { + it('should analyze sql injection', () => { + const ctx = { + query: { + text: 'SELECT 1' + } + } + const req = {} + datadogCore.storage.getStore.returns({ req }) + + pgQueryStart.publish(ctx) + + const persistent = { + [addresses.DB_STATEMENT]: 'SELECT 1', + [addresses.DB_SYSTEM]: 'postgresql' + } + sinon.assert.calledOnceWithExactly(waf.run, { persistent }, req, 'sql_injection') + }) + + it('should not analyze sql injection if rasp is disabled', () => { + sqli.disable() + + const ctx = { + query: { + text: 'SELECT 1' + } + } + const req = {} + datadogCore.storage.getStore.returns({ req }) + + pgQueryStart.publish(ctx) + + sinon.assert.notCalled(waf.run) + }) + + it('should not analyze sql injection if no store', () => { + const ctx = { + query: { + text: 'SELECT 1' + } + } + datadogCore.storage.getStore.returns(undefined) + + pgQueryStart.publish(ctx) + + sinon.assert.notCalled(waf.run) + }) + + it('should not analyze sql injection if no req', () => { + const ctx = { + query: { + text: 'SELECT 1' + } + } + datadogCore.storage.getStore.returns({}) + + pgQueryStart.publish(ctx) + + sinon.assert.notCalled(waf.run) + }) + + it('should not analyze sql injection if no query', () => { + const ctx = { + query: {} + } + datadogCore.storage.getStore.returns({}) + + pgQueryStart.publish(ctx) + + sinon.assert.notCalled(waf.run) + }) + }) + + describe('analyzeMysql2SqlInjection', () => { + it('should analyze sql injection', () => { + const ctx = { + sql: 'SELECT 1' + } + const req = {} + datadogCore.storage.getStore.returns({ req }) + + mysql2OuterQueryStart.publish(ctx) + + const persistent = { + [addresses.DB_STATEMENT]: 'SELECT 1', + [addresses.DB_SYSTEM]: 'mysql' + } + sinon.assert.calledOnceWithExactly(waf.run, { persistent }, req, 'sql_injection') + }) + + it('should not analyze sql injection if rasp is disabled', () => { + sqli.disable() + + const ctx = { + sql: 'SELECT 1' + } + const req = {} + datadogCore.storage.getStore.returns({ req }) + + mysql2OuterQueryStart.publish(ctx) + + sinon.assert.notCalled(waf.run) + }) + + it('should not analyze sql injection if no store', () => { + const ctx = { + sql: 'SELECT 1' + } + datadogCore.storage.getStore.returns(undefined) + + mysql2OuterQueryStart.publish(ctx) + + sinon.assert.notCalled(waf.run) + }) + + it('should not analyze sql injection if no req', () => { + const ctx = { + sql: 'SELECT 1' + } + datadogCore.storage.getStore.returns({}) + + mysql2OuterQueryStart.publish(ctx) + + sinon.assert.notCalled(waf.run) + }) + + it('should not analyze sql injection if no query', () => { + const ctx = { + sql: 'SELECT 1' + } + datadogCore.storage.getStore.returns({}) + + mysql2OuterQueryStart.publish(ctx) + + sinon.assert.notCalled(waf.run) + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/rasp/ssrf.express.plugin.spec.js b/packages/dd-trace/test/appsec/rasp/ssrf.express.plugin.spec.js new file mode 100644 index 00000000000..6b5ba45ad0a --- /dev/null +++ b/packages/dd-trace/test/appsec/rasp/ssrf.express.plugin.spec.js @@ -0,0 +1,285 @@ +'use strict' + +const Axios = require('axios') +const agent = require('../../plugins/agent') +const appsec = require('../../../src/appsec') +const Config = require('../../../src/config') +const path = require('path') +const { assert } = require('chai') +const { checkRaspExecutedAndNotThreat, checkRaspExecutedAndHasThreat } = require('./utils') + +function noop () {} + +describe('RASP - ssrf', () => { + withVersions('express', 'express', expressVersion => { + let app, server, axios + + before(() => { + return agent.load(['express', 'http'], { client: false }) + }) + + before((done) => { + const express = require(`../../../../../versions/express@${expressVersion}`).get() + const expressApp = express() + + expressApp.get('/', (req, res) => { + app(req, res) + }) + + appsec.enable(new Config({ + appsec: { + enabled: true, + rules: path.join(__dirname, 'resources', 'rasp_rules.json'), + rasp: { enabled: true } + } + })) + + server = expressApp.listen(0, () => { + const port = server.address().port + axios = Axios.create({ + baseURL: `http://localhost:${port}` + }) + done() + }) + }) + + after(() => { + appsec.disable() + server.close() + return agent.close({ ritmReset: false }) + }) + + describe('ssrf', () => { + async function testBlockingRequest () { + try { + await axios.get('/?host=localhost/ifconfig.pro') + } catch (e) { + if (!e.response) { + throw e + } + + return checkRaspExecutedAndHasThreat(agent, 'rasp-ssrf-rule-id-1') + } + + assert.fail('Request should be blocked') + } + + ['http', 'https'].forEach(protocol => { + describe(`Test using ${protocol}`, () => { + it('Should not detect threat', async () => { + app = (req, res) => { + const clientRequest = require(protocol).get(`${protocol}://${req.query.host}`) + clientRequest.on('error', noop) + res.end('end') + } + + axios.get('/?host=www.datadoghq.com') + + return checkRaspExecutedAndNotThreat(agent) + }) + + it('Should detect threat doing a GET request', async () => { + app = (req, res) => { + const clientRequest = require(protocol).get(`${protocol}://${req.query.host}`) + clientRequest.on('error', (e) => { + if (e.message === 'DatadogRaspAbortError') { + res.writeHead(500) + } + res.end('end') + }) + } + + await testBlockingRequest() + }) + + it('Should detect threat doing a POST request', async () => { + app = (req, res) => { + const clientRequest = require(protocol) + .request(`${protocol}://${req.query.host}`, { method: 'POST' }) + clientRequest.write('dummy_post_data') + clientRequest.end() + clientRequest.on('error', (e) => { + if (e.message === 'DatadogRaspAbortError') { + res.writeHead(500) + } + res.end('end') + }) + } + + await testBlockingRequest() + }) + }) + }) + + describe('Test using axios', () => { + withVersions('express', 'axios', axiosVersion => { + let axiosToTest + + beforeEach(() => { + axiosToTest = require(`../../../../../versions/axios@${axiosVersion}`).get() + }) + + it('Should not detect threat', async () => { + app = (req, res) => { + axiosToTest.get(`https://${req.query.host}`) + res.end('end') + } + + axios.get('/?host=www.datadoghq.com') + + return checkRaspExecutedAndNotThreat(agent) + }) + + it('Should detect threat doing a GET request', async () => { + app = async (req, res) => { + try { + await axiosToTest.get(`https://${req.query.host}`) + res.end('end') + } catch (e) { + if (e.cause.message === 'DatadogRaspAbortError') { + res.writeHead(500) + } + res.end('end') + } + } + + await testBlockingRequest() + }) + + it('Should detect threat doing a POST request', async () => { + app = async (req, res) => { + try { + await axiosToTest.post(`https://${req.query.host}`, { key: 'value' }) + } catch (e) { + if (e.cause.message === 'DatadogRaspAbortError') { + res.writeHead(500) + } + res.end('end') + } + } + + await testBlockingRequest() + }) + }) + }) + + describe('Test using request', () => { + withVersions('express', 'request', requestVersion => { + let requestToTest + + beforeEach(() => { + requestToTest = require(`../../../../../versions/request@${requestVersion}`).get() + }) + + it('Should not detect threat', async () => { + app = (req, res) => { + requestToTest.get(`https://${req.query.host}`).on('response', () => { + res.end('end') + }) + } + + axios.get('/?host=www.datadoghq.com') + + return checkRaspExecutedAndNotThreat(agent) + }) + + it('Should detect threat doing a GET request', async () => { + app = async (req, res) => { + try { + requestToTest.get(`https://${req.query.host}`) + .on('error', (e) => { + if (e.message === 'DatadogRaspAbortError') { + res.writeHead(500) + } + res.end('end') + }) + } catch (e) { + if (e.cause.message === 'DatadogRaspAbortError') { + res.writeHead(500) + } + res.end('end') + } + } + + await testBlockingRequest() + }) + }) + }) + }) + }) + + describe('without express', () => { + let app, server, axios + + before(() => { + return agent.load(['http'], { client: false }) + }) + + before((done) => { + const http = require('http') + server = http.createServer((req, res) => { + if (app) { + app(req, res) + } else { + res.end('end') + } + }) + + appsec.enable(new Config({ + appsec: { + enabled: true, + rules: path.join(__dirname, 'resources', 'rasp_rules.json'), + rasp: { enabled: true } + } + })) + + server.listen(0, () => { + const port = server.address().port + axios = Axios.create({ + baseURL: `http://localhost:${port}` + }) + + done() + }) + }) + + after(() => { + appsec.disable() + server.close() + return agent.close({ ritmReset: false }) + }) + + it('Should detect threat without blocking doing a GET request', async () => { + app = (req, res) => { + const clientRequest = require('http').get(`http://${req.headers.host}`, { timeout: 10 }, function () { + res.end('end') + }) + + clientRequest.on('timeout', () => { + res.writeHead(200) + res.end('timeout') + }) + + clientRequest.on('error', (e) => { + if (e.name !== 'DatadogRaspAbortError') { + res.writeHead(200) + res.end('not-blocking-error') + } else { + res.writeHead(500) + res.end('unexpected-blocking-error') + } + }) + } + + const response = await axios.get('/', { + headers: { + host: 'localhost/ifconfig.pro' + } + }) + + assert.equal(response.status, 200) + + return checkRaspExecutedAndHasThreat(agent, 'rasp-ssrf-rule-id-1') + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/rasp/ssrf.spec.js b/packages/dd-trace/test/appsec/rasp/ssrf.spec.js new file mode 100644 index 00000000000..c40867ea254 --- /dev/null +++ b/packages/dd-trace/test/appsec/rasp/ssrf.spec.js @@ -0,0 +1,112 @@ +'use strict' + +const proxyquire = require('proxyquire') +const { httpClientRequestStart } = require('../../../src/appsec/channels') +const addresses = require('../../../src/appsec/addresses') + +describe('RASP - ssrf.js', () => { + let waf, datadogCore, ssrf + + beforeEach(() => { + datadogCore = { + storage: { + getStore: sinon.stub() + } + } + + waf = { + run: sinon.stub() + } + + ssrf = proxyquire('../../../src/appsec/rasp/ssrf', { + '../../../../datadog-core': datadogCore, + '../waf': waf + }) + + const config = { + appsec: { + stackTrace: { + enabled: true, + maxStackTraces: 2, + maxDepth: 42 + } + } + } + + ssrf.enable(config) + }) + + afterEach(() => { + sinon.restore() + ssrf.disable() + }) + + describe('analyzeSsrf', () => { + it('should analyze ssrf', () => { + const ctx = { + args: { + uri: 'http://example.com' + } + } + const req = {} + datadogCore.storage.getStore.returns({ req }) + + httpClientRequestStart.publish(ctx) + + const persistent = { [addresses.HTTP_OUTGOING_URL]: 'http://example.com' } + sinon.assert.calledOnceWithExactly(waf.run, { persistent }, req, 'ssrf') + }) + + it('should not analyze ssrf if rasp is disabled', () => { + ssrf.disable() + const ctx = { + args: { + uri: 'http://example.com' + } + } + const req = {} + datadogCore.storage.getStore.returns({ req }) + + httpClientRequestStart.publish(ctx) + + sinon.assert.notCalled(waf.run) + }) + + it('should not analyze ssrf if no store', () => { + const ctx = { + args: { + uri: 'http://example.com' + } + } + datadogCore.storage.getStore.returns(undefined) + + httpClientRequestStart.publish(ctx) + + sinon.assert.notCalled(waf.run) + }) + + it('should not analyze ssrf if no req', () => { + const ctx = { + args: { + uri: 'http://example.com' + } + } + datadogCore.storage.getStore.returns({}) + + httpClientRequestStart.publish(ctx) + + sinon.assert.notCalled(waf.run) + }) + + it('should not analyze ssrf if no url', () => { + const ctx = { + args: {} + } + datadogCore.storage.getStore.returns({}) + + httpClientRequestStart.publish(ctx) + + sinon.assert.notCalled(waf.run) + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/rasp/utils.js b/packages/dd-trace/test/appsec/rasp/utils.js new file mode 100644 index 00000000000..0d8a3e076a4 --- /dev/null +++ b/packages/dd-trace/test/appsec/rasp/utils.js @@ -0,0 +1,45 @@ +'use strict' + +const { assert } = require('chai') + +function getWebSpan (traces) { + for (const trace of traces) { + for (const span of trace) { + if (span.type === 'web') { + return span + } + } + } + throw new Error('web span not found') +} + +function checkRaspExecutedAndNotThreat (agent, checkRuleEval = true) { + return agent.use((traces) => { + const span = getWebSpan(traces) + assert.notProperty(span.meta, '_dd.appsec.json') + assert.notProperty(span.meta_struct || {}, '_dd.stack') + if (checkRuleEval) { + assert.equal(span.metrics['_dd.appsec.rasp.rule.eval'], 1) + } + }) +} + +function checkRaspExecutedAndHasThreat (agent, ruleId, ruleEvalCount = 1) { + return agent.use((traces) => { + const span = getWebSpan(traces) + assert.property(span.meta, '_dd.appsec.json') + assert(span.meta['_dd.appsec.json'].includes(ruleId)) + assert.equal(span.metrics['_dd.appsec.rasp.rule.eval'], ruleEvalCount) + assert(span.metrics['_dd.appsec.rasp.duration'] > 0) + assert(span.metrics['_dd.appsec.rasp.duration_ext'] > 0) + assert.property(span.meta_struct, '_dd.stack') + + return span + }) +} + +module.exports = { + getWebSpan, + checkRaspExecutedAndNotThreat, + checkRaspExecutedAndHasThreat +} diff --git a/packages/dd-trace/test/appsec/rasp/utils.spec.js b/packages/dd-trace/test/appsec/rasp/utils.spec.js new file mode 100644 index 00000000000..255f498a117 --- /dev/null +++ b/packages/dd-trace/test/appsec/rasp/utils.spec.js @@ -0,0 +1,79 @@ +'use strict' + +const proxyquire = require('proxyquire') + +describe('RASP - utils.js', () => { + let web, utils, stackTrace, config + + beforeEach(() => { + web = { + root: sinon.stub() + } + + stackTrace = { + reportStackTrace: sinon.stub() + } + + utils = proxyquire('../../../src/appsec/rasp/utils', { + '../../plugins/util/web': web, + '../stack_trace': stackTrace + }) + + config = { + appsec: { + stackTrace: { + enabled: true, + maxStackTraces: 2, + maxDepth: 42 + } + } + } + }) + + describe('handleResult', () => { + it('should report stack trace when generate_stack action is present in waf result', () => { + const req = {} + const rootSpan = {} + const stackId = 'test_stack_id' + const result = { + generate_stack: { + stack_id: stackId + } + } + + web.root.returns(rootSpan) + + utils.handleResult(result, req, undefined, undefined, config) + sinon.assert.calledOnceWithExactly(stackTrace.reportStackTrace, rootSpan, stackId, 42, 2) + }) + + it('should not report stack trace when no action is present in waf result', () => { + const req = {} + const result = {} + + utils.handleResult(result, req, undefined, undefined, config) + sinon.assert.notCalled(stackTrace.reportStackTrace) + }) + + it('should not report stack trace when stack trace reporting is disabled', () => { + const req = {} + const result = { + generate_stack: { + stack_id: 'stackId' + } + } + const config = { + appsec: { + stackTrace: { + enabled: false, + maxStackTraces: 2, + maxDepth: 42 + } + } + } + + utils.handleResult(result, req, undefined, undefined, config) + sinon.assert.notCalled(stackTrace.reportStackTrace) + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/remote_config/index.spec.js b/packages/dd-trace/test/appsec/remote_config/index.spec.js index 9712463923f..dbd710d6a4e 100644 --- a/packages/dd-trace/test/appsec/remote_config/index.spec.js +++ b/packages/dd-trace/test/appsec/remote_config/index.spec.js @@ -9,6 +9,7 @@ let RemoteConfigManager let RuleManager let appsec let remoteConfig +let apiSecuritySampler describe('Remote Config index', () => { beforeEach(() => { @@ -21,7 +22,9 @@ describe('Remote Config index', () => { rc = { updateCapabilities: sinon.spy(), on: sinon.spy(), - off: sinon.spy() + off: sinon.spy(), + setProductHandler: sinon.spy(), + removeProductHandler: sinon.spy() } RemoteConfigManager = sinon.stub().returns(rc) @@ -30,6 +33,11 @@ describe('Remote Config index', () => { updateWafFromRC: sinon.stub() } + apiSecuritySampler = { + configure: sinon.stub(), + setRequestSampling: sinon.stub() + } + appsec = { enable: sinon.spy(), disable: sinon.spy() @@ -38,6 +46,7 @@ describe('Remote Config index', () => { remoteConfig = proxyquire('../src/appsec/remote_config', { './manager': RemoteConfigManager, '../rule_manager': RuleManager, + '../api_security_sampler': apiSecuritySampler, '..': appsec }) }) @@ -49,19 +58,52 @@ describe('Remote Config index', () => { remoteConfig.enable(config) expect(RemoteConfigManager).to.have.been.calledOnceWithExactly(config) - expect(rc.updateCapabilities).to.have.been.calledOnceWithExactly(RemoteConfigCapabilities.ASM_ACTIVATION, true) - expect(rc.on).to.have.been.calledOnceWith('ASM_FEATURES') - expect(rc.on.firstCall.args[1]).to.be.a('function') + expect(rc.updateCapabilities).to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_ACTIVATION, true) + expect(rc.setProductHandler).to.have.been.calledWith('ASM_FEATURES') + expect(rc.setProductHandler.firstCall.args[1]).to.be.a('function') + }) + + it('should listen to remote config when appsec is explicitly configured as enabled=true', () => { + config.appsec = { enabled: true } + + remoteConfig.enable(config) + + expect(RemoteConfigManager).to.have.been.calledOnceWithExactly(config) + expect(rc.updateCapabilities).to.not.have.been.calledWith('ASM_ACTIVATION') + expect(rc.setProductHandler).to.have.been.calledOnceWith('ASM_FEATURES') + expect(rc.setProductHandler.firstCall.args[1]).to.be.a('function') }) - it('should not listen to remote config when appsec is explicitly configured', () => { + it('should not listen to remote config when appsec is explicitly configured as enabled=false', () => { config.appsec = { enabled: false } remoteConfig.enable(config) expect(RemoteConfigManager).to.have.been.calledOnceWithExactly(config) - expect(rc.updateCapabilities).to.not.have.been.called - expect(rc.on).to.not.have.been.called + expect(rc.updateCapabilities).to.not.have.been.calledWith(RemoteConfigCapabilities.ASM_ACTIVATION, true) + expect(rc.setProductHandler).to.not.have.been.called + }) + + it('should listen ASM_API_SECURITY_SAMPLE_RATE when appsec.enabled=undefined and appSecurity.enabled=true', () => { + config.appsec = { enabled: undefined, apiSecurity: { enabled: true } } + + remoteConfig.enable(config) + + expect(RemoteConfigManager).to.have.been.calledOnceWithExactly(config) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_ACTIVATION, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_API_SECURITY_SAMPLE_RATE, true) + }) + + it('should listen ASM_API_SECURITY_SAMPLE_RATE when appsec.enabled=true and appSecurity.enabled=true', () => { + config.appsec = { enabled: true, apiSecurity: { enabled: true } } + + remoteConfig.enable(config) + + expect(RemoteConfigManager).to.have.been.calledOnceWithExactly(config) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_API_SECURITY_SAMPLE_RATE, true) }) describe('ASM_FEATURES remote config listener', () => { @@ -70,9 +112,9 @@ describe('Remote Config index', () => { beforeEach(() => { config.appsec = { enabled: undefined } - remoteConfig.enable(config) + remoteConfig.enable(config, appsec) - listener = rc.on.firstCall.args[1] + listener = rc.setProductHandler.firstCall.args[1] }) it('should enable appsec when listener is called with apply and enabled', () => { @@ -100,6 +142,106 @@ describe('Remote Config index', () => { expect(appsec.disable).to.not.have.been.called }) }) + + describe('API Security Request Sampling', () => { + describe('OneClick', () => { + let listener + + beforeEach(() => { + config = { + appsec: { + enabled: undefined, + apiSecurity: { + requestSampling: 0.1 + } + } + } + + remoteConfig.enable(config) + + listener = rc.setProductHandler.firstCall.args[1] + }) + + it('should update apiSecuritySampler config', () => { + listener('apply', { + api_security: { + request_sample_rate: 0.5 + } + }) + + expect(apiSecuritySampler.setRequestSampling).to.be.calledOnceWithExactly(0.5) + }) + + it('should update apiSecuritySampler config and disable it', () => { + listener('apply', { + api_security: { + request_sample_rate: 0 + } + }) + + expect(apiSecuritySampler.setRequestSampling).to.be.calledOnceWithExactly(0) + }) + + it('should not update apiSecuritySampler config with values greater than 1', () => { + listener('apply', { + api_security: { + request_sample_rate: 5 + } + }) + + expect(apiSecuritySampler.configure).to.not.be.called + }) + + it('should not update apiSecuritySampler config with values less than 0', () => { + listener('apply', { + api_security: { + request_sample_rate: -0.4 + } + }) + + expect(apiSecuritySampler.configure).to.not.be.called + }) + + it('should not update apiSecuritySampler config with incorrect values', () => { + listener('apply', { + api_security: { + request_sample_rate: 'not_a_number' + } + }) + + expect(apiSecuritySampler.configure).to.not.be.called + }) + }) + + describe('Enabled', () => { + let listener + + beforeEach(() => { + config = { + appsec: { + enabled: true, + apiSecurity: { + requestSampling: 0.1 + } + } + } + + remoteConfig.enable(config) + + listener = rc.setProductHandler.firstCall.args[1] + }) + + it('should update config apiSecurity.requestSampling property value', () => { + listener('apply', { + api_security: { + request_sample_rate: 0.5 + } + }) + + expect(apiSecuritySampler.setRequestSampling).to.be.calledOnceWithExactly(0.5) + }) + }) + }) }) describe('enableWafUpdate', () => { @@ -109,102 +251,178 @@ describe('Remote Config index', () => { remoteConfig.enableWafUpdate(config.appsec) expect(rc.updateCapabilities).to.not.have.been.called - expect(rc.on).to.not.have.been.called + expect(rc.setProductHandler).to.not.have.been.called }) it('should not enable when custom appsec rules are provided', () => { - config.appsec = { enabled: true, rules: {}, customRulesProvided: true } + config.appsec = { enabled: true, rules: {} } remoteConfig.enable(config) remoteConfig.enableWafUpdate(config.appsec) - expect(rc.updateCapabilities).to.not.have.been.called - expect(rc.on).to.not.have.been.called + expect(rc.updateCapabilities).to.not.have.been.calledWith('ASM_ACTIVATION') + expect(rc.setProductHandler).to.have.been.called }) it('should enable when using default rules', () => { - config.appsec = { enabled: true, rules: {}, customRulesProvided: false } + config.appsec = { enabled: true, rules: null, rasp: { enabled: true } } remoteConfig.enable(config) remoteConfig.enableWafUpdate(config.appsec) - expect(rc.updateCapabilities.callCount).to.be.equal(8) - expect(rc.updateCapabilities.getCall(0)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_IP_BLOCKING, true) - expect(rc.updateCapabilities.getCall(1)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_USER_BLOCKING, true) - expect(rc.updateCapabilities.getCall(2)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_DD_RULES, true) - expect(rc.updateCapabilities.getCall(3)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_EXCLUSIONS, true) - expect(rc.updateCapabilities.getCall(4)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_REQUEST_BLOCKING, true) - expect(rc.updateCapabilities.getCall(5)) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RESPONSE_BLOCKING, true) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_CUSTOM_RULES, true) - expect(rc.updateCapabilities.getCall(6)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_CUSTOM_BLOCKING_RESPONSE, true) - expect(rc.updateCapabilities.getCall(7)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_TRUSTED_IPS, true) - - expect(rc.on.callCount).to.be.equal(4) - expect(rc.on.getCall(0)).to.have.been.calledWith('ASM_DATA') - expect(rc.on.getCall(1)).to.have.been.calledWith('ASM_DD') - expect(rc.on.getCall(2)).to.have.been.calledWith('ASM') - expect(rc.on.getCall(3)).to.have.been.calledWithExactly(kPreUpdate, RuleManager.updateWafFromRC) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_ENDPOINT_FINGERPRINT, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_NETWORK_FINGERPRINT, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_HEADER_FINGERPRINT, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_SSRF, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_SQLI, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_LFI, true) + + expect(rc.setProductHandler).to.have.been.calledWith('ASM_DATA') + expect(rc.setProductHandler).to.have.been.calledWith('ASM_DD') + expect(rc.setProductHandler).to.have.been.calledWith('ASM') + expect(rc.on).to.have.been.calledWithExactly(kPreUpdate, RuleManager.updateWafFromRC) }) it('should activate if appsec is manually enabled', () => { - config.appsec = { enabled: true } + config.appsec = { enabled: true, rasp: { enabled: true } } remoteConfig.enable(config) remoteConfig.enableWafUpdate(config.appsec) - expect(rc.updateCapabilities.callCount).to.be.equal(8) - expect(rc.updateCapabilities.getCall(0)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_IP_BLOCKING, true) - expect(rc.updateCapabilities.getCall(1)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_USER_BLOCKING, true) - expect(rc.updateCapabilities.getCall(2)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_DD_RULES, true) - expect(rc.updateCapabilities.getCall(3)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_EXCLUSIONS, true) - expect(rc.updateCapabilities.getCall(4)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_REQUEST_BLOCKING, true) - expect(rc.updateCapabilities.getCall(5)) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RESPONSE_BLOCKING, true) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_CUSTOM_RULES, true) - expect(rc.updateCapabilities.getCall(6)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_CUSTOM_BLOCKING_RESPONSE, true) - expect(rc.updateCapabilities.getCall(7)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_TRUSTED_IPS, true) - - expect(rc.on.callCount).to.be.equal(4) - expect(rc.on.getCall(0)).to.have.been.calledWith('ASM_DATA') - expect(rc.on.getCall(1)).to.have.been.calledWith('ASM_DD') - expect(rc.on.getCall(2)).to.have.been.calledWith('ASM') - expect(rc.on.getCall(3)).to.have.been.calledWithExactly(kPreUpdate, RuleManager.updateWafFromRC) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_ENDPOINT_FINGERPRINT, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_NETWORK_FINGERPRINT, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_HEADER_FINGERPRINT, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_SSRF, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_SQLI, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_LFI, true) + + expect(rc.setProductHandler).to.have.been.calledWith('ASM_DATA') + expect(rc.setProductHandler).to.have.been.calledWith('ASM_DD') + expect(rc.setProductHandler).to.have.been.calledWith('ASM') + expect(rc.on).to.have.been.calledWithExactly(kPreUpdate, RuleManager.updateWafFromRC) }) it('should activate if appsec enabled is not defined', () => { - config.appsec = {} + config.appsec = { rasp: { enabled: true } } + remoteConfig.enable(config) + remoteConfig.enableWafUpdate(config.appsec) + + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_ACTIVATION, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_IP_BLOCKING, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_USER_BLOCKING, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_DD_RULES, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_EXCLUSIONS, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_REQUEST_BLOCKING, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RESPONSE_BLOCKING, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_CUSTOM_RULES, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_CUSTOM_BLOCKING_RESPONSE, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_TRUSTED_IPS, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_ENDPOINT_FINGERPRINT, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_NETWORK_FINGERPRINT, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_HEADER_FINGERPRINT, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_SSRF, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_SQLI, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_LFI, true) + }) + + it('should not activate rasp capabilities if rasp is disabled', () => { + config.appsec = { rasp: { enabled: false } } remoteConfig.enable(config) remoteConfig.enableWafUpdate(config.appsec) - expect(rc.updateCapabilities.callCount).to.be.equal(9) - expect(rc.updateCapabilities.getCall(0)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_ACTIVATION, true) - expect(rc.updateCapabilities.getCall(1)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_IP_BLOCKING, true) - expect(rc.updateCapabilities.getCall(2)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_USER_BLOCKING, true) - expect(rc.updateCapabilities.getCall(3)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_DD_RULES, true) - expect(rc.updateCapabilities.getCall(4)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_EXCLUSIONS, true) - expect(rc.updateCapabilities.getCall(5)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_REQUEST_BLOCKING, true) - expect(rc.updateCapabilities.getCall(6)) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RESPONSE_BLOCKING, true) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_CUSTOM_RULES, true) - expect(rc.updateCapabilities.getCall(7)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_CUSTOM_BLOCKING_RESPONSE, true) - expect(rc.updateCapabilities.getCall(8)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_TRUSTED_IPS, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_ENDPOINT_FINGERPRINT, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_NETWORK_FINGERPRINT, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_HEADER_FINGERPRINT, true) + expect(rc.updateCapabilities) + .to.not.have.been.calledWith(RemoteConfigCapabilities.ASM_RASP_SSRF) + expect(rc.updateCapabilities) + .to.not.have.been.calledWith(RemoteConfigCapabilities.ASM_RASP_SQLI) + expect(rc.updateCapabilities) + .to.not.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_LFI) }) }) @@ -214,29 +432,41 @@ describe('Remote Config index', () => { rc.updateCapabilities.resetHistory() remoteConfig.disableWafUpdate() - expect(rc.updateCapabilities.callCount).to.be.equal(8) - expect(rc.updateCapabilities.getCall(0)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_IP_BLOCKING, false) - expect(rc.updateCapabilities.getCall(1)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_USER_BLOCKING, false) - expect(rc.updateCapabilities.getCall(2)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_DD_RULES, false) - expect(rc.updateCapabilities.getCall(3)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_EXCLUSIONS, false) - expect(rc.updateCapabilities.getCall(4)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_REQUEST_BLOCKING, false) - expect(rc.updateCapabilities.getCall(5)) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RESPONSE_BLOCKING, false) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_CUSTOM_RULES, false) - expect(rc.updateCapabilities.getCall(6)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_CUSTOM_BLOCKING_RESPONSE, false) - expect(rc.updateCapabilities.getCall(7)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_TRUSTED_IPS, false) - - expect(rc.off.callCount).to.be.equal(4) - expect(rc.off.getCall(0)).to.have.been.calledWith('ASM_DATA') - expect(rc.off.getCall(1)).to.have.been.calledWith('ASM_DD') - expect(rc.off.getCall(2)).to.have.been.calledWith('ASM') - expect(rc.off.getCall(3)).to.have.been.calledWithExactly(kPreUpdate, RuleManager.updateWafFromRC) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_ENDPOINT_FINGERPRINT, false) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_NETWORK_FINGERPRINT, false) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_HEADER_FINGERPRINT, false) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_SSRF, false) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_SQLI, false) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_LFI, false) + + expect(rc.removeProductHandler).to.have.been.calledWith('ASM_DATA') + expect(rc.removeProductHandler).to.have.been.calledWith('ASM_DD') + expect(rc.removeProductHandler).to.have.been.calledWith('ASM') + expect(rc.off).to.have.been.calledWithExactly(kPreUpdate, RuleManager.updateWafFromRC) }) }) }) diff --git a/packages/dd-trace/test/appsec/remote_config/manager.spec.js b/packages/dd-trace/test/appsec/remote_config/manager.spec.js index 744eb7dfd12..f9aea97ce08 100644 --- a/packages/dd-trace/test/appsec/remote_config/manager.spec.js +++ b/packages/dd-trace/test/appsec/remote_config/manager.spec.js @@ -76,11 +76,7 @@ describe('RemoteConfigManager', () => { expect(rc.scheduler).to.equal(scheduler) - expect(rc.requestOptions).to.deep.equal({ - method: 'POST', - url: config.url, - path: '/v0.7/config' - }) + expect(rc.url).to.deep.equal(config.url) expect(rc.state).to.deep.equal({ client: { @@ -153,37 +149,46 @@ describe('RemoteConfigManager', () => { }) }) - describe('on/off', () => { + describe('setProductHandler/removeProductHandler', () => { it('should update the product list and autostart or autostop', () => { - expect(rc.on('ASM_FEATURES', noop)).to.equal(rc) + expect(rc.scheduler.start).to.not.have.been.called + + rc.setProductHandler('ASM_FEATURES', noop) expect(rc.state.client.products).to.deep.equal(['ASM_FEATURES']) - expect(rc.scheduler.start).to.have.been.calledOnce + expect(rc.scheduler.start).to.have.been.called - rc.on('ASM_DATA', noop) - rc.on('ASM_DD', noop) + rc.setProductHandler('ASM_DATA', noop) + rc.setProductHandler('ASM_DD', noop) expect(rc.state.client.products).to.deep.equal(['ASM_FEATURES', 'ASM_DATA', 'ASM_DD']) - expect(rc.scheduler.start).to.have.been.calledThrice - expect(rc.off('ASM_FEATURES', noop)).to.equal(rc) + rc.removeProductHandler('ASM_FEATURES') expect(rc.state.client.products).to.deep.equal(['ASM_DATA', 'ASM_DD']) - rc.off('ASM_DATA', noop) + rc.removeProductHandler('ASM_DATA') expect(rc.scheduler.stop).to.not.have.been.called - rc.off('ASM_DD', noop) + rc.removeProductHandler('ASM_DD') - expect(rc.scheduler.stop).to.have.been.calledOnce + expect(rc.scheduler.stop).to.have.been.called expect(rc.state.client.products).to.be.empty }) }) describe('poll', () => { + let expectedPayload + beforeEach(() => { sinon.stub(rc, 'parseConfig') + expectedPayload = { + url: rc.url, + method: 'POST', + path: '/v0.7/config', + headers: { 'Content-Type': 'application/json; charset=utf-8' } + } }) it('should request and do nothing when received status 404', (cb) => { @@ -192,7 +197,7 @@ describe('RemoteConfigManager', () => { const payload = JSON.stringify(rc.state) rc.poll(() => { - expect(request).to.have.been.calledOnceWith(payload, rc.requestOptions) + expect(request).to.have.been.calledOnceWith(payload, expectedPayload) expect(log.error).to.not.have.been.called expect(rc.parseConfig).to.not.have.been.called cb() @@ -206,7 +211,7 @@ describe('RemoteConfigManager', () => { const payload = JSON.stringify(rc.state) rc.poll(() => { - expect(request).to.have.been.calledOnceWith(payload, rc.requestOptions) + expect(request).to.have.been.calledOnceWith(payload, expectedPayload) expect(log.error).to.have.been.calledOnceWithExactly(err) expect(rc.parseConfig).to.not.have.been.called cb() @@ -219,7 +224,7 @@ describe('RemoteConfigManager', () => { const payload = JSON.stringify(rc.state) rc.poll(() => { - expect(request).to.have.been.calledOnceWith(payload, rc.requestOptions) + expect(request).to.have.been.calledOnceWith(payload, expectedPayload) expect(log.error).to.not.have.been.called expect(rc.parseConfig).to.have.been.calledOnceWithExactly({ a: 'b' }) cb() @@ -235,7 +240,7 @@ describe('RemoteConfigManager', () => { const payload = JSON.stringify(rc.state) rc.poll(() => { - expect(request).to.have.been.calledOnceWith(payload, rc.requestOptions) + expect(request).to.have.been.calledOnceWith(payload, expectedPayload) expect(rc.parseConfig).to.have.been.calledOnceWithExactly({ a: 'b' }) expect(log.error).to.have.been .calledOnceWithExactly('Could not parse remote config response: Error: Unable to parse config') @@ -246,7 +251,7 @@ describe('RemoteConfigManager', () => { rc.poll(() => { expect(request).to.have.been.calledTwice - expect(request.secondCall).to.have.been.calledWith(payload2, rc.requestOptions) + expect(request.secondCall).to.have.been.calledWith(payload2, expectedPayload) expect(rc.parseConfig).to.have.been.calledOnce expect(log.error).to.have.been.calledOnce expect(rc.state.client.state.has_error).to.be.false @@ -262,7 +267,7 @@ describe('RemoteConfigManager', () => { const payload = JSON.stringify(rc.state) rc.poll(() => { - expect(request).to.have.been.calledOnceWith(payload, rc.requestOptions) + expect(request).to.have.been.calledOnceWith(payload, expectedPayload) expect(log.error).to.not.have.been.called expect(rc.parseConfig).to.not.have.been.called cb() @@ -279,7 +284,7 @@ describe('RemoteConfigManager', () => { expect(JSON.parse(payload).client.client_tracer.extra_services).to.deep.equal(extraServices) rc.poll(() => { - expect(request).to.have.been.calledOnceWith(payload, rc.requestOptions) + expect(request).to.have.been.calledOnceWith(payload, expectedPayload) cb() }) }) @@ -548,63 +553,101 @@ describe('RemoteConfigManager', () => { }) describe('dispatch', () => { - beforeEach(() => { - sinon.stub(rc, 'emit') - }) - - it('should call emit for each config, catch errors, and update the state', () => { - rc.emit.onFirstCall().returns(true) - rc.emit.onSecondCall().throws(new Error('Unable to apply config')) - rc.emit.onThirdCall().returns(true) - - const list = [ - { - id: 'asm_features', - path: 'datadog/42/ASM_FEATURES/confId/config', - product: 'ASM_FEATURES', - apply_state: UNACKNOWLEDGED, - apply_error: '', - file: { asm: { enabled: true } } - }, - { - id: 'asm_data', - path: 'datadog/42/ASM_DATA/confId/config', - product: 'ASM_DATA', - apply_state: UNACKNOWLEDGED, - apply_error: '', - file: { data: [1, 2, 3] } - }, - { - id: 'asm_dd', - path: 'datadog/42/ASM_DD/confId/config', - product: 'ASM_DD', + it('should call registered handler for each config, catch errors, and update the state', (done) => { + const syncGoodNonAckHandler = sinon.spy() + const syncBadNonAckHandler = sinon.spy(() => { throw new Error('sync fn') }) + const asyncGoodHandler = sinon.spy(async () => {}) + const asyncBadHandler = sinon.spy(async () => { throw new Error('async fn') }) + const syncGoodAckHandler = sinon.spy((action, conf, id, ack) => { ack() }) + const syncBadAckHandler = sinon.spy((action, conf, id, ack) => { ack(new Error('sync ack fn')) }) + const asyncGoodAckHandler = sinon.spy((action, conf, id, ack) => { setImmediate(ack) }) + const asyncBadAckHandler = sinon.spy((action, conf, id, ack) => { + setImmediate(ack.bind(null, new Error('async ack fn'))) + }) + const unackHandler = sinon.spy((action, conf, id, ack) => {}) + + rc.setProductHandler('PRODUCT_0', syncGoodNonAckHandler) + rc.setProductHandler('PRODUCT_1', syncBadNonAckHandler) + rc.setProductHandler('PRODUCT_2', asyncGoodHandler) + rc.setProductHandler('PRODUCT_3', asyncBadHandler) + rc.setProductHandler('PRODUCT_4', syncGoodAckHandler) + rc.setProductHandler('PRODUCT_5', syncBadAckHandler) + rc.setProductHandler('PRODUCT_6', asyncGoodAckHandler) + rc.setProductHandler('PRODUCT_7', asyncBadAckHandler) + rc.setProductHandler('PRODUCT_8', unackHandler) + + const list = [] + for (let i = 0; i < 9; i++) { + list[i] = { + id: `id_${i}`, + path: `datadog/42/PRODUCT_${i}/confId/config`, + product: `PRODUCT_${i}`, apply_state: UNACKNOWLEDGED, apply_error: '', - file: { rules: [4, 5, 6] } + file: { index: i } } - ] + } rc.dispatch(list, 'apply') - expect(rc.emit).to.have.been.calledThrice - expect(rc.emit.firstCall).to.have.been - .calledWithExactly('ASM_FEATURES', 'apply', { asm: { enabled: true } }, 'asm_features') - expect(rc.emit.secondCall).to.have.been.calledWithExactly('ASM_DATA', 'apply', { data: [1, 2, 3] }, 'asm_data') - expect(rc.emit.thirdCall).to.have.been.calledWithExactly('ASM_DD', 'apply', { rules: [4, 5, 6] }, 'asm_dd') + expect(syncGoodNonAckHandler).to.have.been.calledOnceWithExactly('apply', list[0].file, list[0].id) + expect(syncBadNonAckHandler).to.have.been.calledOnceWithExactly('apply', list[1].file, list[1].id) + expect(asyncGoodHandler).to.have.been.calledOnceWithExactly('apply', list[2].file, list[2].id) + expect(asyncBadHandler).to.have.been.calledOnceWithExactly('apply', list[3].file, list[3].id) + assertAsyncHandlerCallArguments(syncGoodAckHandler, 'apply', list[4].file, list[4].id) + assertAsyncHandlerCallArguments(syncBadAckHandler, 'apply', list[5].file, list[5].id) + assertAsyncHandlerCallArguments(asyncGoodAckHandler, 'apply', list[6].file, list[6].id) + assertAsyncHandlerCallArguments(asyncBadAckHandler, 'apply', list[7].file, list[7].id) + assertAsyncHandlerCallArguments(unackHandler, 'apply', list[8].file, list[8].id) expect(list[0].apply_state).to.equal(ACKNOWLEDGED) expect(list[0].apply_error).to.equal('') expect(list[1].apply_state).to.equal(ERROR) - expect(list[1].apply_error).to.equal('Error: Unable to apply config') - expect(list[2].apply_state).to.equal(ACKNOWLEDGED) + expect(list[1].apply_error).to.equal('Error: sync fn') + expect(list[2].apply_state).to.equal(UNACKNOWLEDGED) expect(list[2].apply_error).to.equal('') + expect(list[3].apply_state).to.equal(UNACKNOWLEDGED) + expect(list[3].apply_error).to.equal('') + expect(list[4].apply_state).to.equal(ACKNOWLEDGED) + expect(list[4].apply_error).to.equal('') + expect(list[5].apply_state).to.equal(ERROR) + expect(list[5].apply_error).to.equal('Error: sync ack fn') + expect(list[6].apply_state).to.equal(UNACKNOWLEDGED) + expect(list[6].apply_error).to.equal('') + expect(list[7].apply_state).to.equal(UNACKNOWLEDGED) + expect(list[7].apply_error).to.equal('') + expect(list[8].apply_state).to.equal(UNACKNOWLEDGED) + expect(list[8].apply_error).to.equal('') + + for (let i = 0; i < list.length; i++) { + expect(rc.appliedConfigs.get(`datadog/42/PRODUCT_${i}/confId/config`)).to.equal(list[i]) + } + + setImmediate(() => { + expect(list[2].apply_state).to.equal(ACKNOWLEDGED) + expect(list[2].apply_error).to.equal('') + expect(list[3].apply_state).to.equal(ERROR) + expect(list[3].apply_error).to.equal('Error: async fn') + expect(list[6].apply_state).to.equal(ACKNOWLEDGED) + expect(list[6].apply_error).to.equal('') + expect(list[7].apply_state).to.equal(ERROR) + expect(list[7].apply_error).to.equal('Error: async ack fn') + expect(list[8].apply_state).to.equal(UNACKNOWLEDGED) + expect(list[8].apply_error).to.equal('') + done() + }) - expect(rc.appliedConfigs.get('datadog/42/ASM_FEATURES/confId/config')).to.equal(list[0]) - expect(rc.appliedConfigs.get('datadog/42/ASM_DATA/confId/config')).to.equal(list[1]) - expect(rc.appliedConfigs.get('datadog/42/ASM_DD/confId/config')).to.equal(list[2]) + function assertAsyncHandlerCallArguments (handler, ...expectedArgs) { + expect(handler).to.have.been.calledOnceWith(...expectedArgs) + expect(handler.args[0].length).to.equal(expectedArgs.length + 1) + expect(handler.args[0][handler.args[0].length - 1]).to.be.a('function') + } }) it('should delete config from state when action is unapply', () => { + const handler = sinon.spy() + rc.setProductHandler('ASM_FEATURES', handler) + rc.appliedConfigs.set('datadog/42/ASM_FEATURES/confId/config', { id: 'asm_data', path: 'datadog/42/ASM_FEATURES/confId/config', @@ -616,8 +659,7 @@ describe('RemoteConfigManager', () => { rc.dispatch([rc.appliedConfigs.get('datadog/42/ASM_FEATURES/confId/config')], 'unapply') - expect(rc.emit).to.have.been - .calledOnceWithExactly('ASM_FEATURES', 'unapply', { asm: { enabled: true } }, 'asm_data') + expect(handler).to.have.been.calledOnceWithExactly('unapply', { asm: { enabled: true } }, 'asm_data') expect(rc.appliedConfigs).to.be.empty }) }) diff --git a/packages/dd-trace/test/appsec/reporter.spec.js b/packages/dd-trace/test/appsec/reporter.spec.js index 36f1e1b5276..0860b2c75ac 100644 --- a/packages/dd-trace/test/appsec/reporter.spec.js +++ b/packages/dd-trace/test/appsec/reporter.spec.js @@ -9,6 +9,7 @@ describe('reporter', () => { let span let web let telemetry + let sample beforeEach(() => { span = { @@ -26,13 +27,20 @@ describe('reporter', () => { telemetry = { incrementWafInitMetric: sinon.stub(), updateWafRequestsMetricTags: sinon.stub(), + updateRaspRequestsMetricTags: sinon.stub(), incrementWafUpdatesMetric: sinon.stub(), - incrementWafRequestsMetric: sinon.stub() + incrementWafRequestsMetric: sinon.stub(), + getRequestMetrics: sinon.stub() } + sample = sinon.stub() + Reporter = proxyquire('../../src/appsec/reporter', { '../plugins/util/web': web, - './telemetry': telemetry + './telemetry': telemetry, + './standalone': { + sample + } }) }) @@ -141,17 +149,21 @@ describe('reporter', () => { }) it('should set duration metrics if set', () => { - Reporter.reportMetrics({ duration: 1337 }) + const metrics = { duration: 1337 } + Reporter.reportMetrics(metrics) expect(web.root).to.have.been.calledOnceWithExactly(req) - expect(span.setTag).to.have.been.calledOnceWithExactly('_dd.appsec.waf.duration', 1337) + expect(telemetry.updateWafRequestsMetricTags).to.have.been.calledOnceWithExactly(metrics, req) + expect(telemetry.updateRaspRequestsMetricTags).to.not.have.been.called }) it('should set ext duration metrics if set', () => { - Reporter.reportMetrics({ durationExt: 42 }) + const metrics = { durationExt: 42 } + Reporter.reportMetrics(metrics) expect(web.root).to.have.been.calledOnceWithExactly(req) - expect(span.setTag).to.have.been.calledOnceWithExactly('_dd.appsec.waf.duration_ext', 42) + expect(telemetry.updateWafRequestsMetricTags).to.have.been.calledOnceWithExactly(metrics, req) + expect(telemetry.updateRaspRequestsMetricTags).to.not.have.been.called }) it('should set rulesVersion if set', () => { @@ -159,6 +171,7 @@ describe('reporter', () => { expect(web.root).to.have.been.calledOnceWithExactly(req) expect(span.setTag).to.have.been.calledOnceWithExactly('_dd.appsec.event_rules.version', '1.2.3') + expect(telemetry.updateRaspRequestsMetricTags).to.not.have.been.called }) it('should call updateWafRequestsMetricTags', () => { @@ -168,6 +181,17 @@ describe('reporter', () => { Reporter.reportMetrics(metrics) expect(telemetry.updateWafRequestsMetricTags).to.have.been.calledOnceWithExactly(metrics, store.req) + expect(telemetry.updateRaspRequestsMetricTags).to.not.have.been.called + }) + + it('should call updateRaspRequestsMetricTags when ruleType if provided', () => { + const metrics = { rulesVersion: '1.2.3' } + const store = storage.getStore() + + Reporter.reportMetrics(metrics, 'rule_type') + + expect(telemetry.updateRaspRequestsMetricTags).to.have.been.calledOnceWithExactly(metrics, store.req, 'rule_type') + expect(telemetry.updateWafRequestsMetricTags).to.not.have.been.called }) }) @@ -201,9 +225,6 @@ describe('reporter', () => { 'manual.keep': 'true', '_dd.origin': 'appsec', '_dd.appsec.json': '{"triggers":[{"rule":{},"rule_matches":[{}]}]}', - 'http.request.headers.host': 'localhost', - 'http.request.headers.user-agent': 'arachni', - 'http.useragent': 'arachni', 'network.client.ip': '8.8.8.8' }) }) @@ -232,7 +253,7 @@ describe('reporter', () => { expect(Reporter.reportAttack('', params)).to.not.be.false expect(addTags.getCall(5).firstArg).to.have.property('manual.keep').that.equals('true') done() - }, 1e3) + }, 1020) }) it('should not overwrite origin tag', () => { @@ -243,12 +264,9 @@ describe('reporter', () => { expect(web.root).to.have.been.calledOnceWith(req) expect(span.addTags).to.have.been.calledOnceWithExactly({ - 'http.request.headers.host': 'localhost', - 'http.request.headers.user-agent': 'arachni', 'appsec.event': 'true', 'manual.keep': 'true', '_dd.appsec.json': '{"triggers":[]}', - 'http.useragent': 'arachni', 'network.client.ip': '8.8.8.8' }) }) @@ -261,16 +279,31 @@ describe('reporter', () => { expect(web.root).to.have.been.calledOnceWith(req) expect(span.addTags).to.have.been.calledOnceWithExactly({ - 'http.request.headers.host': 'localhost', - 'http.request.headers.user-agent': 'arachni', 'appsec.event': 'true', 'manual.keep': 'true', '_dd.origin': 'appsec', '_dd.appsec.json': '{"triggers":[{"rule":{},"rule_matches":[{}]},{"rule":{}},{"rule":{},"rule_matches":[{}]}]}', - 'http.useragent': 'arachni', 'network.client.ip': '8.8.8.8' }) }) + + it('should call standalone sample', () => { + span.context()._tags = { '_dd.appsec.json': '{"triggers":[{"rule":{},"rule_matches":[{}]}]}' } + + const result = Reporter.reportAttack('[{"rule":{}},{"rule":{},"rule_matches":[{}]}]') + expect(result).to.not.be.false + expect(web.root).to.have.been.calledOnceWith(req) + + expect(span.addTags).to.have.been.calledOnceWithExactly({ + 'appsec.event': 'true', + 'manual.keep': 'true', + '_dd.origin': 'appsec', + '_dd.appsec.json': '{"triggers":[{"rule":{},"rule_matches":[{}]},{"rule":{}},{"rule":{},"rule_matches":[{}]}]}', + 'network.client.ip': '8.8.8.8' + }) + + expect(sample).to.have.been.calledOnceWithExactly(span) + }) }) describe('reportWafUpdate', () => { @@ -281,20 +314,24 @@ describe('reporter', () => { }) }) - describe('reportSchemas', () => { + describe('reportDerivatives', () => { it('should not call addTags if parameter is undefined', () => { - Reporter.reportSchemas(undefined) + Reporter.reportDerivatives(undefined) expect(span.addTags).not.to.be.called }) it('should call addTags with an empty array', () => { - Reporter.reportSchemas([]) + Reporter.reportDerivatives([]) expect(span.addTags).to.be.calledOnceWithExactly({}) }) - it('should call addTags with matched tags', () => { - const schemaValue = [{ 'key': [8] }] + it('should call addTags', () => { + const schemaValue = [{ key: [8] }] const derivatives = { + '_dd.appsec.fp.http.endpoint': 'endpoint_fingerprint', + '_dd.appsec.fp.http.header': 'header_fingerprint', + '_dd.appsec.fp.http.network': 'network_fingerprint', + '_dd.appsec.fp.session': 'session_fingerprint', '_dd.appsec.s.req.headers': schemaValue, '_dd.appsec.s.req.query': schemaValue, '_dd.appsec.s.req.params': schemaValue, @@ -303,15 +340,20 @@ describe('reporter', () => { 'custom.processor.output': schemaValue } - Reporter.reportSchemas(derivatives) + Reporter.reportDerivatives(derivatives) const schemaEncoded = zlib.gzipSync(JSON.stringify(schemaValue)).toString('base64') expect(span.addTags).to.be.calledOnceWithExactly({ + '_dd.appsec.fp.http.endpoint': 'endpoint_fingerprint', + '_dd.appsec.fp.http.header': 'header_fingerprint', + '_dd.appsec.fp.http.network': 'network_fingerprint', + '_dd.appsec.fp.session': 'session_fingerprint', '_dd.appsec.s.req.headers': schemaEncoded, '_dd.appsec.s.req.query': schemaEncoded, '_dd.appsec.s.req.params': schemaEncoded, '_dd.appsec.s.req.cookies': schemaEncoded, - '_dd.appsec.s.req.body': schemaEncoded + '_dd.appsec.s.req.body': schemaEncoded, + 'custom.processor.output': schemaEncoded }) }) }) @@ -319,6 +361,33 @@ describe('reporter', () => { describe('finishRequest', () => { let wafContext + const requestHeadersToTrackOnEvent = [ + 'x-forwarded-for', + 'x-real-ip', + 'true-client-ip', + 'x-client-ip', + 'x-forwarded', + 'forwarded-for', + 'x-cluster-client-ip', + 'fastly-client-ip', + 'cf-connecting-ip', + 'cf-connecting-ipv6', + 'forwarded', + 'via', + 'content-length', + 'content-encoding', + 'content-language', + 'host', + 'accept-encoding', + 'accept-language' + ] + const requestHeadersAndValuesToTrackOnEvent = {} + const expectedRequestTagsToTrackOnEvent = {} + requestHeadersToTrackOnEvent.forEach((header, index) => { + requestHeadersAndValuesToTrackOnEvent[header] = `val-${index}` + expectedRequestTagsToTrackOnEvent[`http.request.headers.${header}`] = `val-${index}` + }) + beforeEach(() => { wafContext = { dispose: sinon.stub() @@ -348,22 +417,52 @@ describe('reporter', () => { Reporter.finishRequest(req, wafContext, {}) expect(web.root).to.have.been.calledOnceWithExactly(req) - expect(span.addTags).to.have.been.calledOnceWithExactly({ a: 1, b: 2 }) + expect(span.addTags).to.have.been.calledWithExactly({ a: 1, b: 2 }) expect(Reporter.metricsQueue).to.be.empty }) - it('should not add http response data when no attack was previously found', () => { - const req = {} + it('should only add mandatory headers when no attack or event was previously found', () => { + const req = { + headers: { + 'not-included': 'hello', + 'x-amzn-trace-id': 'a', + 'cloudfront-viewer-ja3-fingerprint': 'b', + 'cf-ray': 'c', + 'x-cloud-trace-context': 'd', + 'x-appgw-trace-id': 'e', + 'x-sigsci-requestid': 'f', + 'x-sigsci-tags': 'g', + 'akamai-user-risk': 'h', + 'content-type': 'i', + accept: 'j', + 'user-agent': 'k' + } + } Reporter.finishRequest(req) expect(web.root).to.have.been.calledOnceWith(req) - expect(span.addTags).to.not.have.been.called + expect(span.addTags).to.have.been.calledOnceWithExactly({ + 'http.request.headers.x-amzn-trace-id': 'a', + 'http.request.headers.cloudfront-viewer-ja3-fingerprint': 'b', + 'http.request.headers.cf-ray': 'c', + 'http.request.headers.x-cloud-trace-context': 'd', + 'http.request.headers.x-appgw-trace-id': 'e', + 'http.request.headers.x-sigsci-requestid': 'f', + 'http.request.headers.x-sigsci-tags': 'g', + 'http.request.headers.akamai-user-risk': 'h', + 'http.request.headers.content-type': 'i', + 'http.request.headers.accept': 'j', + 'http.request.headers.user-agent': 'k' + }) }) it('should add http response data inside request span', () => { const req = { route: { path: '/path/:param' + }, + headers: { + 'x-cloud-trace-context': 'd' } } @@ -381,7 +480,11 @@ describe('reporter', () => { Reporter.finishRequest(req, res) expect(web.root).to.have.been.calledOnceWith(req) - expect(span.addTags).to.have.been.calledOnceWithExactly({ + expect(span.addTags).to.have.been.calledTwice + expect(span.addTags.firstCall).to.have.been.calledWithExactly({ + 'http.request.headers.x-cloud-trace-context': 'd' + }) + expect(span.addTags.secondCall).to.have.been.calledWithExactly({ 'http.response.headers.content-type': 'application/json', 'http.response.headers.content-length': '42', 'http.endpoint': '/path/:param' @@ -404,12 +507,110 @@ describe('reporter', () => { Reporter.finishRequest(req, res) expect(web.root).to.have.been.calledOnceWith(req) - expect(span.addTags).to.have.been.calledOnceWithExactly({ + expect(span.addTags).to.have.been.calledWithExactly({ 'http.response.headers.content-type': 'application/json', 'http.response.headers.content-length': '42' }) }) + it('should add http request data inside request span when appsec.event is true', () => { + const req = { + headers: { + 'user-agent': 'arachni', + ...requestHeadersAndValuesToTrackOnEvent + } + } + const res = { + getHeaders: () => { + return {} + } + } + span.context()._tags['appsec.event'] = 'true' + + Reporter.finishRequest(req, res) + + expect(span.addTags).to.have.been.calledWithExactly({ + 'http.request.headers.user-agent': 'arachni' + }) + + expect(span.addTags).to.have.been.calledWithExactly(expectedRequestTagsToTrackOnEvent) + }) + + it('should add http request data inside request span when user login success is tracked', () => { + const req = { + headers: { + 'user-agent': 'arachni', + ...requestHeadersAndValuesToTrackOnEvent + } + } + const res = { + getHeaders: () => { + return {} + } + } + + span.context() + ._tags['appsec.events.users.login.success.track'] = 'true' + + Reporter.finishRequest(req, res) + + expect(span.addTags).to.have.been.calledWithExactly({ + 'http.request.headers.user-agent': 'arachni' + }) + + expect(span.addTags).to.have.been.calledWithExactly(expectedRequestTagsToTrackOnEvent) + }) + + it('should add http request data inside request span when user login failure is tracked', () => { + const req = { + headers: { + 'user-agent': 'arachni', + ...requestHeadersAndValuesToTrackOnEvent + } + } + const res = { + getHeaders: () => { + return {} + } + } + + span.context() + ._tags['appsec.events.users.login.failure.track'] = 'true' + + Reporter.finishRequest(req, res) + + expect(span.addTags).to.have.been.calledWithExactly({ + 'http.request.headers.user-agent': 'arachni' + }) + + expect(span.addTags).to.have.been.calledWithExactly(expectedRequestTagsToTrackOnEvent) + }) + + it('should add http request data inside request span when user custom event is tracked', () => { + const req = { + headers: { + 'user-agent': 'arachni', + ...requestHeadersAndValuesToTrackOnEvent + } + } + const res = { + getHeaders: () => { + return {} + } + } + + span.context() + ._tags['appsec.events.custon.event.track'] = 'true' + + Reporter.finishRequest(req, res) + + expect(span.addTags).to.have.been.calledWithExactly({ + 'http.request.headers.user-agent': 'arachni' + }) + + expect(span.addTags).to.have.been.calledWithExactly(expectedRequestTagsToTrackOnEvent) + }) + it('should call incrementWafRequestsMetric', () => { const req = {} const res = {} @@ -417,5 +618,29 @@ describe('reporter', () => { expect(telemetry.incrementWafRequestsMetric).to.be.calledOnceWithExactly(req) }) + + it('should set waf.duration tags if there are metrics stored', () => { + telemetry.getRequestMetrics.returns({ duration: 1337, durationExt: 42 }) + + Reporter.finishRequest({}, {}) + + expect(span.setTag).to.have.been.calledWithExactly('_dd.appsec.waf.duration', 1337) + expect(span.setTag).to.have.been.calledWithExactly('_dd.appsec.waf.duration_ext', 42) + expect(span.setTag).to.not.have.been.calledWith('_dd.appsec.rasp.duration') + expect(span.setTag).to.not.have.been.calledWith('_dd.appsec.rasp.duration_ext') + expect(span.setTag).to.not.have.been.calledWith('_dd.appsec.rasp.rule.eval') + }) + + it('should set rasp.duration tags if there are metrics stored', () => { + telemetry.getRequestMetrics.returns({ raspDuration: 123, raspDurationExt: 321, raspEvalCount: 3 }) + + Reporter.finishRequest({}, {}) + + expect(span.setTag).to.not.have.been.calledWith('_dd.appsec.waf.duration') + expect(span.setTag).to.not.have.been.calledWith('_dd.appsec.waf.duration_ext') + expect(span.setTag).to.have.been.calledWithExactly('_dd.appsec.rasp.duration', 123) + expect(span.setTag).to.have.been.calledWithExactly('_dd.appsec.rasp.duration_ext', 321) + expect(span.setTag).to.have.been.calledWithExactly('_dd.appsec.rasp.rule.eval', 3) + }) }) }) diff --git a/packages/dd-trace/test/appsec/response_blocking.spec.js b/packages/dd-trace/test/appsec/response_blocking.spec.js new file mode 100644 index 00000000000..03541858955 --- /dev/null +++ b/packages/dd-trace/test/appsec/response_blocking.spec.js @@ -0,0 +1,275 @@ +'use strict' + +const { assert } = require('chai') +const agent = require('../plugins/agent') +const Axios = require('axios') +const appsec = require('../../src/appsec') +const Config = require('../../src/config') +const path = require('path') +const WafContext = require('../../src/appsec/waf/waf_context_wrapper') +const blockingResponse = JSON.parse(require('../../src/appsec/blocked_templates').json) +const fs = require('fs') + +describe('HTTP Response Blocking', () => { + let server + let responseHandler + let axios + + before(async () => { + await agent.load('http') + + const http = require('http') + + server = new http.Server((req, res) => { + // little polyfill, older versions of node don't have setHeaders() + if (typeof res.setHeaders !== 'function') { + res.setHeaders = headers => headers.forEach((v, k) => res.setHeader(k, v)) + } + + if (responseHandler) { + responseHandler(req, res) + } else { + res.writeHead(200) + res.end('OK') + } + }) + + await new Promise((resolve, reject) => { + server.listen(0, 'localhost') + .once('listening', (...args) => { + const port = server.address().port + + axios = Axios.create(({ + baseURL: `http://localhost:${port}`, + validateStatus: null + })) + + resolve(...args) + }) + .once('error', reject) + }) + + appsec.enable(new Config({ + appsec: { + enabled: true, + rules: path.join(__dirname, 'response_blocking_rules.json'), + rasp: { + enabled: false // disable rasp to not trigger waf.run executions due to lfi + } + } + })) + }) + + beforeEach(() => { + sinon.spy(WafContext.prototype, 'run') + }) + + afterEach(() => { + sinon.restore() + responseHandler = null + }) + + after(() => { + appsec.disable() + server?.close() + return agent.close({ ritmReset: false }) + }) + + it('should block with implicit statusCode + setHeader() + end()', async () => { + responseHandler = (req, res) => { + res.statusCode = 404 + res.setHeader('k', '404') + res.end('end') + } + + const res = await axios.get('/') + + assertBlocked(res) + }) + + it('should block with setHeader() + setHeaders() + writeHead() headers', async () => { + responseHandler = (req, res) => { + res.setHeaders(new Map(Object.entries({ a: 'bad1', b: 'good' }))) + res.setHeader('c', 'bad2') + res.writeHead(200, { d: 'bad3' }) + res.end('end') + } + + const res = await axios.get('/') + + assertBlocked(res) + }) + + it('should block with setHeader() + array writeHead() ', async () => { + responseHandler = (req, res) => { + res.setHeader('a', 'bad1') + res.writeHead(200, 'OK', ['b', 'bad2', 'c', 'bad3']) + res.end('end') + } + + const res = await axios.get('/') + + assertBlocked(res) + }) + + it('should not block with array writeHead() when attack is in the header name and not in header value', async () => { + responseHandler = (req, res) => { + res.writeHead(200, 'OK', ['a', 'bad1', 'b', 'bad2', 'bad3', 'c']) + res.end('end') + } + + const res = await axios.get('/') + + assert.equal(res.status, 200) + assert.hasAllKeys(cloneHeaders(res.headers), [ + 'a', + 'b', + 'bad3', + 'date', + 'connection', + 'transfer-encoding' + ]) + assert.deepEqual(res.data, 'end') + }) + + it('should block with implicit statusCode + setHeader() + flushHeaders()', async () => { + responseHandler = (req, res) => { + res.statusCode = 404 + res.setHeader('k', '404') + res.flushHeaders() + res.end('end') + } + + const res = await axios.get('/') + + assertBlocked(res) + }) + + it('should block with implicit statusCode + setHeader() + write()', async () => { + responseHandler = (req, res) => { + res.statusCode = 404 + res.setHeader('k', '404') + res.write('write') + res.end('end') + } + + const res = await axios.get('/') + + assertBlocked(res) + }) + + it('should block with implicit statusCode + setHeader() + stream pipe', async () => { + responseHandler = (req, res) => { + res.statusCode = 404 + res.setHeader('k', '404') + streamFile(res) + } + + const res = await axios.get('/') + + assertBlocked(res) + }) + + it('should block with writeHead() + write()', async () => { + responseHandler = (req, res) => { + res.writeHead(404, { k: '404' }) + res.write('write') + res.end('end') + } + + const res = await axios.get('/') + + assertBlocked(res) + }) + + it('should block with every methods combined', async () => { + responseHandler = (req, res) => { + res.setHeaders(new Map(Object.entries({ a: 'bad1', b: 'good' }))) + res.setHeader('c', 'bad2') + res.setHeader('d', 'good') + res.writeHead(200, 'OK', { d: 'good', e: 'bad3' }) + res.flushHeaders() + res.write('write') + res.addTrailers({ k: 'v' }) + streamFile(res) + } + + const res = await axios.get('/') + + assertBlocked(res) + }) + + it('should not block with every methods combined but no attack', async () => { + responseHandler = (req, res) => { + res.setHeaders(new Map(Object.entries({ a: 'good', b: 'good' }))) + res.setHeader('c', 'good') + res.setHeader('d', 'good') + res.writeHead(201, 'OK', { d: 'good', e: 'good' }) + res.flushHeaders() + res.write('write') + res.addTrailers({ k: 'v' }) + streamFile(res) + } + + const res = await axios.get('/') + + assert.equal(res.status, 201) + assert.hasAllKeys(cloneHeaders(res.headers), [ + 'a', + 'b', + 'c', + 'd', + 'e', + 'date', + 'connection', + 'transfer-encoding' + ]) + assert.deepEqual(res.data, 'writefileend') + }) + + it('should ignore subsequent response writes after blocking', async () => { + responseHandler = (req, res) => { + res.statusCode = 404 + res.setHeader('k', '404') + res.flushHeaders() + res.writeHead(200, { k: '200' }) + res.write('write1') + setTimeout(() => { + res.write('write2') + res.end('end') + }, 1000) + } + + const res = await axios.get('/') + + assertBlocked(res) + }) +}) + +function cloneHeaders (headers) { + // clone the headers accessor to a flat object + // and delete the keep-alive header as it's not always present + headers = Object.fromEntries(Object.entries(headers)) + delete headers['keep-alive'] + + return headers +} + +function assertBlocked (res) { + assert.equal(res.status, 403) + assert.hasAllKeys(cloneHeaders(res.headers), [ + 'content-type', + 'content-length', + 'date', + 'connection' + ]) + assert.deepEqual(res.data, blockingResponse) + + sinon.assert.callCount(WafContext.prototype.run, 2) +} + +function streamFile (res) { + const stream = fs.createReadStream(path.join(__dirname, 'streamtest.txt'), { encoding: 'utf8' }) + stream.pipe(res, { end: false }) + stream.on('end', () => res.end('end')) +} diff --git a/packages/dd-trace/test/appsec/response_blocking_rules.json b/packages/dd-trace/test/appsec/response_blocking_rules.json new file mode 100644 index 00000000000..bd7d1279892 --- /dev/null +++ b/packages/dd-trace/test/appsec/response_blocking_rules.json @@ -0,0 +1,110 @@ +{ + "version": "2.2", + "metadata": { + "rules_version": "1.5.0" + }, + "rules": [ + { + "id": "test-rule-id-1", + "name": "test-rule-name-1", + "tags": { + "type": "security_scanner1", + "category": "attack_attempt1" + }, + "conditions": [ + { + "operator": "match_regex", + "parameters": { + "inputs": [ + { + "address": "server.response.status" + } + ], + "regex": "^404$", + "options": { + "case_sensitive": true + } + } + }, + { + "operator": "match_regex", + "parameters": { + "inputs": [ + { + "address": "server.response.headers.no_cookies" + } + ], + "regex": "^404$", + "options": { + "case_sensitive": false + } + } + } + ], + "transformers": [ + "lowercase" + ], + "on_match": [ + "block" + ] + }, + { + "id": "test-rule-id-2", + "name": "test-rule-name-2", + "tags": { + "type": "security_scanner2", + "category": "attack_attempt2" + }, + "conditions": [ + { + "operator": "match_regex", + "parameters": { + "inputs": [ + { + "address": "server.response.headers.no_cookies" + } + ], + "regex": "^bad1$", + "options": { + "case_sensitive": false + } + } + }, + { + "operator": "match_regex", + "parameters": { + "inputs": [ + { + "address": "server.response.headers.no_cookies" + } + ], + "regex": "^bad2$", + "options": { + "case_sensitive": false + } + } + }, + { + "operator": "match_regex", + "parameters": { + "inputs": [ + { + "address": "server.response.headers.no_cookies" + } + ], + "regex": "^bad3$", + "options": { + "case_sensitive": false + } + } + } + ], + "transformers": [ + "lowercase" + ], + "on_match": [ + "block" + ] + } + ] +} diff --git a/packages/dd-trace/test/appsec/rule_manager.spec.js b/packages/dd-trace/test/appsec/rule_manager.spec.js index b2162b42a82..3b0265870eb 100644 --- a/packages/dd-trace/test/appsec/rule_manager.spec.js +++ b/packages/dd-trace/test/appsec/rule_manager.spec.js @@ -1,7 +1,7 @@ 'use strict' -const fs = require('fs') const path = require('path') +const fs = require('fs') const { loadRules, clearAllRules, updateWafFromRC } = require('../../src/appsec/rule_manager') const Config = require('../../src/config') const { ACKNOWLEDGED } = require('../../src/appsec/remote_config/apply_states') @@ -17,11 +17,11 @@ describe('AppSec Rule Manager', () => { clearAllRules() config = new Config() - sinon.stub(waf, 'init').callThrough() - sinon.stub(waf, 'destroy').callThrough() - sinon.stub(waf, 'update').callThrough() + sinon.stub(waf, 'init') + sinon.stub(waf, 'destroy') + sinon.stub(waf, 'update') - sinon.stub(blocking, 'updateBlockingConfiguration').callThrough() + sinon.stub(blocking, 'setDefaultBlockingActionParameters') }) afterEach(() => { @@ -34,7 +34,15 @@ describe('AppSec Rule Manager', () => { loadRules(config.appsec) expect(waf.init).to.have.been.calledOnceWithExactly(rules, config.appsec) - expect(blocking.updateBlockingConfiguration).not.to.have.been.called + }) + + it('should throw if null/undefined are passed', () => { + // TODO: fix the exception thrown in the waf or catch it in rule_manager? + config.appsec.rules = './not/existing/file.json' + expect(() => { loadRules(config.appsec) }).to.throw() + + config.appsec.rules = './bad-formatted-rules.json' + expect(() => { loadRules(config.appsec) }).to.throw() }) it('should call updateBlockingConfiguration with proper params', () => { @@ -46,19 +54,7 @@ describe('AppSec Rule Manager', () => { loadRules(config.appsec) expect(waf.init).to.have.been.calledOnceWithExactly(testRules, config.appsec) - expect(blocking.updateBlockingConfiguration).to.have.been.calledOnceWithExactly({ - id: 'block', - otherParam: 'other' - }) - }) - - it('should throw if null/undefined are passed', () => { - // TODO: fix the exception thrown in the waf or catch it in rule_manager? - config.appsec.rules = './not/existing/file.json' - expect(() => { loadRules(config.appsec) }).to.throw() - - config.appsec.rules = './bad-formatted-rules.json' - expect(() => { loadRules(config.appsec) }).to.throw() + expect(blocking.setDefaultBlockingActionParameters).to.have.been.calledOnceWithExactly(testRules.actions) }) }) @@ -67,9 +63,11 @@ describe('AppSec Rule Manager', () => { loadRules(config.appsec) expect(waf.init).to.have.been.calledOnce + blocking.setDefaultBlockingActionParameters.resetHistory() + clearAllRules() expect(waf.destroy).to.have.been.calledOnce - expect(blocking.updateBlockingConfiguration).to.have.been.calledOnceWithExactly(undefined) + expect(blocking.setDefaultBlockingActionParameters).to.have.been.calledOnceWithExactly(undefined) }) }) @@ -290,16 +288,43 @@ describe('AppSec Rule Manager', () => { it('should apply new rules', () => { const testRules = { version: '2.2', - metadata: { 'rules_version': '1.5.0' }, + metadata: { rules_version: '1.5.0' }, rules: [{ - 'id': 'test-id', - 'name': 'test-name', - 'tags': { - 'type': 'security_scanner', - 'category': 'attack_attempt', - 'confidence': '1' + id: 'test-id', + name: 'test-name', + tags: { + type: 'security_scanner', + category: 'attack_attempt', + confidence: '1' + }, + conditions: [] + }], + processors: [{ + id: 'test-processor-id', + generator: 'test-generator', + evaluate: false, + output: true + }], + scanners: [{ + id: 'test-scanner-id', + name: 'Test name', + key: { + operator: 'match_regex', + parameters: { + regex: 'test-regex' + } }, - 'conditions': [] + value: { + operator: 'match_regex', + parameters: { + regex: 'test-regex-2' + } + }, + tags: { + type: 'card', + card_type: 'test', + category: 'payment' + } }] } @@ -323,16 +348,43 @@ describe('AppSec Rule Manager', () => { } const testRules = { version: '2.2', - metadata: { 'rules_version': '1.5.0' }, + metadata: { rules_version: '1.5.0' }, rules: [{ - 'id': 'test-id', - 'name': 'test-name', - 'tags': { - 'type': 'security_scanner', - 'category': 'attack_attempt', - 'confidence': '1' + id: 'test-id', + name: 'test-name', + tags: { + type: 'security_scanner', + category: 'attack_attempt', + confidence: '1' + }, + conditions: [] + }], + processors: [{ + id: 'test-processor-id', + generator: 'test-generator', + evaluate: false, + output: true + }], + scanners: [{ + id: 'test-scanner-id', + name: 'Test name', + key: { + operator: 'match_regex', + parameters: { + regex: 'test-regex' + } }, - 'conditions': [] + value: { + operator: 'match_regex', + parameters: { + regex: 'test-regex-2' + } + }, + tags: { + type: 'card', + card_type: 'test', + category: 'payment' + } }] } @@ -359,14 +411,14 @@ describe('AppSec Rule Manager', () => { id: 'rules1', file: { version: '2.2', - metadata: { 'rules_version': '1.5.0' }, + metadata: { rules_version: '1.5.0' }, rules: [{ - 'id': 'test-id', - 'name': 'test-name', - 'tags': { - 'type': 'security_scanner', - 'category': 'attack_attempt', - 'confidence': '1' + id: 'test-id', + name: 'test-name', + tags: { + type: 'security_scanner', + category: 'attack_attempt', + confidence: '1' }, conditions: [ { @@ -388,14 +440,14 @@ describe('AppSec Rule Manager', () => { id: 'rules2', file: { version: '2.2', - metadata: { 'rules_version': '1.5.0' }, + metadata: { rules_version: '1.5.0' }, rules: [{ - 'id': 'test-id', - 'name': 'test-name', - 'tags': { - 'type': 'security_scanner', - 'category': 'attack_attempt', - 'confidence': '1' + id: 'test-id', + name: 'test-name', + tags: { + type: 'security_scanner', + category: 'attack_attempt', + confidence: '1' }, conditions: [ { @@ -426,13 +478,13 @@ describe('AppSec Rule Manager', () => { describe('ASM', () => { it('should apply both rules_override and exclusions', () => { const asm = { - 'exclusions': [{ + exclusions: [{ ekey: 'eValue' }], - 'rules_override': [{ + rules_override: [{ roKey: 'roValue' }], - 'custom_rules': [{ + custom_rules: [{ piKey: 'piValue' }] } @@ -451,35 +503,62 @@ describe('AppSec Rule Manager', () => { }) it('should apply blocking actions', () => { - const asm = { - actions: [ - { - id: 'block', - otherParam: 'other' - }, - { - id: 'otherId', - moreParams: 'more' - } - ] - } - const toApply = [ { product: 'ASM', id: '1', - file: asm + file: { + actions: [ + { + id: 'notblock', + parameters: { + location: '/notfound', + status_code: 404 + } + } + ] + } + }, + { + product: 'ASM', + id: '2', + file: { + actions: [ + { + id: 'block', + parameters: { + location: '/redirected', + status_code: 302 + } + } + ] + } } ] updateWafFromRC({ toUnapply: [], toApply, toModify: [] }) - expect(waf.update).not.to.have.been.called - expect(blocking.updateBlockingConfiguration).to.have.been.calledOnceWithExactly( - { - id: 'block', - otherParam: 'other' - }) + const expectedPayload = { + actions: [ + { + id: 'notblock', + parameters: { + location: '/notfound', + status_code: 404 + } + }, + { + id: 'block', + parameters: { + location: '/redirected', + status_code: 302 + } + } + ] + } + + expect(waf.update).to.have.been.calledOnceWithExactly(expectedPayload) + expect(blocking.setDefaultBlockingActionParameters).to.have.been.calledOnceWithExactly(expectedPayload.actions) }) it('should unapply blocking actions', () => { @@ -503,8 +582,11 @@ describe('AppSec Rule Manager', () => { } ] updateWafFromRC({ toUnapply: [], toApply, toModify: [] }) - // reset counters - blocking.updateBlockingConfiguration.reset() + + expect(waf.update).to.have.been.calledOnceWithExactly(asm) + expect(blocking.setDefaultBlockingActionParameters).to.have.been.calledOnceWithExactly(asm.actions) + + sinon.resetHistory() const toUnapply = [ { @@ -515,19 +597,19 @@ describe('AppSec Rule Manager', () => { updateWafFromRC({ toUnapply, toApply: [], toModify: [] }) - expect(waf.update).not.to.have.been.called - expect(blocking.updateBlockingConfiguration).to.have.been.calledOnceWithExactly(undefined) + expect(waf.update).to.have.been.calledOnceWithExactly({ actions: [] }) + expect(blocking.setDefaultBlockingActionParameters).to.have.been.calledOnceWithExactly([]) }) it('should ignore other properties', () => { const asm = { - 'exclusions': [{ + exclusions: [{ ekey: 'eValue' }], - 'rules_override': [{ + rules_override: [{ roKey: 'roValue' }], - 'not_supported': [{ + not_supported: [{ nsKey: 'nsValue' }] } @@ -543,8 +625,8 @@ describe('AppSec Rule Manager', () => { updateWafFromRC({ toUnapply: [], toApply, toModify: [] }) expect(waf.update).to.have.been.calledOnceWithExactly({ - 'exclusions': asm['exclusions'], - 'rules_override': asm['rules_override'] + exclusions: asm.exclusions, + rules_override: asm.rules_override }) }) }) diff --git a/packages/dd-trace/test/appsec/sdk/set_user.spec.js b/packages/dd-trace/test/appsec/sdk/set_user.spec.js index a582837e419..9327a88afcd 100644 --- a/packages/dd-trace/test/appsec/sdk/set_user.spec.js +++ b/packages/dd-trace/test/appsec/sdk/set_user.spec.js @@ -3,7 +3,6 @@ const proxyquire = require('proxyquire') const agent = require('../../plugins/agent') const tracer = require('../../../../../index') -const getPort = require('get-port') const axios = require('axios') describe('set_user', () => { @@ -83,7 +82,6 @@ describe('set_user', () => { } before(async () => { - port = await getPort() await agent.load('http') http = require('http') }) @@ -91,7 +89,10 @@ describe('set_user', () => { before(done => { const server = new http.Server(listener) appListener = server - .listen(port, 'localhost', () => done()) + .listen(port, 'localhost', () => { + port = appListener.address().port + done() + }) }) after(() => { diff --git a/packages/dd-trace/test/appsec/sdk/track_event.spec.js b/packages/dd-trace/test/appsec/sdk/track_event.spec.js index 93d8959783e..e3739488b81 100644 --- a/packages/dd-trace/test/appsec/sdk/track_event.spec.js +++ b/packages/dd-trace/test/appsec/sdk/track_event.spec.js @@ -2,9 +2,9 @@ const proxyquire = require('proxyquire') const agent = require('../../plugins/agent') -const getPort = require('get-port') const axios = require('axios') const tracer = require('../../../../../index') +const { LOGIN_SUCCESS, LOGIN_FAILURE } = require('../../../src/appsec/addresses') describe('track_event', () => { describe('Internal API', () => { @@ -14,6 +14,8 @@ describe('track_event', () => { let getRootSpan let setUserTags let trackUserLoginSuccessEvent, trackUserLoginFailureEvent, trackCustomEvent, trackEvent + let sample + let waf beforeEach(() => { log = { @@ -28,6 +30,12 @@ describe('track_event', () => { setUserTags = sinon.stub() + sample = sinon.stub() + + waf = { + run: sinon.spy() + } + const trackEvents = proxyquire('../../../src/appsec/sdk/track_event', { '../../log': log, './utils': { @@ -35,7 +43,11 @@ describe('track_event', () => { }, './set_user': { setUserTags - } + }, + '../standalone': { + sample + }, + '../waf': waf }) trackUserLoginSuccessEvent = trackEvents.trackUserLoginSuccessEvent @@ -44,6 +56,10 @@ describe('track_event', () => { trackEvent = trackEvents.trackEvent }) + afterEach(() => { + sinon.restore() + }) + describe('trackUserLoginSuccessEvent', () => { it('should log warning when passed invalid user', () => { trackUserLoginSuccessEvent(tracer, null, { key: 'value' }) @@ -101,6 +117,16 @@ describe('track_event', () => { '_dd.appsec.events.users.login.success.sdk': 'true' }) }) + + it('should call waf run with login success address', () => { + const user = { id: 'user_id' } + + trackUserLoginSuccessEvent(tracer, user) + sinon.assert.calledOnceWithExactly( + waf.run, + { persistent: { [LOGIN_SUCCESS]: null } } + ) + }) }) describe('trackUserLoginFailureEvent', () => { @@ -177,6 +203,14 @@ describe('track_event', () => { 'appsec.events.users.login.failure.usr.exists': 'true' }) }) + + it('should call waf run with login failure address', () => { + trackUserLoginFailureEvent(tracer, 'user_id') + sinon.assert.calledOnceWithExactly( + waf.run, + { persistent: { [LOGIN_FAILURE]: null } } + ) + }) }) describe('trackCustomEvent', () => { @@ -249,6 +283,16 @@ describe('track_event', () => { 'appsec.events.event.metakey2': 'metaValue2' }) }) + + it('should call standalone sample', () => { + trackEvent('event', undefined, 'trackEvent', rootSpan, undefined) + + expect(rootSpan.addTags).to.have.been.calledOnceWithExactly({ + 'appsec.events.event.track': 'true', + 'manual.keep': 'true' + }) + expect(sample).to.have.been.calledOnceWithExactly(rootSpan) + }) }) }) @@ -265,7 +309,6 @@ describe('track_event', () => { } before(async () => { - port = await getPort() await agent.load('http') http = require('http') }) @@ -273,7 +316,10 @@ describe('track_event', () => { before(done => { const server = new http.Server(listener) appListener = server - .listen(port, 'localhost', () => done()) + .listen(port, 'localhost', () => { + port = appListener.address().port + done() + }) }) after(() => { diff --git a/packages/dd-trace/test/appsec/sdk/user_blocking.spec.js b/packages/dd-trace/test/appsec/sdk/user_blocking.spec.js index 64579a94eee..6df68104e85 100644 --- a/packages/dd-trace/test/appsec/sdk/user_blocking.spec.js +++ b/packages/dd-trace/test/appsec/sdk/user_blocking.spec.js @@ -5,11 +5,19 @@ const agent = require('../../plugins/agent') const tracer = require('../../../../../index') const appsec = require('../../../src/appsec') const Config = require('../../../src/config') -const getPort = require('get-port') const axios = require('axios') const path = require('path') const waf = require('../../../src/appsec/waf') const { USER_ID } = require('../../../src/appsec/addresses') +const blocking = require('../../../src/appsec/blocking') + +const resultActions = { + block_request: { + status_code: '401', + type: 'auto', + grpc_status_code: '10' + } +} describe('user_blocking', () => { describe('Internal API', () => { @@ -21,8 +29,8 @@ describe('user_blocking', () => { before(() => { const runStub = sinon.stub(waf, 'run') - runStub.withArgs({ [USER_ID]: 'user' }).returns(['block']) - runStub.withArgs({ [USER_ID]: 'gooduser' }).returns(['']) + runStub.withArgs({ persistent: { [USER_ID]: 'user' } }).returns(resultActions) + runStub.withArgs({ persistent: { [USER_ID]: 'gooduser' } }).returns({}) }) beforeEach(() => { @@ -158,7 +166,6 @@ describe('user_blocking', () => { } before(async () => { - port = await getPort() await agent.load('http') http = require('http') }) @@ -166,7 +173,10 @@ describe('user_blocking', () => { before(done => { const server = new http.Server(listener) appListener = server - .listen(port, 'localhost', () => done()) + .listen(port, 'localhost', () => { + port = appListener.address().port + done() + }) appsec.enable(config) }) @@ -219,6 +229,16 @@ describe('user_blocking', () => { }) describe('blockRequest', () => { + beforeEach(() => { + // reset to default, other tests may have changed it with RC + blocking.setDefaultBlockingActionParameters(undefined) + }) + + afterEach(() => { + // reset to default + blocking.setDefaultBlockingActionParameters(undefined) + }) + it('should set the proper tags', (done) => { controller = (req, res) => { const ret = tracer.appsec.blockRequest(req, res) @@ -255,6 +275,34 @@ describe('user_blocking', () => { }).then(done).catch(done) axios.get(`http://localhost:${port}/`) }) + + it('should block using redirect data if it is configured', (done) => { + blocking.setDefaultBlockingActionParameters([ + { + id: 'notblock', + parameters: { + location: '/notfound', + status_code: 404 + } + }, + { + id: 'block', + parameters: { + location: '/redirected', + status_code: 302 + } + } + ]) + controller = (req, res) => { + const ret = tracer.appsec.blockRequest(req, res) + expect(ret).to.be.true + } + agent.use(traces => { + expect(traces[0][0].meta).to.have.property('appsec.blocked', 'true') + expect(traces[0][0].meta).to.have.property('http.status_code', '302') + }).then(done).catch(done) + axios.get(`http://localhost:${port}/`, { maxRedirects: 0 }) + }) }) }) }) diff --git a/packages/dd-trace/test/appsec/stack_trace.spec.js b/packages/dd-trace/test/appsec/stack_trace.spec.js new file mode 100644 index 00000000000..1ac2ca4db5e --- /dev/null +++ b/packages/dd-trace/test/appsec/stack_trace.spec.js @@ -0,0 +1,357 @@ +'use strict' + +const { assert } = require('chai') +const path = require('path') + +const { reportStackTrace } = require('../../src/appsec/stack_trace') + +describe('Stack trace reporter', () => { + describe('frame filtering', () => { + it('should filer out frames from library', () => { + const callSiteList = + Array(10).fill().map((_, i) => ( + { + getFileName: () => path.join(__dirname, `file${i}`), + getLineNumber: () => i, + getColumnNumber: () => i, + getFunctionName: () => `libraryFunction${i}`, + getTypeName: () => `LibraryClass${i}` + } + )).concat( + Array(10).fill().map((_, i) => ( + { + getFileName: () => `file${i}`, + getLineNumber: () => i, + getColumnNumber: () => i, + getFunctionName: () => `function${i}`, + getTypeName: () => `Class${i}` + } + )) + ).concat([ + { + getFileName: () => null, + getLineNumber: () => null, + getColumnNumber: () => null, + getFunctionName: () => null, + getTypeName: () => null + } + ]) + + const expectedFrames = Array(10).fill().map((_, i) => ( + { + id: i, + file: `file${i}`, + line: i, + column: i, + function: `function${i}`, + class_name: `Class${i}` + } + )) + .concat([ + { + id: 10, + file: null, + line: null, + column: null, + function: null, + class_name: null + } + ]) + + const rootSpan = {} + const stackId = 'test_stack_id' + const maxDepth = 32 + const maxStackTraces = 2 + reportStackTrace(rootSpan, stackId, maxDepth, maxStackTraces, () => callSiteList) + + assert.deepEqual(rootSpan.meta_struct['_dd.stack'].exploit[0].frames, expectedFrames) + }) + }) + + describe('report stack traces', () => { + const callSiteList = Array(20).fill().map((_, i) => ( + { + getFileName: () => `file${i}`, + getLineNumber: () => i, + getColumnNumber: () => i, + getFunctionName: () => `function${i}`, + getTypeName: () => `type${i}` + } + )) + + it('should not fail if no root span is passed', () => { + const rootSpan = undefined + const stackId = 'test_stack_id' + const maxDepth = 32 + try { + reportStackTrace(rootSpan, stackId, maxDepth, 2, () => callSiteList) + } catch (e) { + assert.fail() + } + }) + + it('should add stack trace to rootSpan when meta_struct is not present', () => { + const rootSpan = {} + const stackId = 'test_stack_id' + const maxDepth = 32 + const expectedFrames = Array(20).fill().map((_, i) => ( + { + id: i, + file: `file${i}`, + line: i, + column: i, + function: `function${i}`, + class_name: `type${i}` + } + )) + + reportStackTrace(rootSpan, stackId, maxDepth, 2, () => callSiteList) + + assert.strictEqual(rootSpan.meta_struct['_dd.stack'].exploit[0].id, stackId) + assert.strictEqual(rootSpan.meta_struct['_dd.stack'].exploit[0].language, 'nodejs') + assert.deepEqual(rootSpan.meta_struct['_dd.stack'].exploit[0].frames, expectedFrames) + }) + + it('should add stack trace to rootSpan when meta_struct is already present', () => { + const rootSpan = { + meta_struct: { + another_tag: [] + } + } + const stackId = 'test_stack_id' + const maxDepth = 32 + const expectedFrames = Array(20).fill().map((_, i) => ( + { + id: i, + file: `file${i}`, + line: i, + column: i, + function: `function${i}`, + class_name: `type${i}` + } + )) + + reportStackTrace(rootSpan, stackId, maxDepth, 2, () => callSiteList) + + assert.strictEqual(rootSpan.meta_struct['_dd.stack'].exploit[0].id, stackId) + assert.strictEqual(rootSpan.meta_struct['_dd.stack'].exploit[0].language, 'nodejs') + assert.deepEqual(rootSpan.meta_struct['_dd.stack'].exploit[0].frames, expectedFrames) + assert.property(rootSpan.meta_struct, 'another_tag') + }) + + it('should add stack trace to rootSpan when meta_struct is already present and contains another stack', () => { + const rootSpan = { + meta_struct: { + another_tag: [], + '_dd.stack': { + exploit: [callSiteList] + } + } + } + const stackId = 'test_stack_id' + const maxDepth = 32 + const expectedFrames = Array(20).fill().map((_, i) => ( + { + id: i, + file: `file${i}`, + line: i, + column: i, + function: `function${i}`, + class_name: `type${i}` + } + )) + + reportStackTrace(rootSpan, stackId, maxDepth, 2, () => callSiteList) + + assert.strictEqual(rootSpan.meta_struct['_dd.stack'].exploit[1].id, stackId) + assert.strictEqual(rootSpan.meta_struct['_dd.stack'].exploit[1].language, 'nodejs') + assert.deepEqual(rootSpan.meta_struct['_dd.stack'].exploit[1].frames, expectedFrames) + assert.property(rootSpan.meta_struct, 'another_tag') + }) + + it('should not report stack trace when the maximum has been reached', () => { + const rootSpan = { + meta_struct: { + '_dd.stack': { + exploit: [callSiteList, callSiteList] + }, + another_tag: [] + } + } + const stackId = 'test_stack_id' + const maxDepth = 32 + + reportStackTrace(rootSpan, stackId, maxDepth, 2, () => callSiteList) + + assert.equal(rootSpan.meta_struct['_dd.stack'].exploit.length, 2) + assert.property(rootSpan.meta_struct, 'another_tag') + }) + + it('should add stack trace when the max stack trace is 0', () => { + const rootSpan = { + meta_struct: { + '_dd.stack': { + exploit: [callSiteList, callSiteList] + }, + another_tag: [] + } + } + const stackId = 'test_stack_id' + const maxDepth = 32 + + reportStackTrace(rootSpan, stackId, maxDepth, 0, () => callSiteList) + + assert.equal(rootSpan.meta_struct['_dd.stack'].exploit.length, 3) + assert.property(rootSpan.meta_struct, 'another_tag') + }) + + it('should add stack trace when the max stack trace is negative', () => { + const rootSpan = { + meta_struct: { + '_dd.stack': { + exploit: [callSiteList, callSiteList] + }, + another_tag: [] + } + } + const stackId = 'test_stack_id' + const maxDepth = 32 + + reportStackTrace(rootSpan, stackId, maxDepth, -1, () => callSiteList) + + assert.equal(rootSpan.meta_struct['_dd.stack'].exploit.length, 3) + assert.property(rootSpan.meta_struct, 'another_tag') + }) + + it('should not report stackTraces if callSiteList is undefined', () => { + const rootSpan = { + meta_struct: { + another_tag: [] + } + } + const stackId = 'test_stack_id' + const maxDepth = 32 + const maxStackTraces = 2 + reportStackTrace(rootSpan, stackId, maxDepth, maxStackTraces, () => undefined) + assert.property(rootSpan.meta_struct, 'another_tag') + assert.notProperty(rootSpan.meta_struct, '_dd.stack') + }) + }) + + describe('limit stack traces frames', () => { + const callSiteList = Array(120).fill().map((_, i) => ( + { + getFileName: () => `file${i}`, + getLineNumber: () => i, + getColumnNumber: () => i, + getFunctionName: () => `function${i}`, + getTypeName: () => `type${i}` + } + )) + + it('limit frames to max depth', () => { + const rootSpan = {} + const stackId = 'test_stack_id' + const maxDepth = 5 + const expectedFrames = [0, 1, 2, 118, 119].map(i => ( + { + id: i, + file: `file${i}`, + line: i, + column: i, + function: `function${i}`, + class_name: `type${i}` + } + )) + + reportStackTrace(rootSpan, stackId, maxDepth, 2, () => callSiteList) + + assert.deepEqual(rootSpan.meta_struct['_dd.stack'].exploit[0].frames, expectedFrames) + }) + + it('limit frames to max depth with filtered frames', () => { + const rootSpan = {} + const stackId = 'test_stack_id' + const maxDepth = 5 + const callSiteListWithLibraryFrames = [ + { + getFileName: () => path.join(__dirname, 'firstFrame'), + getLineNumber: () => 314, + getColumnNumber: () => 271, + getFunctionName: () => 'libraryFunction', + getTypeName: () => 'libraryType' + } + ].concat(Array(120).fill().map((_, i) => ( + { + getFileName: () => `file${i}`, + getLineNumber: () => i, + getColumnNumber: () => i, + getFunctionName: () => `function${i}`, + getTypeName: () => `type${i}` + } + )).concat([ + { + getFileName: () => path.join(__dirname, 'lastFrame'), + getLineNumber: () => 271, + getColumnNumber: () => 314, + getFunctionName: () => 'libraryFunction', + getTypeName: () => 'libraryType' + } + ])) + const expectedFrames = [0, 1, 2, 118, 119].map(i => ( + { + id: i, + file: `file${i}`, + line: i, + column: i, + function: `function${i}`, + class_name: `type${i}` + } + )) + + reportStackTrace(rootSpan, stackId, maxDepth, 2, () => callSiteListWithLibraryFrames) + + assert.deepEqual(rootSpan.meta_struct['_dd.stack'].exploit[0].frames, expectedFrames) + }) + + it('no limit if maxDepth is 0', () => { + const rootSpan = {} + const stackId = 'test_stack_id' + const maxDepth = 0 + const expectedFrames = Array(120).fill().map((_, i) => ( + { + id: i, + file: `file${i}`, + line: i, + column: i, + function: `function${i}`, + class_name: `type${i}` + } + )) + + reportStackTrace(rootSpan, stackId, maxDepth, 2, () => callSiteList) + + assert.deepEqual(rootSpan.meta_struct['_dd.stack'].exploit[0].frames, expectedFrames) + }) + + it('no limit if maxDepth is negative', () => { + const rootSpan = {} + const stackId = 'test_stack_id' + const maxDepth = -1 + const expectedFrames = Array(120).fill().map((_, i) => ( + { + id: i, + file: `file${i}`, + line: i, + column: i, + function: `function${i}`, + class_name: `type${i}` + } + )) + + reportStackTrace(rootSpan, stackId, maxDepth, 2, () => callSiteList) + + assert.deepEqual(rootSpan.meta_struct['_dd.stack'].exploit[0].frames, expectedFrames) + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/standalone.spec.js b/packages/dd-trace/test/appsec/standalone.spec.js new file mode 100644 index 00000000000..027e23c3b5e --- /dev/null +++ b/packages/dd-trace/test/appsec/standalone.spec.js @@ -0,0 +1,455 @@ +'use strict' + +const { channel } = require('dc-polyfill') +const { assert } = require('chai') +const standalone = require('../../src/appsec/standalone') +const DatadogSpan = require('../../src/opentracing/span') +const { + APM_TRACING_ENABLED_KEY, + APPSEC_PROPAGATION_KEY, + SAMPLING_MECHANISM_APPSEC, + DECISION_MAKER_KEY +} = require('../../src/constants') +const { USER_KEEP, AUTO_KEEP, AUTO_REJECT, USER_REJECT } = require('../../../../ext/priority') +const TextMapPropagator = require('../../src/opentracing/propagation/text_map') +const TraceState = require('../../src/opentracing/propagation/tracestate') + +const startCh = channel('dd-trace:span:start') +const injectCh = channel('dd-trace:span:inject') +const extractCh = channel('dd-trace:span:extract') + +describe('Appsec Standalone', () => { + let config + let tracer, processor, prioritySampler + + beforeEach(() => { + config = { + appsec: { standalone: { enabled: true } }, + + tracePropagationStyle: { + inject: ['datadog', 'tracecontext'], + extract: ['datadog'] + } + } + + tracer = {} + processor = {} + prioritySampler = {} + }) + + afterEach(() => { sinon.restore() }) + + describe('configure', () => { + let startChSubscribe + let startChUnsubscribe + let injectChSubscribe + let injectChUnsubscribe + let extractChSubscribe + let extractChUnsubscribe + + beforeEach(() => { + startChSubscribe = sinon.stub(startCh, 'subscribe') + startChUnsubscribe = sinon.stub(startCh, 'unsubscribe') + injectChSubscribe = sinon.stub(injectCh, 'subscribe') + injectChUnsubscribe = sinon.stub(injectCh, 'unsubscribe') + extractChSubscribe = sinon.stub(extractCh, 'subscribe') + extractChUnsubscribe = sinon.stub(extractCh, 'unsubscribe') + }) + + it('should subscribe to start span if standalone enabled', () => { + standalone.configure(config) + + sinon.assert.calledOnce(startChSubscribe) + sinon.assert.calledOnce(injectChSubscribe) + sinon.assert.calledOnce(extractChSubscribe) + }) + + it('should not subscribe to start span if standalone disabled', () => { + delete config.appsec.standalone + + standalone.configure(config) + + sinon.assert.notCalled(startChSubscribe) + sinon.assert.notCalled(injectChSubscribe) + sinon.assert.notCalled(extractChSubscribe) + sinon.assert.notCalled(startChUnsubscribe) + sinon.assert.notCalled(injectChUnsubscribe) + sinon.assert.notCalled(extractChUnsubscribe) + }) + + it('should subscribe only once', () => { + standalone.configure(config) + standalone.configure(config) + standalone.configure(config) + + sinon.assert.calledOnce(startChSubscribe) + }) + + it('should not return a prioritySampler when standalone ASM is disabled', () => { + const prioritySampler = standalone.configure({ appsec: { standalone: { enabled: false } } }) + + assert.isUndefined(prioritySampler) + }) + + it('should return a StandAloneAsmPrioritySampler when standalone ASM is enabled', () => { + const prioritySampler = standalone.configure(config) + + assert.instanceOf(prioritySampler, standalone.StandAloneAsmPrioritySampler) + }) + }) + + describe('sample', () => { + it('should add _dd.p.appsec tag if enabled', () => { + standalone.configure(config) + + const span = new DatadogSpan(tracer, processor, prioritySampler, { + operationName: 'operation' + }) + + standalone.sample(span) + + assert.propertyVal(span.context()._trace.tags, APPSEC_PROPAGATION_KEY, '1') + }) + + it('should reset priority', () => { + standalone.configure(config) + + const span = new DatadogSpan(tracer, processor, prioritySampler, { + operationName: 'operation' + }) + + span.context()._sampling.priority = USER_REJECT + + standalone.sample(span) + + assert.strictEqual(span.context()._sampling.priority, undefined) + }) + + it('should not add _dd.p.appsec tag if disabled', () => { + delete config.appsec.standalone + + standalone.configure(config) + + const span = new DatadogSpan(tracer, processor, prioritySampler, { + operationName: 'operation' + }) + + standalone.sample(span) + + assert.notProperty(span.context()._trace.tags, APPSEC_PROPAGATION_KEY) + }) + }) + + describe('onStartSpan', () => { + it('should not add _dd.apm.enabled tag when standalone is disabled', () => { + delete config.appsec.standalone + standalone.configure(config) + + const span = new DatadogSpan(tracer, processor, prioritySampler, { + operationName: 'operation' + }) + + assert.notProperty(span.context()._tags, APM_TRACING_ENABLED_KEY) + }) + + it('should add _dd.apm.enabled tag when standalone is enabled', () => { + standalone.configure(config) + + const span = new DatadogSpan(tracer, processor, prioritySampler, { + operationName: 'operation' + }) + + assert.property(span.context()._tags, APM_TRACING_ENABLED_KEY) + }) + + it('should not add _dd.apm.enabled tag in child spans with local parent', () => { + standalone.configure(config) + + const parent = new DatadogSpan(tracer, processor, prioritySampler, { + operationName: 'operation' + }) + + assert.propertyVal(parent.context()._tags, APM_TRACING_ENABLED_KEY, 0) + + const child = new DatadogSpan(tracer, processor, prioritySampler, { + operationName: 'operation', + parent + }) + + assert.notProperty(child.context()._tags, APM_TRACING_ENABLED_KEY) + }) + + it('should add _dd.apm.enabled tag in child spans with remote parent', () => { + standalone.configure(config) + + const parent = new DatadogSpan(tracer, processor, prioritySampler, { + operationName: 'operation' + }) + + parent._isRemote = true + + const child = new DatadogSpan(tracer, processor, prioritySampler, { + operationName: 'operation', + parent + }) + + assert.propertyVal(child.context()._tags, APM_TRACING_ENABLED_KEY, 0) + }) + }) + + describe('onSpanExtract', () => { + it('should reset priority if _dd.p.appsec not present', () => { + standalone.configure(config) + + const carrier = { + 'x-datadog-trace-id': 123123, + 'x-datadog-parent-id': 345345, + 'x-datadog-sampling-priority': 2 + } + + const propagator = new TextMapPropagator(config) + const spanContext = propagator.extract(carrier) + + assert.isUndefined(spanContext._sampling.priority) + }) + + it('should not reset dm if _dd.p.appsec not present', () => { + standalone.configure(config) + + const carrier = { + 'x-datadog-trace-id': 123123, + 'x-datadog-parent-id': 345345, + 'x-datadog-sampling-priority': 2, + 'x-datadog-tags': '_dd.p.dm=-4' + } + + const propagator = new TextMapPropagator(config) + const spanContext = propagator.extract(carrier) + + assert.propertyVal(spanContext._trace.tags, DECISION_MAKER_KEY, '-4') + }) + + it('should keep priority if _dd.p.appsec is present', () => { + standalone.configure(config) + + const carrier = { + 'x-datadog-trace-id': 123123, + 'x-datadog-parent-id': 345345, + 'x-datadog-sampling-priority': 2, + 'x-datadog-tags': '_dd.p.appsec=1,_dd.p.dm=-5' + } + + const propagator = new TextMapPropagator(config) + const spanContext = propagator.extract(carrier) + + assert.strictEqual(spanContext._sampling.priority, USER_KEEP) + assert.propertyVal(spanContext._trace.tags, DECISION_MAKER_KEY, '-5') + }) + + it('should set USER_KEEP priority if _dd.p.appsec=1 is present', () => { + standalone.configure(config) + + const carrier = { + 'x-datadog-trace-id': 123123, + 'x-datadog-parent-id': 345345, + 'x-datadog-sampling-priority': 1, + 'x-datadog-tags': '_dd.p.appsec=1' + } + + const propagator = new TextMapPropagator(config) + const spanContext = propagator.extract(carrier) + + assert.strictEqual(spanContext._sampling.priority, USER_KEEP) + }) + + it('should keep priority if standalone is disabled', () => { + delete config.appsec.standalone + standalone.configure(config) + + const carrier = { + 'x-datadog-trace-id': 123123, + 'x-datadog-parent-id': 345345, + 'x-datadog-sampling-priority': 2 + } + + const propagator = new TextMapPropagator(config) + const spanContext = propagator.extract(carrier) + + assert.strictEqual(spanContext._sampling.priority, USER_KEEP) + }) + }) + + describe('onSpanInject', () => { + it('should reset priority if standalone enabled and there is no appsec event', () => { + standalone.configure(config) + + const span = new DatadogSpan(tracer, processor, prioritySampler, { + operationName: 'operation' + }) + + span._spanContext._sampling = { + priority: USER_KEEP, + mechanism: SAMPLING_MECHANISM_APPSEC + } + + const carrier = {} + const propagator = new TextMapPropagator(config) + propagator.inject(span._spanContext, carrier) + + assert.notProperty(carrier, 'x-datadog-trace-id') + assert.notProperty(carrier, 'x-datadog-parent-id') + assert.notProperty(carrier, 'x-datadog-sampling-priority') + }) + + it('should keep priority if standalone enabled and there is an appsec event', () => { + standalone.configure(config) + + const span = new DatadogSpan(tracer, processor, prioritySampler, { + operationName: 'operation' + }) + + span._spanContext._sampling = { + priority: USER_KEEP, + mechanism: SAMPLING_MECHANISM_APPSEC + } + + span._spanContext._trace.tags[APPSEC_PROPAGATION_KEY] = '1' + + const carrier = {} + const propagator = new TextMapPropagator(config) + propagator.inject(span._spanContext, carrier) + + assert.property(carrier, 'x-datadog-trace-id') + assert.property(carrier, 'x-datadog-parent-id') + assert.property(carrier, 'x-datadog-sampling-priority') + assert.propertyVal(carrier, 'x-datadog-tags', '_dd.p.appsec=1') + }) + + it('should not reset priority if standalone disabled', () => { + delete config.appsec.standalone + standalone.configure(config) + + const span = new DatadogSpan(tracer, processor, prioritySampler, { + operationName: 'operation' + }) + + span._spanContext._sampling = { + priority: USER_KEEP, + mechanism: SAMPLING_MECHANISM_APPSEC + } + + const carrier = {} + const propagator = new TextMapPropagator(config) + propagator.inject(span._spanContext, carrier) + + assert.property(carrier, 'x-datadog-trace-id') + assert.property(carrier, 'x-datadog-parent-id') + assert.property(carrier, 'x-datadog-sampling-priority') + }) + + it('should clear tracestate datadog info', () => { + standalone.configure(config) + + const span = new DatadogSpan(tracer, processor, prioritySampler, { + operationName: 'operation' + }) + + span._spanContext._sampling = { + priority: USER_KEEP, + mechanism: SAMPLING_MECHANISM_APPSEC + } + + const tracestate = new TraceState() + tracestate.set('dd', 't.tid:666b118100000000;t.dm:-1;s:1;p:73a164d716fcddff') + tracestate.set('other', 'id:0xC0FFEE') + span._spanContext._tracestate = tracestate + + const carrier = {} + const propagator = new TextMapPropagator(config) + propagator.inject(span._spanContext, carrier) + + assert.propertyVal(carrier, 'tracestate', 'other=id:0xC0FFEE') + }) + }) + + describe('StandaloneASMPriorityManager', () => { + let prioritySampler + let tags + let context + let root + + beforeEach(() => { + tags = { 'manual.keep': 'true' } + prioritySampler = new standalone.StandAloneAsmPrioritySampler('test') + + root = {} + context = { + _sampling: {}, + _trace: { + tags: {}, + started: [root] + } + } + sinon.stub(prioritySampler, '_getContext').returns(context) + }) + + describe('sample', () => { + it('should provide the context when invoking _getPriorityFromTags', () => { + const span = new DatadogSpan(tracer, processor, prioritySampler, { + operationName: 'operation' + }) + + const _getPriorityFromTags = sinon.stub(prioritySampler, '_getPriorityFromTags') + + prioritySampler.sample(span, false) + + sinon.assert.calledWithExactly(_getPriorityFromTags, context._tags, context) + }) + }) + + describe('_getPriorityFromTags', () => { + it('should keep the trace if manual.keep and _dd.p.appsec are present', () => { + context._trace.tags[APPSEC_PROPAGATION_KEY] = 1 + assert.strictEqual(prioritySampler._getPriorityFromTags(tags, context), USER_KEEP) + }) + + it('should return undefined if manual.keep or _dd.p.appsec are not present', () => { + assert.isUndefined(prioritySampler._getPriorityFromTags(tags, context)) + }) + }) + + describe('_getPriorityFromAuto', () => { + it('should keep one trace per 1 min', () => { + const span = { + _trace: {} + } + + const clock = sinon.useFakeTimers() + + assert.strictEqual(prioritySampler._getPriorityFromAuto(span), AUTO_KEEP) + + assert.strictEqual(prioritySampler._getPriorityFromAuto(span), AUTO_REJECT) + + clock.tick(30000) + + assert.strictEqual(prioritySampler._getPriorityFromAuto(span), AUTO_REJECT) + + clock.tick(60000) + + assert.strictEqual(prioritySampler._getPriorityFromAuto(span), AUTO_KEEP) + + clock.restore() + }) + + it('should keep trace if it contains _dd.p.appsec tag', () => { + const span = { + _trace: {} + } + + context._trace.tags[APPSEC_PROPAGATION_KEY] = 1 + + assert.strictEqual(prioritySampler._getPriorityFromAuto(span), USER_KEEP) + }) + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/streamtest.txt b/packages/dd-trace/test/appsec/streamtest.txt new file mode 100644 index 00000000000..1a010b1c0f0 --- /dev/null +++ b/packages/dd-trace/test/appsec/streamtest.txt @@ -0,0 +1 @@ +file \ No newline at end of file diff --git a/packages/dd-trace/test/appsec/telemetry.spec.js b/packages/dd-trace/test/appsec/telemetry.spec.js index 41b79893ea5..a297ede3280 100644 --- a/packages/dd-trace/test/appsec/telemetry.spec.js +++ b/packages/dd-trace/test/appsec/telemetry.spec.js @@ -142,6 +142,89 @@ describe('Appsec Telemetry metrics', () => { expect(track).to.have.been.calledTwice }) + + it('should sum waf.duration metrics', () => { + appsecTelemetry.updateWafRequestsMetricTags({ + duration: 42, + durationExt: 52 + }, req) + + appsecTelemetry.updateWafRequestsMetricTags({ + duration: 24, + durationExt: 25 + }, req) + + const { duration, durationExt } = appsecTelemetry.getRequestMetrics(req) + + expect(duration).to.be.eq(66) + expect(durationExt).to.be.eq(77) + }) + }) + + describe('updateRaspRequestsMetricTags', () => { + it('should increment rasp.rule.eval metric', () => { + appsecTelemetry.updateRaspRequestsMetricTags({ + duration: 42, + durationExt: 52 + }, req, 'rule-type') + + expect(count).to.have.been.calledWith('rasp.rule.eval') + expect(count).to.not.have.been.calledWith('rasp.timeout') + expect(count).to.not.have.been.calledWith('rasp.rule.match') + expect(inc).to.have.been.calledOnceWith(1) + }) + + it('should increment rasp.timeout metric if timeout', () => { + appsecTelemetry.updateRaspRequestsMetricTags({ + duration: 42, + durationExt: 52, + wafTimeout: true + }, req, 'rule-type') + + expect(count).to.have.been.calledWith('rasp.rule.eval') + expect(count).to.have.been.calledWith('rasp.timeout') + expect(count).to.not.have.been.calledWith('rasp.rule.match') + expect(inc).to.have.been.calledTwice + }) + + it('should increment rasp.rule.match metric if ruleTriggered', () => { + appsecTelemetry.updateRaspRequestsMetricTags({ + duration: 42, + durationExt: 52, + ruleTriggered: true + }, req, 'rule-type') + + expect(count).to.have.been.calledWith('rasp.rule.match') + expect(count).to.have.been.calledWith('rasp.rule.eval') + expect(count).to.not.have.been.calledWith('rasp.timeout') + expect(inc).to.have.been.calledTwice + }) + + it('should sum rasp.duration and eval metrics', () => { + appsecTelemetry.updateRaspRequestsMetricTags({ + duration: 42, + durationExt: 52 + }, req, 'rule-type') + + appsecTelemetry.updateRaspRequestsMetricTags({ + duration: 24, + durationExt: 25 + }, req, 'rule-type') + + const { + duration, + durationExt, + raspDuration, + raspDurationExt, + raspEvalCount + } = appsecTelemetry.getRequestMetrics(req) + + expect(duration).to.be.eq(0) + expect(durationExt).to.be.eq(0) + expect(raspDuration).to.be.eq(66) + expect(raspDurationExt).to.be.eq(77) + expect(raspEvalCount).to.be.eq(2) + }) }) describe('incWafInitMetric', () => { @@ -221,6 +304,35 @@ describe('Appsec Telemetry metrics', () => { }) }) + it('should not modify waf.requests metric tags when rasp rule type is provided', () => { + appsecTelemetry.updateWafRequestsMetricTags({ + blockTriggered: false, + ruleTriggered: false, + wafTimeout: false, + wafVersion, + rulesVersion + }, req) + + appsecTelemetry.updateRaspRequestsMetricTags({ + blockTriggered: true, + ruleTriggered: true, + wafTimeout: true, + wafVersion, + rulesVersion + }, req, 'rule_type') + + expect(count).to.have.not.been.calledWith('waf.requests') + appsecTelemetry.incrementWafRequestsMetric(req) + + expect(count).to.have.been.calledWithExactly('waf.requests', { + request_blocked: false, + rule_triggered: false, + waf_timeout: false, + waf_version: wafVersion, + event_rules_version: rulesVersion + }) + }) + it('should not fail if req has no previous tag', () => { appsecTelemetry.incrementWafRequestsMetric(req) @@ -253,5 +365,114 @@ describe('Appsec Telemetry metrics', () => { expect(count).to.not.have.been.called expect(inc).to.not.have.been.called }) + + describe('updateWafRequestMetricTags', () => { + it('should sum waf.duration and waf.durationExt request metrics', () => { + appsecTelemetry.enable({ + enabled: false, + metrics: true + }) + + appsecTelemetry.updateWafRequestsMetricTags({ + duration: 42, + durationExt: 52 + }, req) + + appsecTelemetry.updateWafRequestsMetricTags({ + duration: 24, + durationExt: 25 + }, req) + + const { duration, durationExt } = appsecTelemetry.getRequestMetrics(req) + + expect(duration).to.be.eq(66) + expect(durationExt).to.be.eq(77) + }) + + it('should sum waf.duration and waf.durationExt with telemetry enabled and metrics disabled', () => { + appsecTelemetry.enable({ + enabled: true, + metrics: false + }) + + appsecTelemetry.updateWafRequestsMetricTags({ + duration: 42, + durationExt: 52 + }, req) + + appsecTelemetry.updateWafRequestsMetricTags({ + duration: 24, + durationExt: 25 + }, req) + + const { duration, durationExt } = appsecTelemetry.getRequestMetrics(req) + + expect(duration).to.be.eq(66) + expect(durationExt).to.be.eq(77) + }) + }) + + describe('updateRaspRequestsMetricTags', () => { + it('should sum rasp.duration and rasp.durationExt request metrics', () => { + appsecTelemetry.enable({ + enabled: false, + metrics: true + }) + + appsecTelemetry.updateRaspRequestsMetricTags({ + duration: 42, + durationExt: 52 + }, req, 'rasp_rule') + + appsecTelemetry.updateRaspRequestsMetricTags({ + duration: 24, + durationExt: 25 + }, req, 'rasp_rule') + + const { raspDuration, raspDurationExt, raspEvalCount } = appsecTelemetry.getRequestMetrics(req) + + expect(raspDuration).to.be.eq(66) + expect(raspDurationExt).to.be.eq(77) + expect(raspEvalCount).to.be.eq(2) + }) + + it('should sum rasp.duration and rasp.durationExt with telemetry enabled and metrics disabled', () => { + appsecTelemetry.enable({ + enabled: true, + metrics: false + }) + + appsecTelemetry.updateRaspRequestsMetricTags({ + duration: 42, + durationExt: 52 + }, req, 'rule_type') + + appsecTelemetry.updateRaspRequestsMetricTags({ + duration: 24, + durationExt: 25 + }, req, 'rule_type') + + const { raspDuration, raspDurationExt, raspEvalCount } = appsecTelemetry.getRequestMetrics(req) + + expect(raspDuration).to.be.eq(66) + expect(raspDurationExt).to.be.eq(77) + expect(raspEvalCount).to.be.eq(2) + }) + + it('should not increment any metric if telemetry metrics are disabled', () => { + appsecTelemetry.enable({ + enabled: true, + metrics: false + }) + + appsecTelemetry.updateRaspRequestsMetricTags({ + duration: 24, + durationExt: 25 + }, req, 'rule_type') + + expect(count).to.not.have.been.called + expect(inc).to.not.have.been.called + }) + }) }) }) diff --git a/packages/dd-trace/test/appsec/waf/index.spec.js b/packages/dd-trace/test/appsec/waf/index.spec.js index 0c01a8ad788..aff0a7e37a0 100644 --- a/packages/dd-trace/test/appsec/waf/index.spec.js +++ b/packages/dd-trace/test/appsec/waf/index.spec.js @@ -7,6 +7,14 @@ const Reporter = require('../../../src/appsec/reporter') const web = require('../../../src/plugins/util/web') describe('WAF Manager', () => { + const knownAddresses = new Set([ + 'server.io.net.url', + 'server.request.headers.no_cookies', + 'server.request.uri.raw', + 'processor.address', + 'server.request.body', + 'waf.context.processor' + ]) let waf, WAFManager let DDWAF let config @@ -19,13 +27,18 @@ describe('WAF Manager', () => { DDWAF.prototype.constructor.version = sinon.stub() DDWAF.prototype.dispose = sinon.stub() DDWAF.prototype.createContext = sinon.stub() - DDWAF.prototype.update = sinon.stub() + DDWAF.prototype.update = sinon.stub().callsFake(function (newRules) { + if (newRules?.metadata?.rules_version) { + this.diagnostics.ruleset_version = newRules?.metadata?.rules_version + } + }) DDWAF.prototype.diagnostics = { ruleset_version: '1.0.0', rules: { loaded: ['rule_1'], failed: [] } } + DDWAF.prototype.knownAddresses = knownAddresses WAFManager = proxyquire('../../../src/appsec/waf/waf_manager', { '@datadog/native-appsec': { DDWAF } @@ -39,11 +52,12 @@ describe('WAF Manager', () => { sinon.stub(Reporter, 'reportMetrics') sinon.stub(Reporter, 'reportAttack') sinon.stub(Reporter, 'reportWafUpdate') - sinon.stub(Reporter, 'reportSchemas') + sinon.stub(Reporter, 'reportDerivatives') webContext = {} sinon.stub(web, 'getContext').returns(webContext) }) + afterEach(() => { sinon.restore() waf.destroy() @@ -77,8 +91,8 @@ describe('WAF Manager', () => { loaded: ['rule_1'], failed: ['rule_2', 'rule_3'], errors: { - 'error_1': ['invalid_1'], - 'error_2': ['invalid_2', 'invalid_3'] + error_1: ['invalid_1'], + error_2: ['invalid_2', 'invalid_3'] } } } @@ -94,6 +108,32 @@ describe('WAF Manager', () => { }) }) + describe('run', () => { + it('should call wafManager.run with raspRuleType', () => { + const run = sinon.stub() + WAFManager.prototype.getWAFContext = sinon.stub().returns({ run }) + waf.init(rules, config.appsec) + + const payload = { persistent: { 'server.io.net.url': 'http://example.com' } } + const req = {} + waf.run(payload, req, 'ssrf') + + expect(run).to.be.calledOnceWithExactly(payload, 'ssrf') + }) + + it('should call wafManager.run without raspRuleType', () => { + const run = sinon.stub() + WAFManager.prototype.getWAFContext = sinon.stub().returns({ run }) + waf.init(rules, config.appsec) + + const payload = { persistent: { 'server.io.net.url': 'http://example.com' } } + const req = {} + waf.run(payload, req) + + expect(run).to.be.calledOnceWithExactly(payload, undefined) + }) + }) + describe('wafManager.createDDWAFContext', () => { beforeEach(() => { DDWAF.prototype.constructor.version.returns('4.5.6') @@ -124,7 +164,7 @@ describe('WAF Manager', () => { it('should call ddwaf.update', () => { const rules = { - 'rules_data': [ + rules_data: [ { id: 'blocked_users', type: 'data_with_expiration', @@ -141,11 +181,15 @@ describe('WAF Manager', () => { waf.update(rules) expect(DDWAF.prototype.update).to.be.calledOnceWithExactly(rules) + expect(Reporter.reportWafUpdate).to.be.calledOnceWithExactly(wafVersion, '1.0.0') }) it('should call Reporter.reportWafUpdate', () => { const rules = { - 'rules_data': [ + metadata: { + rules_version: '4.2.0' + }, + rules_data: [ { id: 'blocked_users', type: 'data_with_expiration', @@ -161,8 +205,7 @@ describe('WAF Manager', () => { waf.update(rules) - expect(Reporter.reportWafUpdate).to.be.calledOnceWithExactly(wafVersion, - DDWAF.prototype.diagnostics.ruleset_version) + expect(Reporter.reportWafUpdate).to.be.calledOnceWithExactly(wafVersion, '4.2.0') }) }) @@ -174,7 +217,7 @@ describe('WAF Manager', () => { url: '/path', headers: { 'user-agent': 'Arachni', - 'host': 'localhost', + host: 'localhost', cookie: 'a=1;b=2' }, method: 'POST', @@ -214,15 +257,19 @@ describe('WAF Manager', () => { ddwafContext.run.returns({ totalRuntime: 1, durationExt: 1 }) wafContextWrapper.run({ - 'server.request.headers.no_cookies': { 'header': 'value' }, - 'server.request.uri.raw': 'https://testurl', - 'processor.address': { 'extract-schema': true } + persistent: { + 'server.request.headers.no_cookies': { header: 'value' }, + 'server.request.uri.raw': 'https://testurl', + 'processor.address': { 'extract-schema': true } + } }) expect(ddwafContext.run).to.be.calledOnceWithExactly({ - 'server.request.headers.no_cookies': { 'header': 'value' }, - 'server.request.uri.raw': 'https://testurl', - 'processor.address': { 'extract-schema': true } + persistent: { + 'server.request.headers.no_cookies': { header: 'value' }, + 'server.request.uri.raw': 'https://testurl', + 'processor.address': { 'extract-schema': true } + } }, config.appsec.wafTimeout) }) @@ -235,7 +282,9 @@ describe('WAF Manager', () => { ddwafContext.run.returns(result) const params = { - 'server.request.headers.no_cookies': { 'header': 'value' } + persistent: { + 'server.request.headers.no_cookies': { header: 'value' } + } } wafContextWrapper.run(params) @@ -252,7 +301,9 @@ describe('WAF Manager', () => { ddwafContext.run.returns(result) const params = { - 'server.request.headers.no_cookies': { 'header': 'value' } + persistent: { + 'server.request.headers.no_cookies': { header: 'value' } + } } wafContextWrapper.run(params) @@ -263,10 +314,50 @@ describe('WAF Manager', () => { expect(reportMetricsArg.ruleTriggered).to.be.true }) + it('should report raspRuleType', () => { + const result = { + totalRuntime: 1, + durationExt: 1 + } + + ddwafContext.run.returns(result) + const params = { + persistent: { + 'server.request.headers.no_cookies': { header: 'value' } + } + } + + wafContextWrapper.run(params, 'rule_type') + + expect(Reporter.reportMetrics).to.be.calledOnce + expect(Reporter.reportMetrics.firstCall.args[1]).to.be.equal('rule_type') + }) + + it('should not report raspRuleType when it is not provided', () => { + const result = { + totalRuntime: 1, + durationExt: 1 + } + + ddwafContext.run.returns(result) + const params = { + persistent: { + 'server.request.headers.no_cookies': { header: 'value' } + } + } + + wafContextWrapper.run(params) + + expect(Reporter.reportMetrics).to.be.calledOnce + expect(Reporter.reportMetrics.firstCall.args[1]).to.be.equal(undefined) + }) + it('should not report attack when ddwafContext does not return events', () => { ddwafContext.run.returns({ totalRuntime: 1, durationExt: 1 }) const params = { - 'server.request.headers.no_cookies': { 'header': 'value' } + persistent: { + 'server.request.headers.no_cookies': { header: 'value' } + } } wafContextWrapper.run(params) @@ -277,7 +368,9 @@ describe('WAF Manager', () => { it('should not report attack when ddwafContext returns empty data', () => { ddwafContext.run.returns({ totalRuntime: 1, durationExt: 1, events: [] }) const params = { - 'server.request.headers.no_cookies': { 'header': 'value' } + persistent: { + 'server.request.headers.no_cookies': { header: 'value' } + } } wafContextWrapper.run(params) @@ -287,10 +380,12 @@ describe('WAF Manager', () => { it('should return the actions', () => { const actions = ['block'] - ddwafContext.run.returns({ totalRuntime: 1, durationExt: 1, events: [], actions: actions }) + ddwafContext.run.returns({ totalRuntime: 1, durationExt: 1, events: [], actions }) const params = { - 'server.request.headers.no_cookies': { 'header': 'value' } + persistent: { + 'server.request.headers.no_cookies': { header: 'value' } + } } const result = wafContextWrapper.run(params) @@ -305,16 +400,40 @@ describe('WAF Manager', () => { derivatives: [{ '_dd.appsec.s.req.body': [8] }] } const params = { - 'server.request.body': 'value', - 'waf.context.processor': { - 'extract-schema': true + persistent: { + 'server.request.body': 'value', + 'waf.context.processor': { + 'extract-schema': true + } } } ddwafContext.run.returns(result) wafContextWrapper.run(params) - expect(Reporter.reportSchemas).to.be.calledOnceWithExactly(result.derivatives) + expect(Reporter.reportDerivatives).to.be.calledOnceWithExactly(result.derivatives) + }) + + it('should report fingerprints when ddwafContext returns fingerprints in results derivatives', () => { + const result = { + totalRuntime: 1, + durationExt: 1, + derivatives: { + '_dd.appsec.s.req.body': [8], + '_dd.appsec.fp.http.endpoint': 'http-post-abcdefgh-12345678-abcdefgh', + '_dd.appsec.fp.http.network': 'net-1-0100000000', + '_dd.appsec.fp.http.headers': 'hdr-0110000110-abcdefgh-5-12345678' + } + } + + ddwafContext.run.returns(result) + + wafContextWrapper.run({ + persistent: { + 'server.request.body': 'foo' + } + }) + sinon.assert.calledOnceWithExactly(Reporter.reportDerivatives, result.derivatives) }) }) }) diff --git a/packages/dd-trace/test/appsec/waf/waf_context_wrapper.spec.js b/packages/dd-trace/test/appsec/waf/waf_context_wrapper.spec.js index 23df1adfdc5..cffe9718ee2 100644 --- a/packages/dd-trace/test/appsec/waf/waf_context_wrapper.spec.js +++ b/packages/dd-trace/test/appsec/waf/waf_context_wrapper.spec.js @@ -1,17 +1,26 @@ 'use strict' +const proxyquire = require('proxyquire') const WAFContextWrapper = require('../../../src/appsec/waf/waf_context_wrapper') const addresses = require('../../../src/appsec/addresses') +const { wafRunFinished } = require('../../../src/appsec/channels') describe('WAFContextWrapper', () => { + const knownAddresses = new Set([ + addresses.HTTP_INCOMING_QUERY, + addresses.HTTP_INCOMING_GRAPHQL_RESOLVER + ]) + it('Should send HTTP_INCOMING_QUERY only once', () => { const ddwafContext = { run: sinon.stub() } - const wafContextWrapper = new WAFContextWrapper(ddwafContext, 1000, '1.14.0', '1.8.0') + const wafContextWrapper = new WAFContextWrapper(ddwafContext, 1000, '1.14.0', '1.8.0', knownAddresses) const payload = { - [addresses.HTTP_INCOMING_QUERY]: { key: 'value' } + persistent: { + [addresses.HTTP_INCOMING_QUERY]: { key: 'value' } + } } wafContextWrapper.run(payload) @@ -19,4 +28,130 @@ describe('WAFContextWrapper', () => { expect(ddwafContext.run).to.have.been.calledOnceWithExactly(payload, 1000) }) + + it('Should send ephemeral addresses every time', () => { + const ddwafContext = { + run: sinon.stub() + } + const wafContextWrapper = new WAFContextWrapper(ddwafContext, 1000, '1.14.0', '1.8.0', knownAddresses) + + const payload = { + persistent: { + [addresses.HTTP_INCOMING_QUERY]: { key: 'value' } + }, + ephemeral: { + [addresses.HTTP_INCOMING_GRAPHQL_RESOLVER]: { anotherKey: 'anotherValue' } + } + } + + wafContextWrapper.run(payload) + wafContextWrapper.run(payload) + + expect(ddwafContext.run).to.have.been.calledTwice + expect(ddwafContext.run.firstCall).to.have.been.calledWithExactly(payload, 1000) + expect(ddwafContext.run.secondCall).to.have.been.calledWithExactly({ + ephemeral: { + [addresses.HTTP_INCOMING_GRAPHQL_RESOLVER]: { + anotherKey: 'anotherValue' + } + } + }, 1000) + }) + + it('Should ignore run without known addresses', () => { + const ddwafContext = { + run: sinon.stub() + } + const wafContextWrapper = new WAFContextWrapper(ddwafContext, 1000, '1.14.0', '1.8.0', knownAddresses) + + const payload = { + persistent: { + 'persistent-unknown-address': { key: 'value' } + }, + ephemeral: { + 'ephemeral-unknown-address': { key: 'value' } + } + } + + wafContextWrapper.run(payload) + + expect(ddwafContext.run).to.have.not.been.called + }) + + it('should publish the payload in the dc channel', () => { + const ddwafContext = { + run: sinon.stub().returns([]) + } + const wafContextWrapper = new WAFContextWrapper(ddwafContext, 1000, '1.14.0', '1.8.0', knownAddresses) + const payload = { + persistent: { + [addresses.HTTP_INCOMING_QUERY]: { key: 'value' } + }, + ephemeral: { + [addresses.HTTP_INCOMING_GRAPHQL_RESOLVER]: { anotherKey: 'anotherValue' } + } + } + const finishedCallback = sinon.stub() + + wafRunFinished.subscribe(finishedCallback) + wafContextWrapper.run(payload) + wafRunFinished.unsubscribe(finishedCallback) + + expect(finishedCallback).to.be.calledOnceWith({ payload }) + }) + + describe('Disposal context check', () => { + let log + let ddwafContext + let wafContextWrapper + + beforeEach(() => { + log = { + warn: sinon.stub() + } + + ddwafContext = { + run: sinon.stub() + } + + const ProxiedWafContextWrapper = proxyquire('../../../src/appsec/waf/waf_context_wrapper', { + '../../log': log + }) + + wafContextWrapper = new ProxiedWafContextWrapper(ddwafContext, 1000, '1.14.0', '1.8.0', knownAddresses) + }) + + afterEach(() => { + sinon.restore() + }) + + it('Should call run if context is not disposed', () => { + ddwafContext.disposed = false + + const payload = { + persistent: { + [addresses.HTTP_INCOMING_QUERY]: { key: 'value' } + } + } + + wafContextWrapper.run(payload) + + sinon.assert.calledOnce(ddwafContext.run) + }) + + it('Should not call run and log a warn if context is disposed', () => { + ddwafContext.disposed = true + + const payload = { + persistent: { + [addresses.HTTP_INCOMING_QUERY]: { key: 'value' } + } + } + + wafContextWrapper.run(payload) + + sinon.assert.notCalled(ddwafContext.run) + sinon.assert.calledOnceWithExactly(log.warn, 'Calling run on a disposed context') + }) + }) }) diff --git a/packages/dd-trace/test/appsec/waf/waf_manager.spec.js b/packages/dd-trace/test/appsec/waf/waf_manager.spec.js new file mode 100644 index 00000000000..ebb7d371049 --- /dev/null +++ b/packages/dd-trace/test/appsec/waf/waf_manager.spec.js @@ -0,0 +1,33 @@ +'use strict' + +const proxyquire = require('proxyquire') + +describe('WAFManager', () => { + let WAFManager, WAFContextWrapper, DDWAF + const knownAddresses = new Set() + + beforeEach(() => { + DDWAF = sinon.stub() + DDWAF.prototype.constructor.version = sinon.stub() + DDWAF.prototype.knownAddresses = knownAddresses + DDWAF.prototype.diagnostics = {} + DDWAF.prototype.createContext = sinon.stub() + + WAFContextWrapper = sinon.stub() + WAFManager = proxyquire('../../../src/appsec/waf/waf_manager', { + './waf_context_wrapper': WAFContextWrapper, + '@datadog/native-appsec': { DDWAF } + }) + }) + + describe('getWAFContext', () => { + it('should construct WAFContextWrapper with knownAddresses', () => { + const wafManager = new WAFManager({}, {}) + + wafManager.getWAFContext({}) + + const any = sinon.match.any + sinon.assert.calledOnceWithMatch(WAFContextWrapper, any, any, any, any, knownAddresses) + }) + }) +}) diff --git a/packages/dd-trace/test/azure_metadata.spec.js b/packages/dd-trace/test/azure_metadata.spec.js new file mode 100644 index 00000000000..7a8cb787d75 --- /dev/null +++ b/packages/dd-trace/test/azure_metadata.spec.js @@ -0,0 +1,109 @@ +'use strict' + +require('./setup/tap') + +const os = require('os') +const { getAzureAppMetadata, getAzureTagsFromMetadata } = require('../src/azure_metadata') + +describe('Azure metadata', () => { + describe('for apps is', () => { + it('not provided without DD_AZURE_APP_SERVICES', () => { + delete process.env.DD_AZURE_APP_SERVICES + expect(getAzureAppMetadata()).to.be.undefined + }) + + it('provided with DD_AZURE_APP_SERVICES', () => { + delete process.env.COMPUTERNAME // actually defined on Windows + process.env.DD_AZURE_APP_SERVICES = '1' + delete process.env.WEBSITE_SITE_NAME + expect(getAzureAppMetadata()).to.deep.equal({ operatingSystem: os.platform(), siteKind: 'app', siteType: 'app' }) + }) + }) + + it('provided completely with minimum vars', () => { + delete process.env.WEBSITE_RESOURCE_GROUP + delete process.env.WEBSITE_OS + delete process.env.FUNCTIONS_EXTENSION_VERSION + delete process.env.FUNCTIONS_WORKER_RUNTIME + delete process.env.FUNCTIONS_WORKER_RUNTIME_VERSION + process.env.COMPUTERNAME = 'boaty_mcboatface' + process.env.DD_AZURE_APP_SERVICES = '1' + process.env.WEBSITE_SITE_NAME = 'website_name' + process.env.WEBSITE_OWNER_NAME = 'subscription_id+resource_group-regionwebspace' + process.env.WEBSITE_INSTANCE_ID = 'instance_id' + process.env.DD_AAS_DOTNET_EXTENSION_VERSION = '1.0' + const expected = { + extensionVersion: '1.0', + instanceID: 'instance_id', + instanceName: 'boaty_mcboatface', + operatingSystem: os.platform(), + resourceGroup: 'resource_group', + resourceID: + '/subscriptions/subscription_id/resourcegroups/resource_group/providers/microsoft.web/sites/website_name', + siteKind: 'app', + siteName: 'website_name', + siteType: 'app', + subscriptionID: 'subscription_id' + } + expect(getAzureAppMetadata()).to.deep.equal(expected) + }) + + it('provided completely with complete vars', () => { + process.env.COMPUTERNAME = 'boaty_mcboatface' + process.env.DD_AZURE_APP_SERVICES = '1' + process.env.WEBSITE_SITE_NAME = 'website_name' + process.env.WEBSITE_RESOURCE_GROUP = 'resource_group' + process.env.WEBSITE_OWNER_NAME = 'subscription_id+foo-regionwebspace' + process.env.WEBSITE_OS = 'windows' + process.env.WEBSITE_INSTANCE_ID = 'instance_id' + process.env.FUNCTIONS_EXTENSION_VERSION = '20' + process.env.FUNCTIONS_WORKER_RUNTIME = 'node' + process.env.FUNCTIONS_WORKER_RUNTIME_VERSION = '14' + process.env.DD_AAS_DOTNET_EXTENSION_VERSION = '1.0' + const expected = { + extensionVersion: '1.0', + functionRuntimeVersion: '20', + instanceID: 'instance_id', + instanceName: 'boaty_mcboatface', + operatingSystem: 'windows', + resourceGroup: 'resource_group', + resourceID: + '/subscriptions/subscription_id/resourcegroups/resource_group/providers/microsoft.web/sites/website_name', + runtime: 'node', + runtimeVersion: '14', + siteKind: 'functionapp', + siteName: 'website_name', + siteType: 'function', + subscriptionID: 'subscription_id' + } + expect(getAzureAppMetadata()).to.deep.equal(expected) + }) + + it('tags are correctly generated from vars', () => { + delete process.env.WEBSITE_RESOURCE_GROUP + delete process.env.WEBSITE_OS + delete process.env.FUNCTIONS_EXTENSION_VERSION + delete process.env.FUNCTIONS_WORKER_RUNTIME + delete process.env.FUNCTIONS_WORKER_RUNTIME_VERSION + process.env.COMPUTERNAME = 'boaty_mcboatface' + process.env.DD_AZURE_APP_SERVICES = '1' + process.env.WEBSITE_SITE_NAME = 'website_name' + process.env.WEBSITE_OWNER_NAME = 'subscription_id+resource_group-regionwebspace' + process.env.WEBSITE_INSTANCE_ID = 'instance_id' + process.env.DD_AAS_DOTNET_EXTENSION_VERSION = '1.0' + const expected = { + 'aas.environment.extension_version': '1.0', + 'aas.environment.instance_id': 'instance_id', + 'aas.environment.instance_name': 'boaty_mcboatface', + 'aas.environment.os': os.platform(), + 'aas.resource.group': 'resource_group', + 'aas.resource.id': + '/subscriptions/subscription_id/resourcegroups/resource_group/providers/microsoft.web/sites/website_name', + 'aas.site.kind': 'app', + 'aas.site.name': 'website_name', + 'aas.site.type': 'app', + 'aas.subscription.id': 'subscription_id' + } + expect(getAzureTagsFromMetadata(getAzureAppMetadata())).to.deep.equal(expected) + }) +}) diff --git a/packages/dd-trace/test/ci-visibility/encode/json-encoder.spec.js b/packages/dd-trace/test/ci-visibility/encode/json-encoder.spec.js index 5a5ddb8288f..1f052d299f1 100644 --- a/packages/dd-trace/test/ci-visibility/encode/json-encoder.spec.js +++ b/packages/dd-trace/test/ci-visibility/encode/json-encoder.spec.js @@ -6,14 +6,17 @@ const { JSONEncoder } = require('../../../src/ci-visibility/encode/json-encoder' describe('CI Visibility JSON encoder', () => { let send, originalSend + beforeEach(() => { send = sinon.spy() originalSend = process.send process.send = send }) + afterEach(() => { process.send = originalSend }) + it('can JSON serialize payloads', () => { const payload = [{ type: 'test' }, { type: 'test', name: 'test2' }] const payloadSecond = { type: 'test', name: 'other' } diff --git a/packages/dd-trace/test/ci-visibility/exporters/agent-proxy/agent-proxy.spec.js b/packages/dd-trace/test/ci-visibility/exporters/agent-proxy/agent-proxy.spec.js index 404852ec633..4ff8f12ace6 100644 --- a/packages/dd-trace/test/ci-visibility/exporters/agent-proxy/agent-proxy.spec.js +++ b/packages/dd-trace/test/ci-visibility/exporters/agent-proxy/agent-proxy.spec.js @@ -71,6 +71,7 @@ describe('AgentProxyCiVisibilityExporter', () => { endpoints: ['/evp_proxy/v2/'] })) }) + it('should initialise AgentlessWriter and CoverageWriter', async () => { const agentProxyCiVisibilityExporter = new AgentProxyCiVisibilityExporter({ port, tags }) await agentProxyCiVisibilityExporter._canUseCiVisProtocolPromise @@ -93,6 +94,7 @@ describe('AgentProxyCiVisibilityExporter', () => { expect(mockWriter.append).to.have.been.calledWith(testSuiteTrace) expect(mockWriter.append).to.have.been.calledWith(testSessionTrace) }) + it('should process coverages', async () => { const mockWriter = { append: sinon.spy(), @@ -106,7 +108,7 @@ describe('AgentProxyCiVisibilityExporter', () => { spanId: '1', files: [] } - agentProxyCiVisibilityExporter._itrConfig = { isCodeCoverageEnabled: true } + agentProxyCiVisibilityExporter._libraryConfig = { isCodeCoverageEnabled: true } agentProxyCiVisibilityExporter.exportCoverage(coverage) expect(mockWriter.append).to.have.been.calledWith({ spanId: '1', traceId: '1', files: [] }) }) @@ -121,12 +123,14 @@ describe('AgentProxyCiVisibilityExporter', () => { endpoints: ['/v0.4/traces'] })) }) + it('should initialise AgentWriter', async () => { const agentProxyCiVisibilityExporter = new AgentProxyCiVisibilityExporter({ port, tags }) await agentProxyCiVisibilityExporter._canUseCiVisProtocolPromise expect(agentProxyCiVisibilityExporter._writer).to.be.instanceOf(AgentWriter) expect(agentProxyCiVisibilityExporter._coverageWriter).to.be.undefined }) + it('should not process test suite level visibility spans', async () => { const mockWriter = { append: sinon.spy(), @@ -213,7 +217,7 @@ describe('AgentProxyCiVisibilityExporter', () => { spanId: '1', files: [] } - agentProxyCiVisibilityExporter._itrConfig = { isCodeCoverageEnabled: true } + agentProxyCiVisibilityExporter._libraryConfig = { isCodeCoverageEnabled: true } agentProxyCiVisibilityExporter.exportCoverage(coverage) expect(mockWriter.append).to.have.been.calledWith({ traceId: '1', spanId: '1', files: [] }) await new Promise(resolve => setTimeout(resolve, flushInterval)) @@ -250,4 +254,76 @@ describe('AgentProxyCiVisibilityExporter', () => { expect(mockCoverageWriter.setUrl).to.have.been.calledWith(coverageUrl) }) }) + + describe('_isGzipCompatible', () => { + it('should set _isGzipCompatible to true if the newest version is v4 or newer', async () => { + const scope = nock('http://localhost:8126') + .get('/info') + .reply(200, JSON.stringify({ + endpoints: ['/evp_proxy/v2', '/evp_proxy/v3', '/evp_proxy/v4/', '/evp_proxy/v5'] + })) + + const agentProxyCiVisibilityExporter = new AgentProxyCiVisibilityExporter({ port, tags }) + + expect(agentProxyCiVisibilityExporter).not.to.be.null + + await agentProxyCiVisibilityExporter._canUseCiVisProtocolPromise + + expect(agentProxyCiVisibilityExporter._isGzipCompatible).to.be.true + expect(scope.isDone()).to.be.true + }) + + it('should set _isGzipCompatible to false if the newest version is v3 or older', async () => { + const scope = nock('http://localhost:8126') + .get('/info') + .reply(200, JSON.stringify({ + endpoints: ['/evp_proxy/v2', '/evp_proxy/v3'] + })) + + const agentProxyCiVisibilityExporter = new AgentProxyCiVisibilityExporter({ port, tags }) + + expect(agentProxyCiVisibilityExporter).not.to.be.null + + await agentProxyCiVisibilityExporter._canUseCiVisProtocolPromise + + expect(agentProxyCiVisibilityExporter._isGzipCompatible).to.be.false + expect(scope.isDone()).to.be.true + }) + }) + + describe('evpProxyPrefix', () => { + it('should set evpProxyPrefix to v2 if the newest version is v3', async () => { + const scope = nock('http://localhost:8126') + .get('/info') + .reply(200, JSON.stringify({ + endpoints: ['/evp_proxy/v2', '/evp_proxy/v3'] + })) + + const agentProxyCiVisibilityExporter = new AgentProxyCiVisibilityExporter({ port, tags }) + + expect(agentProxyCiVisibilityExporter).not.to.be.null + + await agentProxyCiVisibilityExporter._canUseCiVisProtocolPromise + + expect(agentProxyCiVisibilityExporter.evpProxyPrefix).to.equal('/evp_proxy/v2') + expect(scope.isDone()).to.be.true + }) + + it('should set evpProxyPrefix to v4 if the newest version is v4', async () => { + const scope = nock('http://localhost:8126') + .get('/info') + .reply(200, JSON.stringify({ + endpoints: ['/evp_proxy/v2', '/evp_proxy/v3', '/evp_proxy/v4/'] + })) + + const agentProxyCiVisibilityExporter = new AgentProxyCiVisibilityExporter({ port, tags }) + + expect(agentProxyCiVisibilityExporter).not.to.be.null + + await agentProxyCiVisibilityExporter._canUseCiVisProtocolPromise + + expect(agentProxyCiVisibilityExporter.evpProxyPrefix).to.equal('/evp_proxy/v4') + expect(scope.isDone()).to.be.true + }) + }) }) diff --git a/packages/dd-trace/test/ci-visibility/exporters/agentless/coverage-writer.spec.js b/packages/dd-trace/test/ci-visibility/exporters/agentless/coverage-writer.spec.js index 7bfdd197866..62e10e9753e 100644 --- a/packages/dd-trace/test/ci-visibility/exporters/agentless/coverage-writer.spec.js +++ b/packages/dd-trace/test/ci-visibility/exporters/agentless/coverage-writer.spec.js @@ -23,7 +23,8 @@ describe('CI Visibility Coverage Writer', () => { count: sinon.stub().returns(0), makePayload: sinon.stub().returns({ getHeaders: () => ({}), - pipe: () => {} + pipe: () => {}, + size: () => 1 }) } @@ -80,7 +81,8 @@ describe('CI Visibility Coverage Writer', () => { encoder.count.returns(2) const payload = { getHeaders: () => ({}), - pipe: () => {} + pipe: () => {}, + size: () => 1 } encoder.makePayload.returns(payload) @@ -101,7 +103,8 @@ describe('CI Visibility Coverage Writer', () => { const payload = { getHeaders: () => ({}), - pipe: () => {} + pipe: () => {}, + size: () => 1 } encoder.count.returns(1) diff --git a/packages/dd-trace/test/ci-visibility/exporters/agentless/exporter.spec.js b/packages/dd-trace/test/ci-visibility/exporters/agentless/exporter.spec.js index 9c12087dbe0..11b3bf1ec4c 100644 --- a/packages/dd-trace/test/ci-visibility/exporters/agentless/exporter.spec.js +++ b/packages/dd-trace/test/ci-visibility/exporters/agentless/exporter.spec.js @@ -17,6 +17,7 @@ describe('CI Visibility Agentless Exporter', () => { sinon.stub(cp, 'execFileSync').returns('false') nock.cleanAll() }) + afterEach(() => { sinon.restore() }) @@ -53,14 +54,14 @@ describe('CI Visibility Agentless Exporter', () => { isIntelligentTestRunnerEnabled: true, tags: {} }) - expect(agentlessExporter.shouldRequestItrConfiguration()).to.be.true - agentlessExporter.getItrConfiguration({}, () => { + agentlessExporter.getLibraryConfiguration({}, () => { expect(scope.isDone()).to.be.true expect(agentlessExporter.canReportCodeCoverage()).to.be.true expect(agentlessExporter.shouldRequestSkippableSuites()).to.be.true done() }) }) + it('will request skippable to api.site by default', (done) => { const scope = nock('https://api.datadoge.c0m') .post('/api/v2/libraries/tests/services/setting') @@ -85,13 +86,14 @@ describe('CI Visibility Agentless Exporter', () => { tags: {} }) agentlessExporter._resolveGit() - agentlessExporter.getItrConfiguration({}, () => { + agentlessExporter.getLibraryConfiguration({}, () => { agentlessExporter.getSkippableSuites({}, () => { expect(scope.isDone()).to.be.true done() }) }) }) + it('can request ITR configuration right away', (done) => { const scope = nock('http://www.example.com') .post('/api/v2/libraries/tests/services/setting') @@ -107,14 +109,14 @@ describe('CI Visibility Agentless Exporter', () => { const agentlessExporter = new AgentlessCiVisibilityExporter({ url, isGitUploadEnabled: true, isIntelligentTestRunnerEnabled: true, tags: {} }) - expect(agentlessExporter.shouldRequestItrConfiguration()).to.be.true - agentlessExporter.getItrConfiguration({}, () => { + agentlessExporter.getLibraryConfiguration({}, () => { expect(scope.isDone()).to.be.true expect(agentlessExporter.canReportCodeCoverage()).to.be.true expect(agentlessExporter.shouldRequestSkippableSuites()).to.be.true done() }) }) + it('can report code coverages if enabled by the API', (done) => { const scope = nock('http://www.example.com') .post('/api/v2/libraries/tests/services/setting') @@ -130,12 +132,13 @@ describe('CI Visibility Agentless Exporter', () => { const agentlessExporter = new AgentlessCiVisibilityExporter({ url, isGitUploadEnabled: true, isIntelligentTestRunnerEnabled: true, tags: {} }) - agentlessExporter.getItrConfiguration({}, () => { + agentlessExporter.getLibraryConfiguration({}, () => { expect(scope.isDone()).to.be.true expect(agentlessExporter.canReportCodeCoverage()).to.be.true done() }) }) + it('will not allow skippable request if ITR configuration fails', (done) => { // request will fail delete process.env.DD_API_KEY @@ -162,8 +165,7 @@ describe('CI Visibility Agentless Exporter', () => { }) } - expect(agentlessExporter.shouldRequestItrConfiguration()).to.be.true - agentlessExporter.getItrConfiguration({}, (err) => { + agentlessExporter.getLibraryConfiguration({}, (err) => { expect(scope.isDone()).not.to.be.true expect(err.message).to.contain( 'Request to settings endpoint was not done because Datadog API key is not defined' diff --git a/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-exporter.spec.js b/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-exporter.spec.js index f3d331be567..b92d5b3ae98 100644 --- a/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-exporter.spec.js +++ b/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-exporter.spec.js @@ -3,6 +3,8 @@ require('../../../../dd-trace/test/setup/tap') const cp = require('child_process') +const fs = require('fs') +const zlib = require('zlib') const CiVisibilityExporter = require('../../../src/ci-visibility/exporters/ci-visibility-exporter') const nock = require('nock') @@ -13,6 +15,7 @@ describe('CI Visibility Exporter', () => { beforeEach(() => { // to make sure `isShallowRepository` in `git.js` returns false sinon.stub(cp, 'execFileSync').returns('false') + sinon.stub(fs, 'readFileSync').returns('') process.env.DD_API_KEY = '1' nock.cleanAll() }) @@ -42,6 +45,7 @@ describe('CI Visibility Exporter', () => { ciVisibilityExporter._resolveCanUseCiVisProtocol(true) ciVisibilityExporter.sendGitMetadata() }) + it('should resolve _gitUploadPromise with an error when git metadata request fails', (done) => { const scope = nock(`http://localhost:${port}`) .post('/api/v2/git/repository/search_commits') @@ -58,6 +62,7 @@ describe('CI Visibility Exporter', () => { ciVisibilityExporter._resolveCanUseCiVisProtocol(true) ciVisibilityExporter.sendGitMetadata() }) + it('should use the input repository URL', (done) => { nock(`http://localhost:${port}`) .post('/api/v2/git/repository/search_commits') @@ -77,8 +82,8 @@ describe('CI Visibility Exporter', () => { }) }) - describe('getItrConfiguration', () => { - it('should upload git metadata when getItrConfiguration is called, regardless of ITR config', (done) => { + describe('getLibraryConfiguration', () => { + it('should upload git metadata when getLibraryConfiguration is called, regardless of ITR config', (done) => { const scope = nock(`http://localhost:${port}`) .post('/api/v2/git/repository/search_commits') .reply(200, JSON.stringify({ @@ -88,20 +93,22 @@ describe('CI Visibility Exporter', () => { .reply(202, '') const ciVisibilityExporter = new CiVisibilityExporter({ port, isGitUploadEnabled: true }) - ciVisibilityExporter.getItrConfiguration({}, () => { - expect(scope.isDone()).not.to.be.true + ciVisibilityExporter._resolveCanUseCiVisProtocol(true) + ciVisibilityExporter.getLibraryConfiguration({}, () => {}) + ciVisibilityExporter._gitUploadPromise.then(() => { + expect(scope.isDone()).to.be.true done() }) }) - context('if ITR is not enabled', () => { - it('should resolve immediately if ITR is not enabled', (done) => { + context('if ITR is disabled', () => { + it('should resolve immediately and not request settings', (done) => { const scope = nock(`http://localhost:${port}`) .post('/api/v2/libraries/tests/services/setting') .reply(200) const ciVisibilityExporter = new CiVisibilityExporter({ port }) - ciVisibilityExporter.getItrConfiguration({}, (err, itrConfig) => { - expect(itrConfig).to.eql({}) + ciVisibilityExporter.getLibraryConfiguration({}, (err, libraryConfig) => { + expect(libraryConfig).to.eql({}) expect(err).to.be.null expect(scope.isDone()).not.to.be.true done() @@ -135,10 +142,10 @@ describe('CI Visibility Exporter', () => { } }) - ciVisibilityExporter.getItrConfiguration({}, () => { + ciVisibilityExporter.getLibraryConfiguration({}, () => { expect(scope.isDone()).to.be.true expect(customConfig).to.eql({ - 'my_custom_config': 'my_custom_config_value' + my_custom_config: 'my_custom_config_value' }) done() }) @@ -160,12 +167,13 @@ describe('CI Visibility Exporter', () => { const ciVisibilityExporter = new CiVisibilityExporter({ port, isIntelligentTestRunnerEnabled: true }) - ciVisibilityExporter.getItrConfiguration({}, (err, itrConfig) => { - expect(itrConfig).to.eql({ + ciVisibilityExporter.getLibraryConfiguration({}, (err, libraryConfig) => { + expect(libraryConfig).to.contain({ requireGit: false, isCodeCoverageEnabled: true, isItrEnabled: true, - isSuitesSkippingEnabled: true + isSuitesSkippingEnabled: true, + isEarlyFlakeDetectionEnabled: false }) expect(err).not.to.exist expect(scope.isDone()).to.be.true @@ -190,7 +198,7 @@ describe('CI Visibility Exporter', () => { const ciVisibilityExporter = new CiVisibilityExporter({ port, isIntelligentTestRunnerEnabled: true }) expect(ciVisibilityExporter.shouldRequestSkippableSuites()).to.be.false - ciVisibilityExporter.getItrConfiguration({}, () => { + ciVisibilityExporter.getLibraryConfiguration({}, () => { expect(ciVisibilityExporter.shouldRequestSkippableSuites()).to.be.true done() }) @@ -225,12 +233,12 @@ describe('CI Visibility Exporter', () => { port, isIntelligentTestRunnerEnabled: true }) ciVisibilityExporter._resolveCanUseCiVisProtocol(true) - expect(ciVisibilityExporter.shouldRequestItrConfiguration()).to.be.true - ciVisibilityExporter.getItrConfiguration({}, (err, itrConfig) => { + expect(ciVisibilityExporter.shouldRequestLibraryConfiguration()).to.be.true + ciVisibilityExporter.getLibraryConfiguration({}, (err, libraryConfig) => { expect(scope.isDone()).to.be.true expect(err).to.be.null // the second request returns require_git: false - expect(itrConfig.requireGit).to.be.false + expect(libraryConfig.requireGit).to.be.false expect(hasUploadedGit).to.be.true done() }) @@ -267,12 +275,12 @@ describe('CI Visibility Exporter', () => { port, isIntelligentTestRunnerEnabled: true }) ciVisibilityExporter._resolveCanUseCiVisProtocol(true) - expect(ciVisibilityExporter.shouldRequestItrConfiguration()).to.be.true - ciVisibilityExporter.getItrConfiguration({}, (err, itrConfig) => { + expect(ciVisibilityExporter.shouldRequestLibraryConfiguration()).to.be.true + ciVisibilityExporter.getLibraryConfiguration({}, (err, libraryConfig) => { expect(scope.isDone()).to.be.true expect(err).to.be.null // the second request returns require_git: false - expect(itrConfig.requireGit).to.be.false + expect(libraryConfig.requireGit).to.be.false done() }) ciVisibilityExporter._resolveGit() @@ -350,13 +358,13 @@ describe('CI Visibility Exporter', () => { } }) - ciVisibilityExporter._itrConfig = { isSuitesSkippingEnabled: true } + ciVisibilityExporter._libraryConfig = { isSuitesSkippingEnabled: true } ciVisibilityExporter._resolveCanUseCiVisProtocol(true) ciVisibilityExporter.getSkippableSuites({}, () => { expect(scope.isDone()).to.be.true expect(customConfig).to.eql({ - 'my_custom_config_2': 'my_custom_config_value_2' + my_custom_config_2: 'my_custom_config_value_2' }) done() }) @@ -374,6 +382,9 @@ describe('CI Visibility Exporter', () => { const scope = nock(`http://localhost:${port}`) .post('/api/v2/ci/tests/skippable') .reply(200, JSON.stringify({ + meta: { + correlation_id: '1234' + }, data: [{ type: 'suite', attributes: { @@ -388,7 +399,7 @@ describe('CI Visibility Exporter', () => { isGitUploadEnabled: true }) - ciVisibilityExporter._itrConfig = { isSuitesSkippingEnabled: true } + ciVisibilityExporter._libraryConfig = { isSuitesSkippingEnabled: true } ciVisibilityExporter._resolveCanUseCiVisProtocol(true) ciVisibilityExporter.getSkippableSuites({}, (err, skippableSuites) => { @@ -408,7 +419,7 @@ describe('CI Visibility Exporter', () => { const ciVisibilityExporter = new CiVisibilityExporter({ port, isIntelligentTestRunnerEnabled: true }) - ciVisibilityExporter._itrConfig = { isSuitesSkippingEnabled: true } + ciVisibilityExporter._libraryConfig = { isSuitesSkippingEnabled: true } ciVisibilityExporter._resolveCanUseCiVisProtocol(true) ciVisibilityExporter.getSkippableSuites({}, (err, skippableSuites) => { @@ -420,6 +431,104 @@ describe('CI Visibility Exporter', () => { ciVisibilityExporter._resolveGit(new Error('could not upload git metadata')) }) }) + context('if ITR is enabled and the exporter can use gzip', () => { + it('should request the API with gzip', (done) => { + nock(`http://localhost:${port}`) + .post('/api/v2/git/repository/search_commits') + .reply(200, JSON.stringify({ + data: [] + })) + .post('/api/v2/git/repository/packfile') + .reply(202, '') + + let requestHeaders = {} + const scope = nock(`http://localhost:${port}`) + .post('/api/v2/ci/tests/skippable') + .reply(200, function () { + requestHeaders = this.req.headers + + return zlib.gzipSync( + JSON.stringify({ + meta: { + correlation_id: '1234' + }, + data: [{ + type: 'suite', + attributes: { + suite: 'ci-visibility/test/ci-visibility-test.js' + } + }] + }) + ) + }, { + 'content-encoding': 'gzip' + }) + const ciVisibilityExporter = new CiVisibilityExporter({ + port, + isIntelligentTestRunnerEnabled: true, + isGitUploadEnabled: true + }) + ciVisibilityExporter._libraryConfig = { isSuitesSkippingEnabled: true } + ciVisibilityExporter._resolveCanUseCiVisProtocol(true) + ciVisibilityExporter._isGzipCompatible = true + + ciVisibilityExporter.getSkippableSuites({}, (err, skippableSuites) => { + expect(err).to.be.null + expect(skippableSuites).to.eql(['ci-visibility/test/ci-visibility-test.js']) + expect(scope.isDone()).to.be.true + expect(requestHeaders['accept-encoding']).to.equal('gzip') + done() + }) + ciVisibilityExporter.sendGitMetadata() + }) + }) + context('if ITR is enabled and the exporter can not use gzip', () => { + it('should request the API without gzip', (done) => { + nock(`http://localhost:${port}`) + .post('/api/v2/git/repository/search_commits') + .reply(200, JSON.stringify({ + data: [] + })) + .post('/api/v2/git/repository/packfile') + .reply(202, '') + + let requestHeaders = {} + const scope = nock(`http://localhost:${port}`) + .post('/api/v2/ci/tests/skippable') + .reply(200, function () { + requestHeaders = this.req.headers + + return JSON.stringify({ + meta: { + correlation_id: '1234' + }, + data: [{ + type: 'suite', + attributes: { + suite: 'ci-visibility/test/ci-visibility-test.js' + } + }] + }) + }) + const ciVisibilityExporter = new CiVisibilityExporter({ + port, + isIntelligentTestRunnerEnabled: true, + isGitUploadEnabled: true + }) + ciVisibilityExporter._libraryConfig = { isSuitesSkippingEnabled: true } + ciVisibilityExporter._resolveCanUseCiVisProtocol(true) + ciVisibilityExporter._isGzipCompatible = false + + ciVisibilityExporter.getSkippableSuites({}, (err, skippableSuites) => { + expect(err).to.be.null + expect(skippableSuites).to.eql(['ci-visibility/test/ci-visibility-test.js']) + expect(scope.isDone()).to.be.true + expect(requestHeaders['accept-encoding']).not.to.equal('gzip') + done() + }) + ciVisibilityExporter.sendGitMetadata() + }) + }) }) describe('export', () => { @@ -538,4 +647,172 @@ describe('CI Visibility Exporter', () => { }) }) }) + + describe('getKnownTests', () => { + context('if early flake detection is disabled', () => { + it('should resolve immediately to undefined', (done) => { + const scope = nock(`http://localhost:${port}`) + .post('/api/v2/ci/libraries/tests') + .reply(200) + + const ciVisibilityExporter = new CiVisibilityExporter({ port, isEarlyFlakeDetectionEnabled: false }) + + ciVisibilityExporter._resolveCanUseCiVisProtocol(true) + + ciVisibilityExporter.getKnownTests({}, (err, knownTests) => { + expect(err).to.be.null + expect(knownTests).to.eql(undefined) + expect(scope.isDone()).not.to.be.true + done() + }) + }) + }) + context('if early flake detection is enabled but can not use CI Visibility protocol', () => { + it('should not request known tests', (done) => { + const scope = nock(`http://localhost:${port}`) + .post('/api/v2/ci/libraries/tests') + .reply(200) + + const ciVisibilityExporter = new CiVisibilityExporter({ port, isEarlyFlakeDetectionEnabled: true }) + + ciVisibilityExporter._resolveCanUseCiVisProtocol(false) + ciVisibilityExporter._libraryConfig = { isEarlyFlakeDetectionEnabled: true } + ciVisibilityExporter.getKnownTests({}, (err) => { + expect(err).to.be.null + expect(scope.isDone()).not.to.be.true + done() + }) + }) + }) + context('if early flake detection is enabled and can use CI Vis Protocol', () => { + it('should request known tests', (done) => { + const scope = nock(`http://localhost:${port}`) + .post('/api/v2/ci/libraries/tests') + .reply(200, JSON.stringify({ + data: { + attributes: { + tests: { + jest: { + suite1: ['test1'], + suite2: ['test2'] + } + } + } + } + })) + + const ciVisibilityExporter = new CiVisibilityExporter({ port, isEarlyFlakeDetectionEnabled: true }) + + ciVisibilityExporter._resolveCanUseCiVisProtocol(true) + ciVisibilityExporter._libraryConfig = { isEarlyFlakeDetectionEnabled: true } + ciVisibilityExporter.getKnownTests({}, (err, knownTests) => { + expect(err).to.be.null + expect(knownTests).to.eql({ + jest: { + suite1: ['test1'], + suite2: ['test2'] + } + }) + expect(scope.isDone()).to.be.true + done() + }) + }) + it('should return an error if the request fails', (done) => { + const scope = nock(`http://localhost:${port}`) + .post('/api/v2/ci/libraries/tests') + .reply(500) + const ciVisibilityExporter = new CiVisibilityExporter({ port, isEarlyFlakeDetectionEnabled: true }) + + ciVisibilityExporter._resolveCanUseCiVisProtocol(true) + ciVisibilityExporter._libraryConfig = { isEarlyFlakeDetectionEnabled: true } + ciVisibilityExporter.getKnownTests({}, (err) => { + expect(err).not.to.be.null + expect(scope.isDone()).to.be.true + done() + }) + }) + it('should accept gzip if the exporter is gzip compatible', (done) => { + let requestHeaders = {} + const scope = nock(`http://localhost:${port}`) + .post('/api/v2/ci/libraries/tests') + .reply(200, function () { + requestHeaders = this.req.headers + + return zlib.gzipSync(JSON.stringify({ + data: { + attributes: { + tests: { + jest: { + suite1: ['test1'], + suite2: ['test2'] + } + } + } + } + })) + }, { + 'content-encoding': 'gzip' + }) + + const ciVisibilityExporter = new CiVisibilityExporter({ port, isEarlyFlakeDetectionEnabled: true }) + + ciVisibilityExporter._resolveCanUseCiVisProtocol(true) + ciVisibilityExporter._libraryConfig = { isEarlyFlakeDetectionEnabled: true } + ciVisibilityExporter._isGzipCompatible = true + ciVisibilityExporter.getKnownTests({}, (err, knownTests) => { + expect(err).to.be.null + expect(knownTests).to.eql({ + jest: { + suite1: ['test1'], + suite2: ['test2'] + } + }) + expect(scope.isDone()).to.be.true + expect(requestHeaders['accept-encoding']).to.equal('gzip') + done() + }) + }) + it('should not accept gzip if the exporter is gzip incompatible', (done) => { + let requestHeaders = {} + const scope = nock(`http://localhost:${port}`) + .post('/api/v2/ci/libraries/tests') + .reply(200, function () { + requestHeaders = this.req.headers + + return JSON.stringify({ + data: { + attributes: { + tests: { + jest: { + suite1: ['test1'], + suite2: ['test2'] + } + } + } + } + }) + }) + + const ciVisibilityExporter = new CiVisibilityExporter({ port, isEarlyFlakeDetectionEnabled: true }) + + ciVisibilityExporter._resolveCanUseCiVisProtocol(true) + ciVisibilityExporter._libraryConfig = { isEarlyFlakeDetectionEnabled: true } + + ciVisibilityExporter._isGzipCompatible = false + + ciVisibilityExporter.getKnownTests({}, (err, knownTests) => { + expect(err).to.be.null + expect(knownTests).to.eql({ + jest: { + suite1: ['test1'], + suite2: ['test2'] + } + }) + expect(scope.isDone()).to.be.true + expect(requestHeaders['accept-encoding']).not.to.equal('gzip') + done() + }) + }) + }) + }) }) diff --git a/packages/dd-trace/test/ci-visibility/exporters/git/git_metadata.spec.js b/packages/dd-trace/test/ci-visibility/exporters/git/git_metadata.spec.js index f7cb6c5e358..b4d0acb747d 100644 --- a/packages/dd-trace/test/ci-visibility/exporters/git/git_metadata.spec.js +++ b/packages/dd-trace/test/ci-visibility/exporters/git/git_metadata.spec.js @@ -23,7 +23,7 @@ describe('git_metadata', () => { let getLatestCommitsStub let getRepositoryUrlStub - let getCommitsToUploadStub + let getCommitsRevListStub let generatePackFilesForCommitsStub let isShallowRepositoryStub let unshallowRepositoryStub @@ -42,7 +42,7 @@ describe('git_metadata', () => { beforeEach(() => { getLatestCommitsStub = sinon.stub().returns(latestCommits) - getCommitsToUploadStub = sinon.stub().returns(latestCommits) + getCommitsRevListStub = sinon.stub().returns(latestCommits) getRepositoryUrlStub = sinon.stub().returns('git@github.com:DataDog/dd-trace-js.git') isShallowRepositoryStub = sinon.stub().returns(false) unshallowRepositoryStub = sinon.stub() @@ -54,7 +54,7 @@ describe('git_metadata', () => { getLatestCommits: getLatestCommitsStub, getRepositoryUrl: getRepositoryUrlStub, generatePackFilesForCommits: generatePackFilesForCommitsStub, - getCommitsToUpload: getCommitsToUploadStub, + getCommitsRevList: getCommitsRevListStub, isShallowRepository: isShallowRepositoryStub, unshallowRepository: unshallowRepositoryStub } @@ -65,15 +65,31 @@ describe('git_metadata', () => { nock.cleanAll() }) - it('should unshallow if the repo is shallow', (done) => { + it('does not unshallow if every commit is already in backend', (done) => { const scope = nock('https://api.test.com') .post('/api/v2/git/repository/search_commits') + .reply(200, JSON.stringify({ data: latestCommits.map((sha) => ({ id: sha, type: 'commit' })) })) + + isShallowRepositoryStub.returns(true) + gitMetadata.sendGitMetadata(new URL('https://api.test.com'), { isEvpProxy: false }, '', (err) => { + expect(unshallowRepositoryStub).not.to.have.been.called + expect(err).to.be.null + expect(scope.isDone()).to.be.true + done() + }) + }) + + it('should unshallow if the repo is shallow and not every commit is in the backend', (done) => { + const scope = nock('https://api.test.com') + .post('/api/v2/git/repository/search_commits') + .reply(200, JSON.stringify({ data: [] })) + .post('/api/v2/git/repository/search_commits') // calls a second time after unshallowing .reply(200, JSON.stringify({ data: [] })) .post('/api/v2/git/repository/packfile') .reply(204) isShallowRepositoryStub.returns(true) - gitMetadata.sendGitMetadata(new URL('https://api.test.com'), false, '', (err) => { + gitMetadata.sendGitMetadata(new URL('https://api.test.com'), { isEvpProxy: false }, '', (err) => { expect(unshallowRepositoryStub).to.have.been.called expect(err).to.be.null expect(scope.isDone()).to.be.true @@ -88,7 +104,7 @@ describe('git_metadata', () => { .post('/api/v2/git/repository/packfile') .reply(204) - gitMetadata.sendGitMetadata(new URL('https://api.test.com'), false, '', (err) => { + gitMetadata.sendGitMetadata(new URL('https://api.test.com'), { isEvpProxy: false }, '', (err) => { expect(err).to.be.null expect(scope.isDone()).to.be.true done() @@ -102,9 +118,9 @@ describe('git_metadata', () => { .post('/api/v2/git/repository/packfile') .reply(204) - getCommitsToUploadStub.returns([]) + getCommitsRevListStub.returns([]) - gitMetadata.sendGitMetadata(new URL('https://api.test.com'), false, '', (err) => { + gitMetadata.sendGitMetadata(new URL('https://api.test.com'), { isEvpProxy: false }, '', (err) => { expect(err).to.be.null // to check that it is not called expect(scope.isDone()).to.be.false @@ -120,7 +136,7 @@ describe('git_metadata', () => { .post('/api/v2/git/repository/packfile') .reply(204) - gitMetadata.sendGitMetadata(new URL('https://api.test.com'), false, '', (err) => { + gitMetadata.sendGitMetadata(new URL('https://api.test.com'), { isEvpProxy: false }, '', (err) => { // eslint-disable-next-line expect(err.message).to.contain('Error fetching commits to exclude: Error from https://api.test.com/api/v2/git/repository/search_commits: 404 Not Found. Response from the endpoint: "Not found SHA"') // to check that it is not called @@ -137,7 +153,7 @@ describe('git_metadata', () => { .post('/api/v2/git/repository/packfile') .reply(204) - gitMetadata.sendGitMetadata(new URL('https://api.test.com'), false, '', (err) => { + gitMetadata.sendGitMetadata(new URL('https://api.test.com'), { isEvpProxy: false }, '', (err) => { expect(err.message).to.contain("Can't parse commits to exclude response: Invalid commit type response") // to check that it is not called expect(scope.isDone()).to.be.false @@ -153,7 +169,7 @@ describe('git_metadata', () => { .post('/api/v2/git/repository/packfile') .reply(204) - gitMetadata.sendGitMetadata(new URL('https://api.test.com'), false, '', (err) => { + gitMetadata.sendGitMetadata(new URL('https://api.test.com'), { isEvpProxy: false }, '', (err) => { expect(err.message).to.contain("Can't parse commits to exclude response: Invalid commit format") // to check that it is not called expect(scope.isDone()).to.be.false @@ -165,17 +181,31 @@ describe('git_metadata', () => { it('should fail if the packfile request returns anything other than 204', (done) => { const scope = nock('https://api.test.com') .post('/api/v2/git/repository/search_commits') - .reply(200, JSON.stringify({ data: latestCommits.map((sha) => ({ id: sha, type: 'commit' })) })) + .reply(200, JSON.stringify({ data: [] })) .post('/api/v2/git/repository/packfile') .reply(502) - gitMetadata.sendGitMetadata(new URL('https://api.test.com'), false, '', (err) => { + gitMetadata.sendGitMetadata(new URL('https://api.test.com'), { isEvpProxy: false }, '', (err) => { expect(err.message).to.contain('Could not upload packfiles: status code 502') expect(scope.isDone()).to.be.true done() }) }) + it('should fail if the getCommitsRevList fails because the repository is too big', (done) => { + // returning null means that the git rev-list failed + getCommitsRevListStub.returns(null) + const scope = nock('https://api.test.com') + .post('/api/v2/git/repository/search_commits') + .reply(200, JSON.stringify({ data: [] })) + + gitMetadata.sendGitMetadata(new URL('https://api.test.com'), { isEvpProxy: false }, '', (err) => { + expect(err.message).to.contain('git rev-list failed') + expect(scope.isDone()).to.be.true + done() + }) + }) + it('should fire a request per packfile', (done) => { const scope = nock('https://api.test.com') .post('/api/v2/git/repository/search_commits') @@ -196,39 +226,43 @@ describe('git_metadata', () => { secondTemporaryPackFile ]) - gitMetadata.sendGitMetadata(new URL('https://api.test.com'), false, '', (err) => { + gitMetadata.sendGitMetadata(new URL('https://api.test.com'), { isEvpProxy: false }, '', (err) => { expect(err).to.be.null expect(scope.isDone()).to.be.true done() }) }) + describe('validateGitRepositoryUrl', () => { it('should return false if Git repository URL is invalid', () => { - const invalidUrl1 = 'https://test.com' - const invalidUrl2 = 'https://test.com' - const invalidUrl3 = 'http://test.com/repo/dummy.4git' - const invalidUrl4 = 'https://test.com/repo/dummy.gi' - const invalidUrl5 = 'www.test.com/repo/dummy.git' - const invalidUrl6 = 'test.com/repo/dummy.git' - - const invalidUrls = [invalidUrl1, invalidUrl2, invalidUrl3, invalidUrl4, invalidUrl5, invalidUrl6] + const invalidUrls = [ + 'www.test.com/repo/dummy.git', + 'test.com/repo/dummy.git', + 'test.com/repo/dummy' + ] invalidUrls.forEach((invalidUrl) => { - expect(validateGitRepositoryUrl(invalidUrl)).to.be.false + expect(validateGitRepositoryUrl(invalidUrl), `${invalidUrl} is a valid URL`).to.be.false }) }) + it('should return true if Git repository URL is valid', () => { - const validUrl1 = 'https://test.com/repo/dummy.git' - const validUrl2 = 'http://test.com/repo/dummy.git' - const validUrl3 = 'https://github.com/DataDog/dd-trace-js.git' - const validUrl4 = 'git@github.com:DataDog/dd-trace-js.git' - const validUrl5 = 'git@github.com:user/repo.git' + const validUrls = [ + 'https://test.com', + 'https://test.com/repo/dummy.git', + 'http://test.com/repo/dummy.git', + 'https://github.com/DataDog/dd-trace-js.git', + 'https://github.com/DataDog/dd-trace-js', + 'git@github.com:DataDog/dd-trace-js.git', + 'git@github.com:user/repo.git', + 'git@github.com:user/repo' + ] - const validUrls = [validUrl1, validUrl2, validUrl3, validUrl4, validUrl5] validUrls.forEach((validUrl) => { - expect(validateGitRepositoryUrl(validUrl)).to.be.true + expect(validateGitRepositoryUrl(validUrl), `${validUrl} is an invalid URL`).to.be.true }) }) }) + describe('validateGitCommitSha', () => { it('should return false if Git commit SHA is invalid', () => { const invalidSha1 = 'cb466452bfe18d4f6be2836c2a5551843013cf382' @@ -243,6 +277,7 @@ describe('git_metadata', () => { expect(validateGitCommitSha(invalidSha)).to.be.false }) }) + it('should return true if Git commit SHA is valid', () => { const validSha1 = 'cb466452bfe18d4f6be2836c2a5551843013cf38' const validSha2 = 'cb466452bfe18d4f6be2836c2a5551843013cf381234223920318230492823f3' @@ -253,6 +288,7 @@ describe('git_metadata', () => { }) }) }) + it('should not crash if packfiles can not be accessed', (done) => { const scope = nock('https://api.test.com') .post('/api/v2/git/repository/search_commits') @@ -265,7 +301,7 @@ describe('git_metadata', () => { 'not there either' ]) - gitMetadata.sendGitMetadata(new URL('https://api.test.com'), false, '', (err) => { + gitMetadata.sendGitMetadata(new URL('https://api.test.com'), { isEvpProxy: false }, '', (err) => { expect(err.message).to.contain('Could not read "not-there"') expect(scope.isDone()).to.be.false done() @@ -281,7 +317,7 @@ describe('git_metadata', () => { generatePackFilesForCommitsStub.returns([]) - gitMetadata.sendGitMetadata(new URL('https://api.test.com'), false, '', (err) => { + gitMetadata.sendGitMetadata(new URL('https://api.test.com'), { isEvpProxy: false }, '', (err) => { expect(err.message).to.contain('Failed to generate packfiles') expect(scope.isDone()).to.be.false done() @@ -289,17 +325,20 @@ describe('git_metadata', () => { }) it('should not crash if git is missing', (done) => { + const oldPath = process.env.PATH + // git will not be found + process.env.PATH = '' + const scope = nock('https://api.test.com') .post('/api/v2/git/repository/search_commits') .reply(200, JSON.stringify({ data: [] })) .post('/api/v2/git/repository/packfile') .reply(204) - getRepositoryUrlStub.returns('') - - gitMetadata.sendGitMetadata(new URL('https://api.test.com'), false, '', (err) => { - expect(err.message).to.contain('Repository URL is empty') + gitMetadata.sendGitMetadata(new URL('https://api.test.com'), { isEvpProxy: false }, '', (err) => { + expect(err.message).to.contain('Git is not available') expect(scope.isDone()).to.be.false + process.env.PATH = oldPath done() }) }) @@ -315,7 +354,7 @@ describe('git_metadata', () => { .post('/api/v2/git/repository/packfile') .reply(204) - gitMetadata.sendGitMetadata(new URL('https://api.test.com'), false, '', (err) => { + gitMetadata.sendGitMetadata(new URL('https://api.test.com'), { isEvpProxy: false }, '', (err) => { expect(err).to.be.null expect(scope.isDone()).to.be.true done() @@ -332,10 +371,14 @@ describe('git_metadata', () => { done() }) - gitMetadata.sendGitMetadata(new URL('https://api.test.com'), true, '', (err) => { - expect(err).to.be.null - expect(scope.isDone()).to.be.true - }) + gitMetadata.sendGitMetadata( + new URL('https://api.test.com'), + { isEvpProxy: true, evpProxyPrefix: '/evp_proxy/v2' }, + '', + (err) => { + expect(err).to.be.null + expect(scope.isDone()).to.be.true + }) }) it('should use the input repository url and not call getRepositoryUrl', (done) => { @@ -353,14 +396,18 @@ describe('git_metadata', () => { .post('/evp_proxy/v2/api/v2/git/repository/packfile') .reply(204) - gitMetadata.sendGitMetadata(new URL('https://api.test.com'), true, 'https://custom-git@datadog.com', (err) => { - expect(err).to.be.null - expect(scope.isDone()).to.be.true - requestPromise.then((repositoryUrl) => { - expect(getRepositoryUrlStub).not.to.have.been.called - expect(repositoryUrl).to.equal('https://custom-git@datadog.com') - done() + gitMetadata.sendGitMetadata( + new URL('https://api.test.com'), + { isEvpProxy: true, evpProxyPrefix: '/evp_proxy/v2' }, + 'https://custom-git@datadog.com', + (err) => { + expect(err).to.be.null + expect(scope.isDone()).to.be.true + requestPromise.then((repositoryUrl) => { + expect(getRepositoryUrlStub).not.to.have.been.called + expect(repositoryUrl).to.equal('https://custom-git@datadog.com') + done() + }) }) - }) }) }) diff --git a/packages/dd-trace/test/ci-visibility/exporters/jest-worker/exporter.spec.js b/packages/dd-trace/test/ci-visibility/exporters/jest-worker/exporter.spec.js deleted file mode 100644 index dd673400d28..00000000000 --- a/packages/dd-trace/test/ci-visibility/exporters/jest-worker/exporter.spec.js +++ /dev/null @@ -1,49 +0,0 @@ -'use strict' - -require('../../../../../dd-trace/test/setup/tap') - -const JestWorkerCiVisibilityExporter = require('../../../../src/ci-visibility/exporters/jest-worker') -const { - JEST_WORKER_TRACE_PAYLOAD_CODE, - JEST_WORKER_COVERAGE_PAYLOAD_CODE -} = require('../../../../src/plugins/util/test') - -describe('CI Visibility Jest Worker Exporter', () => { - let send, originalSend - beforeEach(() => { - send = sinon.spy() - originalSend = process.send - process.send = send - }) - afterEach(() => { - process.send = originalSend - }) - it('can export traces', () => { - const trace = [{ type: 'test' }] - const traceSecond = [{ type: 'test', name: 'other' }] - const jestWorkerExporter = new JestWorkerCiVisibilityExporter() - jestWorkerExporter.export(trace) - jestWorkerExporter.export(traceSecond) - jestWorkerExporter.flush() - expect(send).to.have.been.calledWith([JEST_WORKER_TRACE_PAYLOAD_CODE, JSON.stringify([trace, traceSecond])]) - }) - it('can export coverages', () => { - const coverage = { sessionId: '1', suiteId: '1', files: ['test.js'] } - const coverageSecond = { sessionId: '2', suiteId: '2', files: ['test2.js'] } - const jestWorkerExporter = new JestWorkerCiVisibilityExporter() - jestWorkerExporter.exportCoverage(coverage) - jestWorkerExporter.exportCoverage(coverageSecond) - jestWorkerExporter.flush() - expect(send).to.have.been.calledWith( - [JEST_WORKER_COVERAGE_PAYLOAD_CODE, JSON.stringify([coverage, coverageSecond])] - ) - }) - it('does not break if process.send is undefined', () => { - delete process.send - const trace = [{ type: 'test' }] - const jestWorkerExporter = new JestWorkerCiVisibilityExporter() - jestWorkerExporter.export(trace) - jestWorkerExporter.flush() - expect(send).not.to.have.been.called - }) -}) diff --git a/packages/dd-trace/test/ci-visibility/exporters/test-worker/exporter.spec.js b/packages/dd-trace/test/ci-visibility/exporters/test-worker/exporter.spec.js new file mode 100644 index 00000000000..3322fbb8e85 --- /dev/null +++ b/packages/dd-trace/test/ci-visibility/exporters/test-worker/exporter.spec.js @@ -0,0 +1,121 @@ +'use strict' + +require('../../../../../dd-trace/test/setup/tap') + +const TestWorkerCiVisibilityExporter = require('../../../../src/ci-visibility/exporters/test-worker') +const { + JEST_WORKER_TRACE_PAYLOAD_CODE, + JEST_WORKER_COVERAGE_PAYLOAD_CODE, + CUCUMBER_WORKER_TRACE_PAYLOAD_CODE, + MOCHA_WORKER_TRACE_PAYLOAD_CODE +} = require('../../../../src/plugins/util/test') + +describe('CI Visibility Test Worker Exporter', () => { + let send, originalSend + + beforeEach(() => { + send = sinon.spy() + originalSend = process.send + process.send = send + }) + + afterEach(() => { + process.send = originalSend + }) + + context('when the process is a jest worker', () => { + beforeEach(() => { + process.env.JEST_WORKER_ID = '1' + }) + afterEach(() => { + delete process.env.JEST_WORKER_ID + }) + + it('can export traces', () => { + const trace = [{ type: 'test' }] + const traceSecond = [{ type: 'test', name: 'other' }] + const jestWorkerExporter = new TestWorkerCiVisibilityExporter() + jestWorkerExporter.export(trace) + jestWorkerExporter.export(traceSecond) + jestWorkerExporter.flush() + expect(send).to.have.been.calledWith([JEST_WORKER_TRACE_PAYLOAD_CODE, JSON.stringify([trace, traceSecond])]) + }) + + it('can export coverages', () => { + const coverage = { sessionId: '1', suiteId: '1', files: ['test.js'] } + const coverageSecond = { sessionId: '2', suiteId: '2', files: ['test2.js'] } + const jestWorkerExporter = new TestWorkerCiVisibilityExporter() + jestWorkerExporter.exportCoverage(coverage) + jestWorkerExporter.exportCoverage(coverageSecond) + jestWorkerExporter.flush() + expect(send).to.have.been.calledWith( + [JEST_WORKER_COVERAGE_PAYLOAD_CODE, JSON.stringify([coverage, coverageSecond])] + ) + }) + + it('does not break if process.send is undefined', () => { + delete process.send + const trace = [{ type: 'test' }] + const jestWorkerExporter = new TestWorkerCiVisibilityExporter() + jestWorkerExporter.export(trace) + jestWorkerExporter.flush() + expect(send).not.to.have.been.called + }) + }) + + context('when the process is a cucumber worker', () => { + beforeEach(() => { + process.env.CUCUMBER_WORKER_ID = '1' + }) + afterEach(() => { + delete process.env.CUCUMBER_WORKER_ID + }) + + it('can export traces', () => { + const trace = [{ type: 'test' }] + const traceSecond = [{ type: 'test', name: 'other' }] + const cucumberWorkerExporter = new TestWorkerCiVisibilityExporter() + cucumberWorkerExporter.export(trace) + cucumberWorkerExporter.export(traceSecond) + cucumberWorkerExporter.flush() + expect(send).to.have.been.calledWith([CUCUMBER_WORKER_TRACE_PAYLOAD_CODE, JSON.stringify([trace, traceSecond])]) + }) + + it('does not break if process.send is undefined', () => { + delete process.send + const trace = [{ type: 'test' }] + const cucumberWorkerExporter = new TestWorkerCiVisibilityExporter() + cucumberWorkerExporter.export(trace) + cucumberWorkerExporter.flush() + expect(send).not.to.have.been.called + }) + }) + + context('when the process is a mocha worker', () => { + beforeEach(() => { + process.env.MOCHA_WORKER_ID = '1' + }) + afterEach(() => { + delete process.env.MOCHA_WORKER_ID + }) + + it('can export traces', () => { + const trace = [{ type: 'test' }] + const traceSecond = [{ type: 'test', name: 'other' }] + const mochaWorkerExporter = new TestWorkerCiVisibilityExporter() + mochaWorkerExporter.export(trace) + mochaWorkerExporter.export(traceSecond) + mochaWorkerExporter.flush() + expect(send).to.have.been.calledWith([MOCHA_WORKER_TRACE_PAYLOAD_CODE, JSON.stringify([trace, traceSecond])]) + }) + + it('does not break if process.send is undefined', () => { + delete process.send + const trace = [{ type: 'test' }] + const mochaWorkerExporter = new TestWorkerCiVisibilityExporter() + mochaWorkerExporter.export(trace) + mochaWorkerExporter.flush() + expect(send).not.to.have.been.called + }) + }) +}) diff --git a/packages/dd-trace/test/config.spec.js b/packages/dd-trace/test/config.spec.js index 3459c50e260..4246167725d 100644 --- a/packages/dd-trace/test/config.spec.js +++ b/packages/dd-trace/test/config.spec.js @@ -4,6 +4,8 @@ require('./setup/tap') const { expect } = require('chai') const { readFileSync } = require('fs') +const sinon = require('sinon') +const { GRPC_CLIENT_ERROR_STATUSES, GRPC_SERVER_ERROR_STATUSES } = require('../src/constants') describe('Config', () => { let Config @@ -15,6 +17,7 @@ describe('Config', () => { let existsSyncParam let existsSyncReturn let osType + let updateConfig const RECOMMENDED_JSON_PATH = require.resolve('../src/appsec/recommended.json') const RULES_JSON_PATH = require.resolve('./fixtures/config/appsec-rules.json') @@ -22,20 +25,33 @@ describe('Config', () => { const BLOCKED_TEMPLATE_HTML = readFileSync(BLOCKED_TEMPLATE_HTML_PATH, { encoding: 'utf8' }) const BLOCKED_TEMPLATE_JSON_PATH = require.resolve('./fixtures/config/appsec-blocked-template.json') const BLOCKED_TEMPLATE_JSON = readFileSync(BLOCKED_TEMPLATE_JSON_PATH, { encoding: 'utf8' }) + const BLOCKED_TEMPLATE_GRAPHQL_PATH = require.resolve('./fixtures/config/appsec-blocked-graphql-template.json') + const BLOCKED_TEMPLATE_GRAPHQL = readFileSync(BLOCKED_TEMPLATE_GRAPHQL_PATH, { encoding: 'utf8' }) const DD_GIT_PROPERTIES_FILE = require.resolve('./fixtures/config/git.properties') + function reloadLoggerAndConfig () { + log = proxyquire('../src/log', {}) + log.use = sinon.spy() + log.toggle = sinon.spy() + log.warn = sinon.spy() + log.error = sinon.spy() + + Config = proxyquire('../src/config', { + './pkg': pkg, + './log': log, + './telemetry': { updateConfig }, + fs, + os + }) + } + beforeEach(() => { pkg = { name: '', version: '' } - log = { - use: sinon.spy(), - toggle: sinon.spy(), - warn: sinon.spy(), - error: sinon.spy() - } + updateConfig = sinon.stub() env = process.env process.env = {} @@ -52,19 +68,136 @@ describe('Config', () => { } osType = 'Linux' - Config = proxyquire('../src/config', { - './pkg': pkg, - './log': log, - fs, - os - }) + reloadLoggerAndConfig() }) afterEach(() => { + updateConfig.reset() process.env = env existsSyncParam = undefined }) + it('should initialize its own logging config based off the loggers config', () => { + process.env.DD_TRACE_DEBUG = 'true' + process.env.DD_TRACE_LOG_LEVEL = 'error' + + reloadLoggerAndConfig() + + const config = new Config() + + expect(config).to.have.property('debug', true) + expect(config).to.have.property('logger', undefined) + expect(config).to.have.property('logLevel', 'error') + }) + + it('should initialize from environment variables with DD env vars taking precedence OTEL env vars', () => { + process.env.DD_SERVICE = 'service' + process.env.OTEL_SERVICE_NAME = 'otel_service' + process.env.DD_TRACE_LOG_LEVEL = 'error' + process.env.DD_TRACE_DEBUG = 'false' + process.env.OTEL_LOG_LEVEL = 'debug' + process.env.DD_TRACE_SAMPLE_RATE = '0.5' + process.env.OTEL_TRACES_SAMPLER = 'traceidratio' + process.env.OTEL_TRACES_SAMPLER_ARG = '0.1' + process.env.DD_TRACE_ENABLED = 'true' + process.env.OTEL_TRACES_EXPORTER = 'none' + process.env.DD_RUNTIME_METRICS_ENABLED = 'true' + process.env.OTEL_METRICS_EXPORTER = 'none' + process.env.DD_TAGS = 'foo:bar,baz:qux' + process.env.OTEL_RESOURCE_ATTRIBUTES = 'foo=bar1,baz=qux1' + process.env.DD_TRACE_PROPAGATION_STYLE_INJECT = 'b3,tracecontext' + process.env.DD_TRACE_PROPAGATION_STYLE_EXTRACT = 'b3,tracecontext' + process.env.OTEL_PROPAGATORS = 'datadog,tracecontext' + + // required if we want to check updates to config.debug and config.logLevel which is fetched from logger + reloadLoggerAndConfig() + + const config = new Config() + + expect(config).to.have.property('debug', false) + expect(config).to.have.property('service', 'service') + expect(config).to.have.property('logLevel', 'error') + expect(config).to.have.property('sampleRate', 0.5) + expect(config).to.have.property('runtimeMetrics', true) + expect(config.tags).to.include({ foo: 'bar', baz: 'qux' }) + expect(config).to.have.nested.deep.property('tracePropagationStyle.inject', ['b3', 'tracecontext']) + expect(config).to.have.nested.deep.property('tracePropagationStyle.extract', ['b3', 'tracecontext']) + expect(config).to.have.nested.deep.property('tracePropagationStyle.otelPropagators', false) + + const indexFile = require('../src/index') + const proxy = require('../src/proxy') + expect(indexFile).to.equal(proxy) + }) + + it('should initialize with OTEL environment variables when DD env vars are not set', () => { + process.env.OTEL_SERVICE_NAME = 'otel_service' + process.env.OTEL_LOG_LEVEL = 'debug' + process.env.OTEL_TRACES_SAMPLER = 'traceidratio' + process.env.OTEL_TRACES_SAMPLER_ARG = '0.1' + process.env.OTEL_TRACES_EXPORTER = 'none' + process.env.OTEL_METRICS_EXPORTER = 'none' + process.env.OTEL_RESOURCE_ATTRIBUTES = 'foo=bar1,baz=qux1' + process.env.OTEL_PROPAGATORS = 'b3,datadog' + + // required if we want to check updates to config.debug and config.logLevel which is fetched from logger + reloadLoggerAndConfig() + + const config = new Config() + + expect(config).to.have.property('debug', true) + expect(config).to.have.property('service', 'otel_service') + expect(config).to.have.property('logLevel', 'debug') + expect(config).to.have.property('sampleRate', 0.1) + expect(config).to.have.property('runtimeMetrics', false) + expect(config.tags).to.include({ foo: 'bar1', baz: 'qux1' }) + expect(config).to.have.nested.deep.property('tracePropagationStyle.inject', ['b3', 'datadog']) + expect(config).to.have.nested.deep.property('tracePropagationStyle.extract', ['b3', 'datadog']) + expect(config).to.have.nested.deep.property('tracePropagationStyle.otelPropagators', true) + + delete require.cache[require.resolve('../src/index')] + const indexFile = require('../src/index') + const noop = require('../src/noop/proxy') + expect(indexFile).to.equal(noop) + }) + + it('should correctly map OTEL_RESOURCE_ATTRIBUTES', () => { + process.env.OTEL_RESOURCE_ATTRIBUTES = + 'deployment.environment=test1,service.name=test2,service.version=5,foo=bar1,baz=qux1' + const config = new Config() + + expect(config).to.have.property('env', 'test1') + expect(config).to.have.property('service', 'test2') + expect(config).to.have.property('version', '5') + expect(config.tags).to.include({ foo: 'bar1', baz: 'qux1' }) + }) + + it('should correctly map OTEL_TRACES_SAMPLER and OTEL_TRACES_SAMPLER_ARG', () => { + process.env.OTEL_TRACES_SAMPLER = 'always_on' + process.env.OTEL_TRACES_SAMPLER_ARG = '0.1' + let config = new Config() + expect(config).to.have.property('sampleRate', 1.0) + + process.env.OTEL_TRACES_SAMPLER = 'always_off' + config = new Config() + expect(config).to.have.property('sampleRate', 0.0) + + process.env.OTEL_TRACES_SAMPLER = 'traceidratio' + config = new Config() + expect(config).to.have.property('sampleRate', 0.1) + + process.env.OTEL_TRACES_SAMPLER = 'parentbased_always_on' + config = new Config() + expect(config).to.have.property('sampleRate', 1.0) + + process.env.OTEL_TRACES_SAMPLER = 'parentbased_always_off' + config = new Config() + expect(config).to.have.property('sampleRate', 0.0) + + process.env.OTEL_TRACES_SAMPLER = 'parentbased_traceidratio' + config = new Config() + expect(config).to.have.property('sampleRate', 0.1) + }) + it('should initialize with the correct defaults', () => { const config = new Config() @@ -83,15 +216,21 @@ describe('Config', () => { expect(config).to.have.property('runtimeMetrics', false) expect(config.tags).to.have.property('service', 'node') expect(config).to.have.property('plugins', true) + expect(config).to.have.property('traceEnabled', true) expect(config).to.have.property('env', undefined) expect(config).to.have.property('reportHostname', false) expect(config).to.have.property('scope', undefined) expect(config).to.have.property('logLevel', 'debug') + expect(config).to.have.nested.property('codeOriginForSpans.enabled', false) + expect(config).to.have.property('dynamicInstrumentationEnabled', false) expect(config).to.have.property('traceId128BitGenerationEnabled', true) expect(config).to.have.property('traceId128BitLoggingEnabled', false) expect(config).to.have.property('spanAttributeSchema', 'v0') + expect(config.grpc.client.error.statuses).to.deep.equal(GRPC_CLIENT_ERROR_STATUSES) + expect(config.grpc.server.error.statuses).to.deep.equal(GRPC_SERVER_ERROR_STATUSES) expect(config).to.have.property('spanComputePeerService', false) expect(config).to.have.property('spanRemoveIntegrationFromService', false) + expect(config).to.have.property('instrumentation_config_id', undefined) expect(config).to.have.deep.property('serviceMapping', {}) expect(config).to.have.nested.deep.property('tracePropagationStyle.inject', ['datadog', 'tracecontext']) expect(config).to.have.nested.deep.property('tracePropagationStyle.extract', ['datadog', 'tracecontext']) @@ -100,17 +239,23 @@ describe('Config', () => { expect(config).to.have.nested.property('experimental.enableGetRumData', false) expect(config).to.have.nested.property('appsec.enabled', undefined) expect(config).to.have.nested.property('appsec.rules', undefined) - expect(config).to.have.nested.property('appsec.customRulesProvided', false) + expect(config).to.have.nested.property('appsec.rasp.enabled', true) expect(config).to.have.nested.property('appsec.rateLimit', 100) + expect(config).to.have.nested.property('appsec.stackTrace.enabled', true) + expect(config).to.have.nested.property('appsec.stackTrace.maxDepth', 32) + expect(config).to.have.nested.property('appsec.stackTrace.maxStackTraces', 2) expect(config).to.have.nested.property('appsec.wafTimeout', 5e3) - expect(config).to.have.nested.property('appsec.obfuscatorKeyRegex').with.length(155) - expect(config).to.have.nested.property('appsec.obfuscatorValueRegex').with.length(443) + expect(config).to.have.nested.property('appsec.obfuscatorKeyRegex').with.length(190) + expect(config).to.have.nested.property('appsec.obfuscatorValueRegex').with.length(550) expect(config).to.have.nested.property('appsec.blockedTemplateHtml', undefined) expect(config).to.have.nested.property('appsec.blockedTemplateJson', undefined) + expect(config).to.have.nested.property('appsec.blockedTemplateGraphql', undefined) expect(config).to.have.nested.property('appsec.eventTracking.enabled', true) expect(config).to.have.nested.property('appsec.eventTracking.mode', 'safe') - expect(config).to.have.nested.property('appsec.apiSecurity.enabled', false) + expect(config).to.have.nested.property('appsec.apiSecurity.enabled', true) expect(config).to.have.nested.property('appsec.apiSecurity.requestSampling', 0.1) + expect(config).to.have.nested.property('appsec.sca.enabled', null) + expect(config).to.have.nested.property('appsec.standalone.enabled', undefined) expect(config).to.have.nested.property('remoteConfig.enabled', true) expect(config).to.have.nested.property('remoteConfig.pollInterval', 5) expect(config).to.have.nested.property('iast.enabled', false) @@ -118,6 +263,121 @@ describe('Config', () => { expect(config).to.have.nested.property('iast.redactionNamePattern', null) expect(config).to.have.nested.property('iast.redactionValuePattern', null) expect(config).to.have.nested.property('iast.telemetryVerbosity', 'INFORMATION') + expect(config).to.have.nested.property('installSignature.id', null) + expect(config).to.have.nested.property('installSignature.time', null) + expect(config).to.have.nested.property('installSignature.type', null) + + expect(updateConfig).to.be.calledOnce + + expect(updateConfig.getCall(0).args[0]).to.deep.include.members([ + { name: 'appsec.blockedTemplateHtml', value: undefined, origin: 'default' }, + { name: 'appsec.blockedTemplateJson', value: undefined, origin: 'default' }, + { name: 'appsec.enabled', value: undefined, origin: 'default' }, + { + name: 'appsec.obfuscatorKeyRegex', + // eslint-disable-next-line max-len + value: '(?i)pass|pw(?:or)?d|secret|(?:api|private|public|access)[_-]?key|token|consumer[_-]?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization|jsessionid|phpsessid|asp\\.net[_-]sessionid|sid|jwt', + origin: 'default' + }, + { + name: 'appsec.obfuscatorValueRegex', + // eslint-disable-next-line max-len + value: '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:[_-]?phrase)?|secret(?:[_-]?key)?|(?:(?:api|private|public|access)[_-]?)key(?:[_-]?id)?|(?:(?:auth|access|id|refresh)[_-]?)?token|consumer[_-]?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?|jsessionid|phpsessid|asp\\.net(?:[_-]|-)sessionid|sid|jwt)(?:\\s*=[^;]|"\\s*:\\s*"[^"]+")|bearer\\s+[a-z0-9\\._\\-]+|token:[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L][\\w=-]+\\.ey[I-L][\\w=-]+(?:\\.[\\w.+\\/=-]+)?|[\\-]{5}BEGIN[a-z\\s]+PRIVATE\\sKEY[\\-]{5}[^\\-]+[\\-]{5}END[a-z\\s]+PRIVATE\\sKEY|ssh-rsa\\s*[a-z0-9\\/\\.+]{100,}', + origin: 'default' + }, + { name: 'appsec.rasp.enabled', value: true, origin: 'default' }, + { name: 'appsec.rateLimit', value: 100, origin: 'default' }, + { name: 'appsec.rules', value: undefined, origin: 'default' }, + { name: 'appsec.sca.enabled', value: null, origin: 'default' }, + { name: 'appsec.standalone.enabled', value: undefined, origin: 'default' }, + { name: 'appsec.stackTrace.enabled', value: true, origin: 'default' }, + { name: 'appsec.stackTrace.maxDepth', value: 32, origin: 'default' }, + { name: 'appsec.stackTrace.maxStackTraces', value: 2, origin: 'default' }, + { name: 'appsec.wafTimeout', value: 5e3, origin: 'default' }, + { name: 'clientIpEnabled', value: false, origin: 'default' }, + { name: 'clientIpHeader', value: null, origin: 'default' }, + { name: 'codeOriginForSpans.enabled', value: false, origin: 'default' }, + { name: 'dbmPropagationMode', value: 'disabled', origin: 'default' }, + { name: 'dogstatsd.hostname', value: '127.0.0.1', origin: 'calculated' }, + { name: 'dogstatsd.port', value: '8125', origin: 'default' }, + { name: 'dsmEnabled', value: false, origin: 'default' }, + { name: 'dynamicInstrumentationEnabled', value: false, origin: 'default' }, + { name: 'env', value: undefined, origin: 'default' }, + { name: 'experimental.enableGetRumData', value: false, origin: 'default' }, + { name: 'experimental.exporter', value: undefined, origin: 'default' }, + { name: 'experimental.runtimeId', value: false, origin: 'default' }, + { name: 'flushInterval', value: 2000, origin: 'default' }, + { name: 'flushMinSpans', value: 1000, origin: 'default' }, + { name: 'gitMetadataEnabled', value: true, origin: 'default' }, + { name: 'headerTags', value: [], origin: 'default' }, + { name: 'hostname', value: '127.0.0.1', origin: 'default' }, + { name: 'iast.cookieFilterPattern', value: '.{32,}', origin: 'default' }, + { name: 'iast.deduplicationEnabled', value: true, origin: 'default' }, + { name: 'iast.enabled', value: false, origin: 'default' }, + { name: 'iast.maxConcurrentRequests', value: 2, origin: 'default' }, + { name: 'iast.maxContextOperations', value: 2, origin: 'default' }, + { name: 'iast.redactionEnabled', value: true, origin: 'default' }, + { name: 'iast.redactionNamePattern', value: null, origin: 'default' }, + { name: 'iast.redactionValuePattern', value: null, origin: 'default' }, + { name: 'iast.requestSampling', value: 30, origin: 'default' }, + { name: 'iast.telemetryVerbosity', value: 'INFORMATION', origin: 'default' }, + { name: 'injectionEnabled', value: [], origin: 'default' }, + { name: 'isCiVisibility', value: false, origin: 'default' }, + { name: 'isEarlyFlakeDetectionEnabled', value: false, origin: 'default' }, + { name: 'isFlakyTestRetriesEnabled', value: false, origin: 'default' }, + { name: 'flakyTestRetriesCount', value: 5, origin: 'default' }, + { name: 'isGCPFunction', value: false, origin: 'env_var' }, + { name: 'isGitUploadEnabled', value: false, origin: 'default' }, + { name: 'isIntelligentTestRunnerEnabled', value: false, origin: 'default' }, + { name: 'isManualApiEnabled', value: false, origin: 'default' }, + { name: 'ciVisibilityTestSessionName', value: '', origin: 'default' }, + { name: 'logInjection', value: false, origin: 'default' }, + { name: 'lookup', value: undefined, origin: 'default' }, + { name: 'openAiLogsEnabled', value: false, origin: 'default' }, + { name: 'openaiSpanCharLimit', value: 128, origin: 'default' }, + { name: 'peerServiceMapping', value: {}, origin: 'default' }, + { name: 'plugins', value: true, origin: 'default' }, + { name: 'port', value: '8126', origin: 'default' }, + { name: 'profiling.enabled', value: undefined, origin: 'default' }, + { name: 'profiling.exporters', value: 'agent', origin: 'default' }, + { name: 'profiling.sourceMap', value: true, origin: 'default' }, + { name: 'protocolVersion', value: '0.4', origin: 'default' }, + { + name: 'queryStringObfuscation', + // eslint-disable-next-line max-len + value: '(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?|access_?|secret_?)key(?:_?id)?|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)(?:(?:\\s|%20)*(?:=|%3D)[^&]+|(?:"|%22)(?:\\s|%20)*(?::|%3A)(?:\\s|%20)*(?:"|%22)(?:%2[^2]|%[^2]|[^"%])+(?:"|%22))|bearer(?:\\s|%20)+[a-z0-9\\._\\-]+|token(?::|%3A)[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L](?:[\\w=-]|%3D)+\\.ey[I-L](?:[\\w=-]|%3D)+(?:\\.(?:[\\w.+\\/=-]|%3D|%2F|%2B)+)?|[\\-]{5}BEGIN(?:[a-z\\s]|%20)+PRIVATE(?:\\s|%20)KEY[\\-]{5}[^\\-]+[\\-]{5}END(?:[a-z\\s]|%20)+PRIVATE(?:\\s|%20)KEY|ssh-rsa(?:\\s|%20)*(?:[a-z0-9\\/\\.+]|%2F|%5C|%2B){100,}', + origin: 'default' + }, + { name: 'remoteConfig.enabled', value: true, origin: 'env_var' }, + { name: 'remoteConfig.pollInterval', value: 5, origin: 'default' }, + { name: 'reportHostname', value: false, origin: 'default' }, + { name: 'reportHostname', value: false, origin: 'default' }, + { name: 'runtimeMetrics', value: false, origin: 'default' }, + { name: 'sampleRate', value: undefined, origin: 'default' }, + { name: 'sampler.rateLimit', value: 100, origin: 'default' }, + { name: 'traceEnabled', value: true, origin: 'default' }, + { name: 'sampler.rules', value: [], origin: 'default' }, + { name: 'scope', value: undefined, origin: 'default' }, + { name: 'service', value: 'node', origin: 'default' }, + { name: 'site', value: 'datadoghq.com', origin: 'default' }, + { name: 'spanAttributeSchema', value: 'v0', origin: 'default' }, + { name: 'spanComputePeerService', value: false, origin: 'calculated' }, + { name: 'spanRemoveIntegrationFromService', value: false, origin: 'default' }, + { name: 'startupLogs', value: false, origin: 'default' }, + { name: 'stats.enabled', value: false, origin: 'calculated' }, + { name: 'tagsHeaderMaxLength', value: 512, origin: 'default' }, + { name: 'telemetry.debug', value: false, origin: 'default' }, + { name: 'telemetry.dependencyCollection', value: true, origin: 'default' }, + { name: 'telemetry.enabled', value: true, origin: 'env_var' }, + { name: 'telemetry.heartbeatInterval', value: 60000, origin: 'default' }, + { name: 'telemetry.logCollection', value: false, origin: 'default' }, + { name: 'telemetry.metrics', value: true, origin: 'default' }, + { name: 'traceId128BitGenerationEnabled', value: true, origin: 'default' }, + { name: 'traceId128BitLoggingEnabled', value: false, origin: 'default' }, + { name: 'tracing', value: true, origin: 'default' }, + { name: 'url', value: undefined, origin: 'default' }, + { name: 'version', value: '', origin: 'default' } + ]) }) it('should support logging', () => { @@ -158,6 +418,7 @@ describe('Config', () => { }) it('should initialize from environment variables', () => { + process.env.DD_CODE_ORIGIN_FOR_SPANS_ENABLED = 'true' process.env.DD_TRACE_AGENT_HOSTNAME = 'agent' process.env.DD_TRACE_AGENT_PORT = '6218' process.env.DD_DOGSTATSD_HOSTNAME = 'dsd-agent' @@ -175,6 +436,7 @@ describe('Config', () => { process.env.DD_RUNTIME_METRICS_ENABLED = 'true' process.env.DD_TRACE_REPORT_HOSTNAME = 'true' process.env.DD_ENV = 'test' + process.env.DD_DYNAMIC_INSTRUMENTATION_ENABLED = 'true' process.env.DD_TRACE_GLOBAL_TAGS = 'foo:bar,baz:qux' process.env.DD_TRACE_SAMPLE_RATE = '0.5' process.env.DD_TRACE_RATE_LIMIT = '-1' @@ -201,20 +463,28 @@ describe('Config', () => { process.env.DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED = 'true' process.env.DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED = true process.env.DD_APPSEC_ENABLED = 'true' + process.env.DD_APPSEC_MAX_STACK_TRACES = '5' + process.env.DD_APPSEC_MAX_STACK_TRACE_DEPTH = '42' + process.env.DD_APPSEC_RASP_ENABLED = 'false' process.env.DD_APPSEC_RULES = RULES_JSON_PATH + process.env.DD_APPSEC_STACK_TRACE_ENABLED = 'false' process.env.DD_APPSEC_TRACE_RATE_LIMIT = '42' process.env.DD_APPSEC_WAF_TIMEOUT = '42' process.env.DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP = '.*' process.env.DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP = '.*' process.env.DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML = BLOCKED_TEMPLATE_HTML_PATH process.env.DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON = BLOCKED_TEMPLATE_JSON_PATH + process.env.DD_APPSEC_GRAPHQL_BLOCKED_TEMPLATE_JSON = BLOCKED_TEMPLATE_GRAPHQL_PATH process.env.DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING = 'extended' + process.env.DD_APPSEC_SCA_ENABLED = true + process.env.DD_EXPERIMENTAL_APPSEC_STANDALONE_ENABLED = 'true' process.env.DD_REMOTE_CONFIGURATION_ENABLED = 'false' process.env.DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS = '42' process.env.DD_IAST_ENABLED = 'true' process.env.DD_IAST_REQUEST_SAMPLING = '40' process.env.DD_IAST_MAX_CONCURRENT_REQUESTS = '3' process.env.DD_IAST_MAX_CONTEXT_OPERATIONS = '4' + process.env.DD_IAST_COOKIE_FILTER_PATTERN = '.*' process.env.DD_IAST_DEDUPLICATION_ENABLED = false process.env.DD_IAST_REDACTION_ENABLED = false process.env.DD_IAST_REDACTION_NAME_PATTERN = 'REDACTION_NAME_PATTERN' @@ -222,9 +492,20 @@ describe('Config', () => { process.env.DD_IAST_TELEMETRY_VERBOSITY = 'DEBUG' process.env.DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED = 'true' process.env.DD_TRACE_128_BIT_TRACEID_LOGGING_ENABLED = 'true' - process.env.DD_EXPERIMENTAL_PROFILING_ENABLED = 'true' - process.env.DD_EXPERIMENTAL_API_SECURITY_ENABLED = 'true' + process.env.DD_PROFILING_ENABLED = 'true' + process.env.DD_INJECTION_ENABLED = 'profiler' + process.env.DD_API_SECURITY_ENABLED = 'true' process.env.DD_API_SECURITY_REQUEST_SAMPLE_RATE = 1 + process.env.DD_INSTRUMENTATION_INSTALL_ID = '68e75c48-57ca-4a12-adfc-575c4b05fcbe' + process.env.DD_INSTRUMENTATION_INSTALL_TYPE = 'k8s_single_step' + process.env.DD_INSTRUMENTATION_INSTALL_TIME = '1703188212' + process.env.DD_INSTRUMENTATION_CONFIG_ID = 'abcdef123' + process.env.DD_TRACE_ENABLED = 'true' + process.env.DD_GRPC_CLIENT_ERROR_STATUSES = '3,13,400-403' + process.env.DD_GRPC_SERVER_ERROR_STATUSES = '3,13,400-403' + + // required if we want to check updates to config.debug and config.logLevel which is fetched from logger + reloadLoggerAndConfig() const config = new Config() @@ -239,17 +520,23 @@ describe('Config', () => { expect(config).to.have.property('queryStringObfuscation', '.*') expect(config).to.have.property('clientIpEnabled', true) expect(config).to.have.property('clientIpHeader', 'x-true-client-ip') + expect(config.grpc.client.error.statuses).to.deep.equal([3, 13, 400, 401, 402, 403]) + expect(config.grpc.server.error.statuses).to.deep.equal([3, 13, 400, 401, 402, 403]) expect(config).to.have.property('runtimeMetrics', true) expect(config).to.have.property('reportHostname', true) + expect(config).to.have.nested.property('codeOriginForSpans.enabled', true) + expect(config).to.have.property('dynamicInstrumentationEnabled', true) expect(config).to.have.property('env', 'test') expect(config).to.have.property('sampleRate', 0.5) + expect(config).to.have.property('traceEnabled', true) expect(config).to.have.property('traceId128BitGenerationEnabled', true) expect(config).to.have.property('traceId128BitLoggingEnabled', true) expect(config).to.have.property('spanAttributeSchema', 'v1') expect(config).to.have.property('spanRemoveIntegrationFromService', true) expect(config).to.have.property('spanComputePeerService', true) + expect(config).to.have.property('instrumentation_config_id', 'abcdef123') expect(config.tags).to.include({ foo: 'bar', baz: 'qux' }) - expect(config.tags).to.include({ service: 'service', 'version': '1.0.0', 'env': 'test' }) + expect(config.tags).to.include({ service: 'service', version: '1.0.0', env: 'test' }) expect(config).to.have.deep.nested.property('sampler', { sampleRate: 0.5, rateLimit: '-1', @@ -280,40 +567,129 @@ describe('Config', () => { expect(config).to.have.nested.property('experimental.exporter', 'log') expect(config).to.have.nested.property('experimental.enableGetRumData', true) expect(config).to.have.nested.property('appsec.enabled', true) + expect(config).to.have.nested.property('appsec.rasp.enabled', false) expect(config).to.have.nested.property('appsec.rules', RULES_JSON_PATH) - expect(config).to.have.nested.property('appsec.customRulesProvided', true) expect(config).to.have.nested.property('appsec.rateLimit', 42) + expect(config).to.have.nested.property('appsec.stackTrace.enabled', false) + expect(config).to.have.nested.property('appsec.stackTrace.maxDepth', 42) + expect(config).to.have.nested.property('appsec.stackTrace.maxStackTraces', 5) expect(config).to.have.nested.property('appsec.wafTimeout', 42) expect(config).to.have.nested.property('appsec.obfuscatorKeyRegex', '.*') expect(config).to.have.nested.property('appsec.obfuscatorValueRegex', '.*') expect(config).to.have.nested.property('appsec.blockedTemplateHtml', BLOCKED_TEMPLATE_HTML) expect(config).to.have.nested.property('appsec.blockedTemplateJson', BLOCKED_TEMPLATE_JSON) + expect(config).to.have.nested.property('appsec.blockedTemplateGraphql', BLOCKED_TEMPLATE_GRAPHQL) expect(config).to.have.nested.property('appsec.eventTracking.enabled', true) expect(config).to.have.nested.property('appsec.eventTracking.mode', 'extended') expect(config).to.have.nested.property('appsec.apiSecurity.enabled', true) expect(config).to.have.nested.property('appsec.apiSecurity.requestSampling', 1) + expect(config).to.have.nested.property('appsec.sca.enabled', true) + expect(config).to.have.nested.property('appsec.standalone.enabled', true) expect(config).to.have.nested.property('remoteConfig.enabled', false) expect(config).to.have.nested.property('remoteConfig.pollInterval', 42) expect(config).to.have.nested.property('iast.enabled', true) expect(config).to.have.nested.property('iast.requestSampling', 40) expect(config).to.have.nested.property('iast.maxConcurrentRequests', 3) expect(config).to.have.nested.property('iast.maxContextOperations', 4) + expect(config).to.have.nested.property('iast.cookieFilterPattern', '.*') expect(config).to.have.nested.property('iast.deduplicationEnabled', false) expect(config).to.have.nested.property('iast.redactionEnabled', false) expect(config).to.have.nested.property('iast.redactionNamePattern', 'REDACTION_NAME_PATTERN') expect(config).to.have.nested.property('iast.redactionValuePattern', 'REDACTION_VALUE_PATTERN') expect(config).to.have.nested.property('iast.telemetryVerbosity', 'DEBUG') + expect(config).to.have.deep.property('installSignature', { + id: '68e75c48-57ca-4a12-adfc-575c4b05fcbe', + type: 'k8s_single_step', + time: '1703188212' + }) + + expect(updateConfig).to.be.calledOnce + + expect(updateConfig.getCall(0).args[0]).to.deep.include.members([ + { name: 'appsec.blockedTemplateHtml', value: BLOCKED_TEMPLATE_HTML_PATH, origin: 'env_var' }, + { name: 'appsec.blockedTemplateJson', value: BLOCKED_TEMPLATE_JSON_PATH, origin: 'env_var' }, + { name: 'appsec.enabled', value: true, origin: 'env_var' }, + { name: 'appsec.obfuscatorKeyRegex', value: '.*', origin: 'env_var' }, + { name: 'appsec.obfuscatorValueRegex', value: '.*', origin: 'env_var' }, + { name: 'appsec.rateLimit', value: '42', origin: 'env_var' }, + { name: 'appsec.rasp.enabled', value: false, origin: 'env_var' }, + { name: 'appsec.rules', value: RULES_JSON_PATH, origin: 'env_var' }, + { name: 'appsec.stackTrace.enabled', value: false, origin: 'env_var' }, + { name: 'appsec.stackTrace.maxDepth', value: '42', origin: 'env_var' }, + { name: 'appsec.stackTrace.maxStackTraces', value: '5', origin: 'env_var' }, + { name: 'appsec.sca.enabled', value: true, origin: 'env_var' }, + { name: 'appsec.standalone.enabled', value: true, origin: 'env_var' }, + { name: 'appsec.wafTimeout', value: '42', origin: 'env_var' }, + { name: 'clientIpEnabled', value: true, origin: 'env_var' }, + { name: 'clientIpHeader', value: 'x-true-client-ip', origin: 'env_var' }, + { name: 'codeOriginForSpans.enabled', value: true, origin: 'env_var' }, + { name: 'dogstatsd.hostname', value: 'dsd-agent', origin: 'env_var' }, + { name: 'dogstatsd.port', value: '5218', origin: 'env_var' }, + { name: 'dynamicInstrumentationEnabled', value: true, origin: 'env_var' }, + { name: 'env', value: 'test', origin: 'env_var' }, + { name: 'experimental.enableGetRumData', value: true, origin: 'env_var' }, + { name: 'experimental.exporter', value: 'log', origin: 'env_var' }, + { name: 'experimental.runtimeId', value: true, origin: 'env_var' }, + { name: 'hostname', value: 'agent', origin: 'env_var' }, + { name: 'iast.cookieFilterPattern', value: '.*', origin: 'env_var' }, + { name: 'iast.deduplicationEnabled', value: false, origin: 'env_var' }, + { name: 'iast.enabled', value: true, origin: 'env_var' }, + { name: 'iast.maxConcurrentRequests', value: '3', origin: 'env_var' }, + { name: 'iast.maxContextOperations', value: '4', origin: 'env_var' }, + { name: 'iast.redactionEnabled', value: false, origin: 'env_var' }, + { name: 'iast.redactionNamePattern', value: 'REDACTION_NAME_PATTERN', origin: 'env_var' }, + { name: 'iast.redactionValuePattern', value: 'REDACTION_VALUE_PATTERN', origin: 'env_var' }, + { name: 'iast.requestSampling', value: '40', origin: 'env_var' }, + { name: 'iast.telemetryVerbosity', value: 'DEBUG', origin: 'env_var' }, + { name: 'instrumentation_config_id', value: 'abcdef123', origin: 'env_var' }, + { name: 'injectionEnabled', value: ['profiler'], origin: 'env_var' }, + { name: 'isGCPFunction', value: false, origin: 'env_var' }, + { name: 'peerServiceMapping', value: process.env.DD_TRACE_PEER_SERVICE_MAPPING, origin: 'env_var' }, + { name: 'port', value: '6218', origin: 'env_var' }, + { name: 'profiling.enabled', value: 'true', origin: 'env_var' }, + { name: 'protocolVersion', value: '0.5', origin: 'env_var' }, + { name: 'queryStringObfuscation', value: '.*', origin: 'env_var' }, + { name: 'remoteConfig.enabled', value: false, origin: 'env_var' }, + { name: 'remoteConfig.pollInterval', value: '42', origin: 'env_var' }, + { name: 'reportHostname', value: true, origin: 'env_var' }, + { name: 'runtimeMetrics', value: true, origin: 'env_var' }, + { name: 'sampleRate', value: 0.5, origin: 'env_var' }, + { name: 'sampler.rateLimit', value: '-1', origin: 'env_var' }, + { + name: 'sampler.rules', + value: process.env.DD_TRACE_SAMPLING_RULES, + origin: 'env_var' + }, + { name: 'service', value: 'service', origin: 'env_var' }, + { name: 'spanAttributeSchema', value: 'v1', origin: 'env_var' }, + { name: 'spanRemoveIntegrationFromService', value: true, origin: 'env_var' }, + { name: 'telemetry.enabled', value: true, origin: 'env_var' }, + { name: 'traceId128BitGenerationEnabled', value: true, origin: 'env_var' }, + { name: 'traceId128BitLoggingEnabled', value: true, origin: 'env_var' }, + { name: 'tracing', value: false, origin: 'env_var' }, + { name: 'version', value: '1.0.0', origin: 'env_var' } + ]) + }) + + it('should ignore empty strings', () => { + process.env.DD_TAGS = 'service:,env:,version:' + + const config = new Config() + + expect(config).to.have.property('service', 'node') + expect(config).to.have.property('env', undefined) + expect(config).to.have.property('version', '') }) it('should read case-insensitive booleans from environment variables', () => { process.env.DD_TRACING_ENABLED = 'False' - process.env.DD_TRACE_DEBUG = 'TRUE' + process.env.DD_TRACE_PROPAGATION_EXTRACT_FIRST = 'TRUE' process.env.DD_RUNTIME_METRICS_ENABLED = '0' const config = new Config() expect(config).to.have.property('tracing', false) - expect(config).to.have.property('debug', true) + expect(config).to.have.property('tracePropagationExtractFirst', true) expect(config).to.have.property('runtimeMetrics', false) }) @@ -323,14 +699,12 @@ describe('Config', () => { process.env.DD_TRACE_AGENT_HOSTNAME = 'agent' process.env.DD_TRACE_AGENT_PORT = '6218' process.env.DD_TRACING_ENABLED = 'false' - process.env.DD_TRACE_DEBUG = 'true' process.env.DD_SERVICE = 'service' process.env.DD_ENV = 'test' const config = new Config() expect(config).to.have.property('tracing', false) - expect(config).to.have.property('debug', true) expect(config).to.have.nested.property('dogstatsd.hostname', 'agent') expect(config).to.have.nested.property('url.protocol', 'https:') expect(config).to.have.nested.property('url.hostname', 'agent2') @@ -354,9 +728,15 @@ describe('Config', () => { it('should initialize from the options', () => { const logger = {} const tags = { - 'foo': 'bar' + foo: 'bar' } const logLevel = 'error' + const samplingRules = [ + { service: 'usersvc', name: 'healthcheck', sampleRate: 0.0 }, + { service: 'usersvc', sampleRate: 0.5 }, + { service: 'authsvc', sampleRate: 1.0 }, + { sampleRate: 0.1 } + ] const config = new Config({ enabled: false, debug: true, @@ -373,14 +753,12 @@ describe('Config', () => { env: 'test', clientIpEnabled: true, clientIpHeader: 'x-true-client-ip', + codeOriginForSpans: { + enabled: false + }, sampleRate: 0.5, rateLimit: 1000, - samplingRules: [ - { service: 'usersvc', name: 'healthcheck', sampleRate: 0.0 }, - { service: 'usersvc', sampleRate: 0.5 }, - { service: 'authsvc', sampleRate: 1.0 }, - { sampleRate: 0.1 } - ], + samplingRules, spanSamplingRules: [ { service: 'mysql', name: 'mysql.query', sampleRate: 0.0, maxPerSecond: 1 }, { service: 'mysql', sampleRate: 0.5 }, @@ -404,13 +782,14 @@ describe('Config', () => { runtimeMetrics: true, reportHostname: true, plugins: false, - logLevel: logLevel, + logLevel, tracePropagationStyle: { inject: ['datadog'], extract: ['datadog'] }, experimental: { b3: true, + dynamicInstrumentationEnabled: true, traceparent: true, runtimeId: true, exporter: 'log', @@ -420,11 +799,17 @@ describe('Config', () => { requestSampling: 50, maxConcurrentRequests: 4, maxContextOperations: 5, + cookieFilterPattern: '.*', deduplicationEnabled: false, redactionEnabled: false, redactionNamePattern: 'REDACTION_NAME_PATTERN', redactionValuePattern: 'REDACTION_VALUE_PATTERN', telemetryVerbosity: 'DEBUG' + }, + appsec: { + standalone: { + enabled: true + } } }, appsec: false, @@ -443,6 +828,7 @@ describe('Config', () => { expect(config).to.have.nested.property('dogstatsd.port', '5218') expect(config).to.have.property('service', 'service') expect(config).to.have.property('version', '0.1.0') + expect(config).to.have.property('dynamicInstrumentationEnabled', true) expect(config).to.have.property('env', 'test') expect(config).to.have.property('sampleRate', 0.5) expect(config).to.have.property('logger', logger) @@ -458,6 +844,7 @@ describe('Config', () => { expect(config).to.have.property('reportHostname', true) expect(config).to.have.property('plugins', false) expect(config).to.have.property('logLevel', logLevel) + expect(config).to.have.nested.property('codeOriginForSpans.enabled', false) expect(config).to.have.property('traceId128BitGenerationEnabled', true) expect(config).to.have.property('traceId128BitLoggingEnabled', true) expect(config).to.have.property('spanRemoveIntegrationFromService', true) @@ -473,11 +860,13 @@ describe('Config', () => { expect(config).to.have.nested.property('experimental.exporter', 'log') expect(config).to.have.nested.property('experimental.enableGetRumData', true) expect(config).to.have.nested.property('appsec.enabled', false) + expect(config).to.have.nested.property('appsec.standalone.enabled', true) expect(config).to.have.nested.property('remoteConfig.pollInterval', 42) expect(config).to.have.nested.property('iast.enabled', true) expect(config).to.have.nested.property('iast.requestSampling', 50) expect(config).to.have.nested.property('iast.maxConcurrentRequests', 4) expect(config).to.have.nested.property('iast.maxContextOperations', 5) + expect(config).to.have.nested.property('iast.cookieFilterPattern', '.*') expect(config).to.have.nested.property('iast.deduplicationEnabled', false) expect(config).to.have.nested.property('iast.redactionEnabled', false) expect(config).to.have.nested.property('iast.redactionNamePattern', 'REDACTION_NAME_PATTERN') @@ -503,6 +892,55 @@ describe('Config', () => { a: 'aa', b: 'bb' }) + + expect(updateConfig).to.be.calledOnce + + expect(updateConfig.getCall(0).args[0]).to.deep.include.members([ + { name: 'appsec.enabled', value: false, origin: 'code' }, + { name: 'appsec.standalone.enabled', value: true, origin: 'code' }, + { name: 'clientIpEnabled', value: true, origin: 'code' }, + { name: 'clientIpHeader', value: 'x-true-client-ip', origin: 'code' }, + { name: 'codeOriginForSpans.enabled', value: false, origin: 'code' }, + { name: 'dogstatsd.hostname', value: 'agent-dsd', origin: 'code' }, + { name: 'dogstatsd.port', value: '5218', origin: 'code' }, + { name: 'dynamicInstrumentationEnabled', value: true, origin: 'code' }, + { name: 'env', value: 'test', origin: 'code' }, + { name: 'experimental.enableGetRumData', value: true, origin: 'code' }, + { name: 'experimental.exporter', value: 'log', origin: 'code' }, + { name: 'experimental.runtimeId', value: true, origin: 'code' }, + { name: 'flushInterval', value: 5000, origin: 'code' }, + { name: 'flushMinSpans', value: 500, origin: 'code' }, + { name: 'hostname', value: 'agent', origin: 'code' }, + { name: 'iast.cookieFilterPattern', value: '.*', origin: 'code' }, + { name: 'iast.deduplicationEnabled', value: false, origin: 'code' }, + { name: 'iast.enabled', value: true, origin: 'code' }, + { name: 'iast.maxConcurrentRequests', value: 4, origin: 'code' }, + { name: 'iast.maxContextOperations', value: 5, origin: 'code' }, + { name: 'iast.redactionEnabled', value: false, origin: 'code' }, + { name: 'iast.redactionNamePattern', value: 'REDACTION_NAME_PATTERN', origin: 'code' }, + { name: 'iast.redactionValuePattern', value: 'REDACTION_VALUE_PATTERN', origin: 'code' }, + { name: 'iast.requestSampling', value: 50, origin: 'code' }, + { name: 'iast.telemetryVerbosity', value: 'DEBUG', origin: 'code' }, + { name: 'peerServiceMapping', value: { d: 'dd' }, origin: 'code' }, + { name: 'plugins', value: false, origin: 'code' }, + { name: 'port', value: '6218', origin: 'code' }, + { name: 'protocolVersion', value: '0.5', origin: 'code' }, + { name: 'remoteConfig.pollInterval', value: 42, origin: 'code' }, + { name: 'reportHostname', value: true, origin: 'code' }, + { name: 'runtimeMetrics', value: true, origin: 'code' }, + { name: 'sampleRate', value: 0.5, origin: 'code' }, + { name: 'sampler.rateLimit', value: 1000, origin: 'code' }, + { name: 'sampler.rules', value: samplingRules, origin: 'code' }, + { name: 'service', value: 'service', origin: 'code' }, + { name: 'site', value: 'datadoghq.eu', origin: 'code' }, + { name: 'spanAttributeSchema', value: 'v1', origin: 'code' }, + { name: 'spanComputePeerService', value: true, origin: 'calculated' }, + { name: 'spanRemoveIntegrationFromService', value: true, origin: 'code' }, + { name: 'stats.enabled', value: false, origin: 'calculated' }, + { name: 'traceId128BitGenerationEnabled', value: true, origin: 'code' }, + { name: 'traceId128BitLoggingEnabled', value: true, origin: 'code' }, + { name: 'version', value: '0.1.0', origin: 'code' } + ]) }) it('should initialize from the options with url taking precedence', () => { @@ -570,6 +1008,32 @@ describe('Config', () => { expect(config).to.have.property('spanAttributeSchema', 'v0') }) + it('should parse integer range sets', () => { + process.env.DD_GRPC_CLIENT_ERROR_STATUSES = '3,13,400-403' + process.env.DD_GRPC_SERVER_ERROR_STATUSES = '3,13,400-403' + + let config = new Config() + + expect(config.grpc.client.error.statuses).to.deep.equal([3, 13, 400, 401, 402, 403]) + expect(config.grpc.server.error.statuses).to.deep.equal([3, 13, 400, 401, 402, 403]) + + process.env.DD_GRPC_CLIENT_ERROR_STATUSES = '1' + process.env.DD_GRPC_SERVER_ERROR_STATUSES = '1' + + config = new Config() + + expect(config.grpc.client.error.statuses).to.deep.equal([1]) + expect(config.grpc.server.error.statuses).to.deep.equal([1]) + + process.env.DD_GRPC_CLIENT_ERROR_STATUSES = '2,10,13-15' + process.env.DD_GRPC_SERVER_ERROR_STATUSES = '2,10,13-15' + + config = new Config() + + expect(config.grpc.client.error.statuses).to.deep.equal([2, 10, 13, 14, 15]) + expect(config.grpc.server.error.statuses).to.deep.equal([2, 10, 13, 14, 15]) + }) + context('peer service tagging', () => { it('should activate peer service only if explicitly true in v0', () => { process.env.DD_TRACE_SPAN_ATTRIBUTE_SCHEMA = 'v0' @@ -637,6 +1101,7 @@ describe('Config', () => { process.env.DD_RUNTIME_METRICS_ENABLED = 'true' process.env.DD_TRACE_REPORT_HOSTNAME = 'true' process.env.DD_ENV = 'test' + process.env.DD_DYNAMIC_INSTRUMENTATION_ENABLED = 'true' process.env.DD_API_KEY = '123' process.env.DD_TRACE_SPAN_ATTRIBUTE_SCHEMA = 'v0' process.env.DD_TRACE_PEER_SERVICE_DEFAULTS_ENABLED = 'false' @@ -653,18 +1118,24 @@ describe('Config', () => { process.env.DD_TRACE_EXPERIMENTAL_GET_RUM_DATA_ENABLED = 'true' process.env.DD_TRACE_EXPERIMENTAL_INTERNAL_ERRORS_ENABLED = 'true' process.env.DD_APPSEC_ENABLED = 'false' + process.env.DD_APPSEC_MAX_STACK_TRACES = '11' + process.env.DD_APPSEC_MAX_STACK_TRACE_DEPTH = '11' + process.env.DD_APPSEC_RASP_ENABLED = 'true' process.env.DD_APPSEC_RULES = RECOMMENDED_JSON_PATH + process.env.DD_APPSEC_STACK_TRACE_ENABLED = 'true' process.env.DD_APPSEC_TRACE_RATE_LIMIT = 11 process.env.DD_APPSEC_WAF_TIMEOUT = 11 process.env.DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP = '^$' process.env.DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP = '^$' - process.env.DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML = BLOCKED_TEMPLATE_JSON // note the inversion between - process.env.DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON = BLOCKED_TEMPLATE_HTML // json and html here + process.env.DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML = BLOCKED_TEMPLATE_JSON_PATH // note the inversion between + process.env.DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON = BLOCKED_TEMPLATE_HTML_PATH // json and html here + process.env.DD_APPSEC_GRAPHQL_BLOCKED_TEMPLATE_JSON = BLOCKED_TEMPLATE_JSON_PATH // json and html here process.env.DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING = 'disabled' - process.env.DD_EXPERIMENTAL_API_SECURITY_ENABLED = 'false' + process.env.DD_API_SECURITY_ENABLED = 'false' process.env.DD_API_SECURITY_REQUEST_SAMPLE_RATE = 0.5 process.env.DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS = 11 process.env.DD_IAST_ENABLED = 'false' + process.env.DD_IAST_COOKIE_FILTER_PATTERN = '.*' process.env.DD_IAST_REDACTION_NAME_PATTERN = 'name_pattern_to_be_overriden_by_options' process.env.DD_IAST_REDACTION_VALUE_PATTERN = 'value_pattern_to_be_overriden_by_options' process.env.DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED = 'true' @@ -705,15 +1176,11 @@ describe('Config', () => { }, experimental: { b3: false, + dynamicInstrumentationEnabled: false, traceparent: false, runtimeId: false, exporter: 'agent', - enableGetRumData: false, - iast: { - enabled: true, - redactionNamePattern: 'REDACTION_NAME_PATTERN', - redactionValuePattern: 'REDACTION_VALUE_PATTERN' - } + enableGetRumData: false }, appsec: { enabled: true, @@ -724,17 +1191,35 @@ describe('Config', () => { obfuscatorValueRegex: '.*', blockedTemplateHtml: BLOCKED_TEMPLATE_HTML_PATH, blockedTemplateJson: BLOCKED_TEMPLATE_JSON_PATH, + blockedTemplateGraphql: BLOCKED_TEMPLATE_GRAPHQL_PATH, eventTracking: { mode: 'safe' }, apiSecurity: { enabled: true, requestSampling: 1.0 + }, + rasp: { + enabled: false + }, + stackTrace: { + enabled: false, + maxDepth: 42, + maxStackTraces: 5 } }, + iast: { + enabled: true, + cookieFilterPattern: '.{10,}', + redactionNamePattern: 'REDACTION_NAME_PATTERN', + redactionValuePattern: 'REDACTION_VALUE_PATTERN' + }, remoteConfig: { pollInterval: 42 }, + codeOriginForSpans: { + enabled: false + }, traceId128BitGenerationEnabled: false, traceId128BitLoggingEnabled: false }) @@ -751,12 +1236,14 @@ describe('Config', () => { expect(config).to.have.property('flushMinSpans', 500) expect(config).to.have.property('service', 'test') expect(config).to.have.property('version', '1.0.0') + expect(config).to.have.nested.property('codeOriginForSpans.enabled', false) + expect(config).to.have.property('dynamicInstrumentationEnabled', false) expect(config).to.have.property('env', 'development') expect(config).to.have.property('clientIpEnabled', true) expect(config).to.have.property('clientIpHeader', 'x-true-client-ip') expect(config).to.have.property('traceId128BitGenerationEnabled', false) expect(config).to.have.property('traceId128BitLoggingEnabled', false) - expect(config.tags).to.include({ foo: 'foo', baz: 'qux' }) + expect(config.tags).to.include({ foo: 'foo' }) expect(config.tags).to.include({ service: 'test', version: '1.0.0', env: 'development' }) expect(config).to.have.deep.property('serviceMapping', { b: 'bb' }) expect(config).to.have.property('spanAttributeSchema', 'v1') @@ -769,14 +1256,18 @@ describe('Config', () => { expect(config).to.have.nested.property('experimental.exporter', 'agent') expect(config).to.have.nested.property('experimental.enableGetRumData', false) expect(config).to.have.nested.property('appsec.enabled', true) + expect(config).to.have.nested.property('appsec.rasp.enabled', false) expect(config).to.have.nested.property('appsec.rules', RULES_JSON_PATH) - expect(config).to.have.nested.property('appsec.customRulesProvided', true) expect(config).to.have.nested.property('appsec.rateLimit', 42) + expect(config).to.have.nested.property('appsec.stackTrace.enabled', false) + expect(config).to.have.nested.property('appsec.stackTrace.maxDepth', 42) + expect(config).to.have.nested.property('appsec.stackTrace.maxStackTraces', 5) expect(config).to.have.nested.property('appsec.wafTimeout', 42) expect(config).to.have.nested.property('appsec.obfuscatorKeyRegex', '.*') expect(config).to.have.nested.property('appsec.obfuscatorValueRegex', '.*') expect(config).to.have.nested.property('appsec.blockedTemplateHtml', BLOCKED_TEMPLATE_HTML) expect(config).to.have.nested.property('appsec.blockedTemplateJson', BLOCKED_TEMPLATE_JSON) + expect(config).to.have.nested.property('appsec.blockedTemplateGraphql', BLOCKED_TEMPLATE_GRAPHQL) expect(config).to.have.nested.property('appsec.eventTracking.enabled', true) expect(config).to.have.nested.property('appsec.eventTracking.mode', 'safe') expect(config).to.have.nested.property('appsec.apiSecurity.enabled', true) @@ -787,6 +1278,7 @@ describe('Config', () => { expect(config).to.have.nested.property('iast.maxConcurrentRequests', 2) expect(config).to.have.nested.property('iast.maxContextOperations', 2) expect(config).to.have.nested.property('iast.deduplicationEnabled', true) + expect(config).to.have.nested.property('iast.cookieFilterPattern', '.{10,}') expect(config).to.have.nested.property('iast.redactionEnabled', true) expect(config).to.have.nested.property('iast.redactionNamePattern', 'REDACTION_NAME_PATTERN') expect(config).to.have.nested.property('iast.redactionValuePattern', 'REDACTION_VALUE_PATTERN') @@ -803,14 +1295,30 @@ describe('Config', () => { obfuscatorValueRegex: '.*', blockedTemplateHtml: undefined, blockedTemplateJson: undefined, + blockedTemplateGraphql: undefined, eventTracking: { mode: 'disabled' }, apiSecurity: { enabled: true, requestSampling: 1.0 + }, + rasp: { + enabled: false } }, + iast: { + enabled: true, + requestSampling: 15, + maxConcurrentRequests: 3, + maxContextOperations: 4, + cookieFilterPattern: '.*', + deduplicationEnabled: false, + redactionEnabled: false, + redactionNamePattern: 'REDACTION_NAME_PATTERN', + redactionValuePattern: 'REDACTION_VALUE_PATTERN', + telemetryVerbosity: 'DEBUG' + }, experimental: { appsec: { enabled: false, @@ -821,13 +1329,29 @@ describe('Config', () => { obfuscatorValueRegex: '^$', blockedTemplateHtml: BLOCKED_TEMPLATE_HTML_PATH, blockedTemplateJson: BLOCKED_TEMPLATE_JSON_PATH, + blockedTemplateGraphql: BLOCKED_TEMPLATE_GRAPHQL_PATH, eventTracking: { mode: 'safe' }, apiSecurity: { enabled: false, requestSampling: 0.5 + }, + rasp: { + enabled: true } + }, + iast: { + enabled: false, + requestSampling: 25, + maxConcurrentRequests: 6, + maxContextOperations: 7, + cookieFilterPattern: '.{10,}', + deduplicationEnabled: true, + redactionEnabled: true, + redactionNamePattern: 'IGNORED_REDACTION_NAME_PATTERN', + redactionValuePattern: 'IGNORED_REDACTION_VALUE_PATTERN', + telemetryVerbosity: 'OFF' } } }) @@ -835,13 +1359,13 @@ describe('Config', () => { expect(config).to.have.deep.property('appsec', { enabled: true, rules: undefined, - customRulesProvided: false, rateLimit: 42, wafTimeout: 42, obfuscatorKeyRegex: '.*', obfuscatorValueRegex: '.*', blockedTemplateHtml: undefined, blockedTemplateJson: undefined, + blockedTemplateGraphql: undefined, eventTracking: { enabled: false, mode: 'disabled' @@ -849,8 +1373,35 @@ describe('Config', () => { apiSecurity: { enabled: true, requestSampling: 1.0 + }, + sca: { + enabled: null + }, + rasp: { + enabled: false + }, + standalone: { + enabled: undefined + }, + stackTrace: { + enabled: true, + maxStackTraces: 2, + maxDepth: 32 } }) + + expect(config).to.have.deep.property('iast', { + enabled: true, + requestSampling: 15, + maxConcurrentRequests: 3, + maxContextOperations: 4, + cookieFilterPattern: '.*', + deduplicationEnabled: false, + redactionEnabled: false, + redactionNamePattern: 'REDACTION_NAME_PATTERN', + redactionValuePattern: 'REDACTION_VALUE_PATTERN', + telemetryVerbosity: 'DEBUG' + }) }) it('should give priority to the options especially url', () => { @@ -1081,6 +1632,67 @@ describe('Config', () => { expect(config.remoteConfig.enabled).to.be.false }) + it('should send empty array when remote config is called on empty options', () => { + const config = new Config() + + config.configure({}, true) + + expect(updateConfig).to.be.calledTwice + expect(updateConfig.getCall(1).args[0]).to.deep.equal([]) + }) + + it('should send remote config changes to telemetry', () => { + const config = new Config() + + config.configure({ + tracing_sampling_rate: 0 + }, true) + + expect(updateConfig.getCall(1).args[0]).to.deep.equal([ + { name: 'sampleRate', value: 0, origin: 'remote_config' } + ]) + }) + + it('should reformat tags from sampling rules when set through remote configuration', () => { + const config = new Config() + + config.configure({ + tracing_sampling_rules: [ + { + resource: '*', + tags: [ + { key: 'tag-a', value_glob: 'tag-a-val*' }, + { key: 'tag-b', value_glob: 'tag-b-val*' } + ], + provenance: 'customer' + } + ] + }, true) + expect(config).to.have.deep.nested.property('sampler', { + spanSamplingRules: [], + rateLimit: 100, + rules: [ + { + resource: '*', + tags: { 'tag-a': 'tag-a-val*', 'tag-b': 'tag-b-val*' }, + provenance: 'customer' + } + ], + sampleRate: undefined + }) + }) + + it('should have consistent runtime-id after remote configuration updates tags', () => { + const config = new Config() + const runtimeId = config.tags['runtime-id'] + config.configure({ + tracing_tags: { foo: 'bar' } + }, true) + + expect(config.tags).to.have.property('foo', 'bar') + expect(config.tags).to.have.property('runtime-id', runtimeId) + }) + it('should ignore invalid iast.requestSampling', () => { const config = new Config({ experimental: { @@ -1122,19 +1734,46 @@ describe('Config', () => { enabled: true, rules: 'path/to/rules.json', blockedTemplateHtml: 'DOES_NOT_EXIST.html', - blockedTemplateJson: 'DOES_NOT_EXIST.json' + blockedTemplateJson: 'DOES_NOT_EXIST.json', + blockedTemplateGraphql: 'DOES_NOT_EXIST.json' } }) - expect(log.error).to.be.callCount(2) + expect(log.error).to.be.callCount(3) expect(log.error.firstCall).to.have.been.calledWithExactly(error) expect(log.error.secondCall).to.have.been.calledWithExactly(error) + expect(log.error.thirdCall).to.have.been.calledWithExactly(error) expect(config.appsec.enabled).to.be.true expect(config.appsec.rules).to.eq('path/to/rules.json') - expect(config.appsec.customRulesProvided).to.be.true expect(config.appsec.blockedTemplateHtml).to.be.undefined expect(config.appsec.blockedTemplateJson).to.be.undefined + expect(config.appsec.blockedTemplateGraphql).to.be.undefined + }) + + it('should enable api security with DD_EXPERIMENTAL_API_SECURITY_ENABLED', () => { + process.env.DD_EXPERIMENTAL_API_SECURITY_ENABLED = 'true' + + const config = new Config() + + expect(config.appsec.apiSecurity.enabled).to.be.true + }) + + it('should disable api security with DD_EXPERIMENTAL_API_SECURITY_ENABLED', () => { + process.env.DD_EXPERIMENTAL_API_SECURITY_ENABLED = 'false' + + const config = new Config() + + expect(config.appsec.apiSecurity.enabled).to.be.false + }) + + it('should ignore DD_EXPERIMENTAL_API_SECURITY_ENABLED with DD_API_SECURITY_ENABLED=true', () => { + process.env.DD_EXPERIMENTAL_API_SECURITY_ENABLED = 'false' + process.env.DD_API_SECURITY_ENABLED = 'true' + + const config = new Config() + + expect(config.appsec.apiSecurity.enabled).to.be.true }) context('auto configuration w/ unix domain sockets', () => { @@ -1231,6 +1870,11 @@ describe('Config', () => { delete process.env.DD_CIVISIBILITY_ITR_ENABLED delete process.env.DD_CIVISIBILITY_GIT_UPLOAD_ENABLED delete process.env.DD_CIVISIBILITY_MANUAL_API_ENABLED + delete process.env.DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED + delete process.env.DD_CIVISIBILITY_FLAKY_RETRY_ENABLED + delete process.env.DD_CIVISIBILITY_FLAKY_RETRY_COUNT + delete process.env.DD_TEST_SESSION_NAME + delete process.env.JEST_WORKER_ID options = {} }) context('ci visibility mode is enabled', () => { @@ -1255,14 +1899,14 @@ describe('Config', () => { const config = new Config(options) expect(config).to.have.property('isIntelligentTestRunnerEnabled', false) }) - it('should disable manual testing API by default', () => { + it('should enable manual testing API by default', () => { const config = new Config(options) - expect(config).to.have.property('isManualApiEnabled', false) + expect(config).to.have.property('isManualApiEnabled', true) }) - it('should enable manual testing API if DD_CIVISIBILITY_MANUAL_API_ENABLED is passed', () => { - process.env.DD_CIVISIBILITY_MANUAL_API_ENABLED = 'true' + it('should disable manual testing API if DD_CIVISIBILITY_MANUAL_API_ENABLED is set to false', () => { + process.env.DD_CIVISIBILITY_MANUAL_API_ENABLED = 'false' const config = new Config(options) - expect(config).to.have.property('isManualApiEnabled', true) + expect(config).to.have.property('isManualApiEnabled', false) }) it('should disable memcached command tagging by default', () => { const config = new Config(options) @@ -1273,6 +1917,52 @@ describe('Config', () => { const config = new Config(options) expect(config).to.have.property('memcachedCommandEnabled', true) }) + it('should enable telemetry', () => { + const config = new Config(options) + expect(config).to.nested.property('telemetry.enabled', true) + }) + it('should enable early flake detection by default', () => { + const config = new Config(options) + expect(config).to.have.property('isEarlyFlakeDetectionEnabled', true) + }) + it('should disable early flake detection if DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED is false', () => { + process.env.DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED = 'false' + const config = new Config(options) + expect(config).to.have.property('isEarlyFlakeDetectionEnabled', false) + }) + it('should enable flaky test retries by default', () => { + const config = new Config(options) + expect(config).to.have.property('isFlakyTestRetriesEnabled', true) + }) + it('should disable flaky test retries if isFlakyTestRetriesEnabled is false', () => { + process.env.DD_CIVISIBILITY_FLAKY_RETRY_ENABLED = 'false' + const config = new Config(options) + expect(config).to.have.property('isFlakyTestRetriesEnabled', false) + }) + it('should read DD_CIVISIBILITY_FLAKY_RETRY_COUNT if present', () => { + process.env.DD_CIVISIBILITY_FLAKY_RETRY_COUNT = '4' + const config = new Config(options) + expect(config).to.have.property('flakyTestRetriesCount', 4) + }) + it('should default DD_CIVISIBILITY_FLAKY_RETRY_COUNT to 5', () => { + const config = new Config(options) + expect(config).to.have.property('flakyTestRetriesCount', 5) + }) + it('should round non integer values of DD_CIVISIBILITY_FLAKY_RETRY_COUNT', () => { + process.env.DD_CIVISIBILITY_FLAKY_RETRY_COUNT = '4.1' + const config = new Config(options) + expect(config).to.have.property('flakyTestRetriesCount', 4) + }) + it('should set the default to DD_CIVISIBILITY_FLAKY_RETRY_COUNT if it is not a number', () => { + process.env.DD_CIVISIBILITY_FLAKY_RETRY_COUNT = 'a' + const config = new Config(options) + expect(config).to.have.property('flakyTestRetriesCount', 5) + }) + it('should set the session name if DD_TEST_SESSION_NAME is set', () => { + process.env.DD_TEST_SESSION_NAME = 'my-test-session' + const config = new Config(options) + expect(config).to.have.property('ciVisibilityTestSessionName', 'my-test-session') + }) }) context('ci visibility mode is not enabled', () => { it('should not activate intelligent test runner or git metadata upload', () => { @@ -1283,6 +1973,11 @@ describe('Config', () => { expect(config).to.have.property('isGitUploadEnabled', false) }) }) + it('disables telemetry if inside a jest worker', () => { + process.env.JEST_WORKER_ID = '1' + const config = new Config(options) + expect(config.telemetry.enabled).to.be.false + }) }) context('sci embedding', () => { @@ -1358,6 +2053,7 @@ describe('Config', () => { expect(config).not.to.have.property('repositoryUrl') }) }) + it('should sanitize values for API Security sampling between 0 and 1', () => { expect(new Config({ appsec: { @@ -1386,4 +2082,83 @@ describe('Config', () => { } })).to.have.nested.property('appsec.apiSecurity.requestSampling', 0.1) }) + + context('payload tagging', () => { + let env + + const staticConfig = require('../src/payload-tagging/config/aws') + + beforeEach(() => { + env = process.env + }) + + afterEach(() => { + process.env = env + }) + + it('defaults', () => { + const taggingConfig = new Config().cloudPayloadTagging + expect(taggingConfig).to.have.property('requestsEnabled', false) + expect(taggingConfig).to.have.property('responsesEnabled', false) + expect(taggingConfig).to.have.property('maxDepth', 10) + }) + + it('enabling requests with no additional filter', () => { + process.env.DD_TRACE_CLOUD_REQUEST_PAYLOAD_TAGGING = 'all' + const taggingConfig = new Config().cloudPayloadTagging + expect(taggingConfig).to.have.property('requestsEnabled', true) + expect(taggingConfig).to.have.property('responsesEnabled', false) + expect(taggingConfig).to.have.property('maxDepth', 10) + const awsRules = taggingConfig.rules.aws + for (const [serviceName, service] of Object.entries(awsRules)) { + expect(service.request).to.deep.equal(staticConfig[serviceName].request) + } + }) + + it('enabling requests with an additional filter', () => { + process.env.DD_TRACE_CLOUD_REQUEST_PAYLOAD_TAGGING = '$.foo.bar' + const taggingConfig = new Config().cloudPayloadTagging + expect(taggingConfig).to.have.property('requestsEnabled', true) + expect(taggingConfig).to.have.property('responsesEnabled', false) + expect(taggingConfig).to.have.property('maxDepth', 10) + const awsRules = taggingConfig.rules.aws + for (const [, service] of Object.entries(awsRules)) { + expect(service.request).to.include('$.foo.bar') + } + }) + + it('enabling responses with no additional filter', () => { + process.env.DD_TRACE_CLOUD_RESPONSE_PAYLOAD_TAGGING = 'all' + const taggingConfig = new Config().cloudPayloadTagging + expect(taggingConfig).to.have.property('requestsEnabled', false) + expect(taggingConfig).to.have.property('responsesEnabled', true) + expect(taggingConfig).to.have.property('maxDepth', 10) + const awsRules = taggingConfig.rules.aws + for (const [serviceName, service] of Object.entries(awsRules)) { + expect(service.response).to.deep.equal(staticConfig[serviceName].response) + } + }) + + it('enabling responses with an additional filter', () => { + process.env.DD_TRACE_CLOUD_RESPONSE_PAYLOAD_TAGGING = '$.foo.bar' + const taggingConfig = new Config().cloudPayloadTagging + expect(taggingConfig).to.have.property('requestsEnabled', false) + expect(taggingConfig).to.have.property('responsesEnabled', true) + expect(taggingConfig).to.have.property('maxDepth', 10) + const awsRules = taggingConfig.rules.aws + for (const [, service] of Object.entries(awsRules)) { + expect(service.response).to.include('$.foo.bar') + } + }) + + it('overriding max depth', () => { + process.env.DD_TRACE_CLOUD_REQUEST_PAYLOAD_TAGGING = 'all' + process.env.DD_TRACE_CLOUD_RESPONSE_PAYLOAD_TAGGING = 'all' + process.env.DD_TRACE_CLOUD_PAYLOAD_TAGGING_MAX_DEPTH = 7 + const taggingConfig = new Config().cloudPayloadTagging + expect(taggingConfig).to.have.property('requestsEnabled', true) + expect(taggingConfig).to.have.property('responsesEnabled', true) + expect(taggingConfig).to.have.property('maxDepth', 7) + }) + }) }) diff --git a/packages/dd-trace/test/config/disabled_instrumentations.spec.js b/packages/dd-trace/test/config/disabled_instrumentations.spec.js index d54ee38f677..c7f9b935fb5 100644 --- a/packages/dd-trace/test/config/disabled_instrumentations.spec.js +++ b/packages/dd-trace/test/config/disabled_instrumentations.spec.js @@ -1,11 +1,23 @@ 'use strict' -process.env.DD_TRACE_DISABLED_INSTRUMENTATIONS = 'express' - require('../setup/tap') describe('config/disabled_instrumentations', () => { it('should disable loading instrumentations completely', () => { + process.env.DD_TRACE_DISABLED_INSTRUMENTATIONS = 'express' + const handleBefore = require('express').application.handle + const tracer = require('../../../..') + const handleAfterImport = require('express').application.handle + tracer.init() + const handleAfterInit = require('express').application.handle + + expect(handleBefore).to.equal(handleAfterImport) + expect(handleBefore).to.equal(handleAfterInit) + delete process.env.DD_TRACE_DISABLED_INSTRUMENTATIONS + }) + + it('should disable loading instrumentations using DD_TRACE__ENABLED', () => { + process.env.DD_TRACE_EXPRESS_ENABLED = 'false' const handleBefore = require('express').application.handle const tracer = require('../../../..') const handleAfterImport = require('express').application.handle @@ -14,5 +26,6 @@ describe('config/disabled_instrumentations', () => { expect(handleBefore).to.equal(handleAfterImport) expect(handleBefore).to.equal(handleAfterInit) + delete process.env.DD_TRACE_EXPRESS_ENABLED }) }) diff --git a/packages/dd-trace/test/custom-metrics-app.js b/packages/dd-trace/test/custom-metrics-app.js new file mode 100644 index 00000000000..c46f41f18b4 --- /dev/null +++ b/packages/dd-trace/test/custom-metrics-app.js @@ -0,0 +1,11 @@ +#!/usr/bin/env node + +/* eslint-disable no-console */ + +console.log('demo app started') + +const tracer = require('../../../').init() + +tracer.dogstatsd.increment('page.views.data') + +console.log('demo app finished') diff --git a/packages/dd-trace/test/custom-metrics.spec.js b/packages/dd-trace/test/custom-metrics.spec.js new file mode 100644 index 00000000000..49725be7e86 --- /dev/null +++ b/packages/dd-trace/test/custom-metrics.spec.js @@ -0,0 +1,62 @@ +'use strict' + +/* eslint-disable no-console */ + +require('./setup/tap') + +const http = require('http') +const path = require('path') +const os = require('os') +const { exec } = require('child_process') + +describe('Custom Metrics', () => { + let httpServer + let httpPort + let metricsData + let sockets + + beforeEach((done) => { + sockets = [] + httpServer = http.createServer((req, res) => { + let httpData = '' + req.on('data', d => { httpData += d.toString() }) + req.on('end', () => { + res.statusCode = 200 + res.end() + if (req.url === '/dogstatsd/v2/proxy') { + metricsData = httpData + } + }) + }).listen(0, () => { + httpPort = httpServer.address().port + if (os.platform() === 'win32') { + done() + return + } + done() + }) + httpServer.on('connection', socket => sockets.push(socket)) + }) + + afterEach(() => { + httpServer.close() + sockets.forEach(socket => socket.destroy()) + }) + + it('should send metrics before process exit', (done) => { + exec(`${process.execPath} ${path.join(__dirname, 'custom-metrics-app.js')}`, { + env: { + DD_TRACE_AGENT_URL: `http://127.0.0.1:${httpPort}` + } + }, (err, stdout, stderr) => { + if (err) return done(err) + if (stdout) console.log(stdout) + if (stderr) console.error(stderr) + + // eslint-disable-next-line no-undef + expect(metricsData.split('#')[0]).to.equal('page.views.data:1|c|') + + done() + }) + }) +}) diff --git a/packages/dd-trace/test/datastreams/data_streams_checkpointer.spec.js b/packages/dd-trace/test/datastreams/data_streams_checkpointer.spec.js new file mode 100644 index 00000000000..db29f96b575 --- /dev/null +++ b/packages/dd-trace/test/datastreams/data_streams_checkpointer.spec.js @@ -0,0 +1,71 @@ +require('../setup/tap') + +const agent = require('../plugins/agent') + +const expectedProducerHash = '11369286567396183453' +const expectedConsumerHash = '11204511019589278729' +const DSM_CONTEXT_HEADER = 'dd-pathway-ctx-base64' + +describe('data streams checkpointer manual api', () => { + let tracer + + before(() => { + process.env.DD_DATA_STREAMS_ENABLED = 'true' + tracer = require('../..').init() + agent.load(null, { dsmEnabled: true }) + }) + + after(() => { + return agent.close({ ritmReset: false }) + }) + + it('should set a checkpoint when calling setProduceCheckpoint', function (done) { + const expectedEdgeTags = ['direction:out', 'manual_checkpoint:true', 'topic:test-queue', 'type:testProduce'] + + agent.expectPipelineStats(dsmStats => { + let statsPointsReceived = 0 + // we should have 1 dsm stats points + for (const timeStatsBucket of dsmStats) { + if (timeStatsBucket && timeStatsBucket.Stats) { + for (const statsBucket of timeStatsBucket.Stats) { + statsPointsReceived += statsBucket.Stats.length + } + } + } + expect(statsPointsReceived).to.equal(1) + expect(agent.dsmStatsExist(agent, expectedProducerHash, expectedEdgeTags)).to.equal(true) + }).then(done, done) + + const headers = {} + + tracer.dataStreamsCheckpointer.setProduceCheckpoint('testProduce', 'test-queue', headers) + + expect(DSM_CONTEXT_HEADER in headers).to.equal(true) + }) + + it('should set a checkpoint when calling setConsumeCheckpoint', function (done) { + const expectedEdgeTags = ['direction:in', 'manual_checkpoint:true', 'topic:test-queue', 'type:testConsume'] + + agent.expectPipelineStats(dsmStats => { + let statsPointsReceived = 0 + // we should have 2 dsm stats points because of the earlier produce + for (const timeStatsBucket of dsmStats) { + if (timeStatsBucket && timeStatsBucket.Stats) { + for (const statsBucket of timeStatsBucket.Stats) { + statsPointsReceived += statsBucket.Stats.length + } + } + } + expect(statsPointsReceived).to.equal(2) + expect(agent.dsmStatsExist(agent, expectedConsumerHash, expectedEdgeTags)).to.equal(true) + }).then(done, done) + + const headers = { + [DSM_CONTEXT_HEADER]: 'tvMEiT2p8cjWzqLRnGTWzqLRnGQ=' // same context as previous produce + } + + tracer.dataStreamsCheckpointer.setConsumeCheckpoint('testConsume', 'test-queue', headers) + + expect(DSM_CONTEXT_HEADER in headers).to.equal(true) + }) +}) diff --git a/packages/dd-trace/test/datastreams/encoding.spec.js b/packages/dd-trace/test/datastreams/encoding.spec.js index ea7a78b17e3..6bec1b32099 100644 --- a/packages/dd-trace/test/datastreams/encoding.spec.js +++ b/packages/dd-trace/test/datastreams/encoding.spec.js @@ -1,7 +1,6 @@ 'use strict' require('../setup/tap') - const { encodeVarint, decodeVarint } = require('../../src/datastreams/encoding') const { expect } = require('chai') @@ -17,6 +16,7 @@ describe('encoding', () => { expect(decoded).to.equal(n) expect(bytes).to.length(0) }) + it('encoding then decoding should be a no op for bigger than int32 numbers', () => { const n = 1679711644352 const expectedEncoded = new Uint8Array([ @@ -33,6 +33,7 @@ describe('encoding', () => { expect(decoded2).to.equal(n) expect(bytes2).to.length(0) }) + it('encoding a number bigger than Max safe int fails.', () => { const n = Number.MAX_SAFE_INTEGER + 10 const encoded = encodeVarint(n) diff --git a/packages/dd-trace/test/datastreams/pathway.spec.js b/packages/dd-trace/test/datastreams/pathway.spec.js index 0722f220f69..02c4121747c 100644 --- a/packages/dd-trace/test/datastreams/pathway.spec.js +++ b/packages/dd-trace/test/datastreams/pathway.spec.js @@ -3,7 +3,14 @@ require('../setup/tap') const { expect } = require('chai') -const { computePathwayHash, encodePathwayContext, decodePathwayContext } = require('../../src/datastreams/pathway') +const { + computePathwayHash, + encodePathwayContext, + decodePathwayContext, + encodePathwayContextBase64, + decodePathwayContextBase64, + DsmPathwayCodec +} = require('../../src/datastreams/pathway') describe('encoding', () => { it('hash should always give the same value', () => { @@ -15,6 +22,7 @@ describe('encoding', () => { expect(hash) .to.deep.equal(Buffer.from('ec99e1e8e682985d', 'hex')) }) + it('encoding and decoding should be a no op', () => { const expectedContext = { hash: Buffer.from('4cce4d8e07685728', 'hex'), @@ -27,6 +35,7 @@ describe('encoding', () => { expect(decoded.pathwayStartNs).to.equal(expectedContext.pathwayStartNs) expect(decoded.edgeStartNs).to.equal(expectedContext.edgeStartNs) }) + it('decoding of a context should be consistent between languages', () => { const data = Buffer.from([76, 206, 77, 142, 7, 104, 87, 40, 196, 231, 192, 159, 143, 98, 200, 217, 195, 159, 143, 98]) @@ -40,4 +49,104 @@ describe('encoding', () => { expect(decoded.pathwayStartNs).to.equal(expectedContext.pathwayStartNs) expect(decoded.edgeStartNs).to.equal(expectedContext.edgeStartNs) }) + + it('should encode and decode to the same value when using base64', () => { + const ctx = { + pathwayStartNs: 1685673482722000000, + edgeStartNs: 1685673506404000000 + } + ctx.hash = computePathwayHash('test-service', 'test-env', + ['direction:in', 'group:group1', 'topic:topic1', 'type:kafka'], Buffer.from('0000000000000000', 'hex')) + + const encodedPathway = encodePathwayContextBase64(ctx) + const decodedPathway = decodePathwayContextBase64(encodedPathway) + + expect(decodedPathway.hash.toString()).to.equal(ctx.hash.toString()) + expect(decodedPathway.pathwayStartNs).to.equal(ctx.pathwayStartNs) + expect(decodedPathway.edgeStartNs).to.equal(ctx.edgeStartNs) + }) + + it('should encode and decode to the same value when using the PathwayCodec', () => { + const ctx = { + pathwayStartNs: 1685673482722000000, + edgeStartNs: 1685673506404000000 + } + const carrier = {} + ctx.hash = computePathwayHash('test-service', 'test-env', + ['direction:in', 'group:group1', 'topic:topic1', 'type:kafka'], Buffer.from('0000000000000000', 'hex')) + + DsmPathwayCodec.encode(ctx, carrier) + const decodedCtx = DsmPathwayCodec.decode(carrier) + + expect(decodedCtx.hash.toString()).to.equal(ctx.hash.toString()) + expect(decodedCtx.pathwayStartNs).to.equal(ctx.pathwayStartNs) + expect(decodedCtx.edgeStartNs).to.equal(ctx.edgeStartNs) + }) + + it('should encode/decode to the same value when using the PathwayCodec, base64 and the deprecated ctx key', () => { + const ctx = { + pathwayStartNs: 1685673482722000000, + edgeStartNs: 1685673506404000000 + } + const carrier = {} + ctx.hash = computePathwayHash('test-service', 'test-env', + ['direction:in', 'group:group1', 'topic:topic1', 'type:kafka'], Buffer.from('0000000000000000', 'hex')) + + DsmPathwayCodec.encode(ctx, carrier) + carrier['dd-pathway-ctx'] = carrier['dd-pathway-ctx-base64'] + delete carrier['dd-pathway-ctx-base64'] + const decodedCtx = DsmPathwayCodec.decode(carrier) + + expect(decodedCtx.hash.toString()).to.equal(ctx.hash.toString()) + expect(decodedCtx.pathwayStartNs).to.equal(ctx.pathwayStartNs) + expect(decodedCtx.edgeStartNs).to.equal(ctx.edgeStartNs) + }) + + it('should encode/decode to the same value when using the PathwayCodec and the deprecated encoding', () => { + const ctx = { + pathwayStartNs: 1685673482722000000, + edgeStartNs: 1685673506404000000 + } + const carrier = {} + ctx.hash = computePathwayHash('test-service', 'test-env', + ['direction:in', 'group:group1', 'topic:topic1', 'type:kafka'], Buffer.from('0000000000000000', 'hex')) + + carrier['dd-pathway-ctx'] = encodePathwayContext(ctx) + const decodedCtx = DsmPathwayCodec.decode(carrier) + + expect(decodedCtx.hash.toString()).to.equal(ctx.hash.toString()) + expect(decodedCtx.pathwayStartNs).to.equal(ctx.pathwayStartNs) + expect(decodedCtx.edgeStartNs).to.equal(ctx.edgeStartNs) + }) + + it('should inject the base64 encoded string to the carrier', () => { + const ctx = { + pathwayStartNs: 1685673482722000000, + edgeStartNs: 1685673506404000000 + } + const carrier = {} + ctx.hash = computePathwayHash('test-service', 'test-env', + ['direction:in', 'group:group1', 'topic:topic1', 'type:kafka'], Buffer.from('0000000000000000', 'hex')) + + DsmPathwayCodec.encode(ctx, carrier) + + const expectedBase64Hash = '7Jnh6OaCmF3E58Cfj2LI2cOfj2I=' + expect(carrier['dd-pathway-ctx-base64']).to.equal(expectedBase64Hash) + }) + + it('should extract the base64 encoded string from the carrier', () => { + const ctx = { + pathwayStartNs: 1685673482722000000, + edgeStartNs: 1685673506404000000 + } + ctx.hash = computePathwayHash('test-service', 'test-env', + ['direction:in', 'group:group1', 'topic:topic1', 'type:kafka'], Buffer.from('0000000000000000', 'hex')) + + const carrier = {} + const expectedBase64Hash = '7Jnh6OaCmF3E58Cfj2LI2cOfj2I=' + carrier['dd-pathway-ctx-base64'] = expectedBase64Hash + const decodedCtx = DsmPathwayCodec.decode(carrier) + + expect(decodedCtx.hash.toString()).to.equal(ctx.hash.toString()) + }) }) diff --git a/packages/dd-trace/test/datastreams/processor.spec.js b/packages/dd-trace/test/datastreams/processor.spec.js index 11425d039a1..0c30bc77947 100644 --- a/packages/dd-trace/test/datastreams/processor.spec.js +++ b/packages/dd-trace/test/datastreams/processor.spec.js @@ -23,6 +23,7 @@ const writer = { const DataStreamsWriter = sinon.stub().returns(writer) const { StatsPoint, + Backlog, StatsBucket, TimeBuckets, DataStreamsProcessor, @@ -75,29 +76,107 @@ describe('StatsPoint', () => { }) describe('StatsBucket', () => { - const buckets = new StatsBucket() + describe('Checkpoints', () => { + let buckets - it('should start empty', () => { - expect(buckets.size).to.equal(0) - }) + beforeEach(() => { buckets = new StatsBucket() }) - it('should add a new entry when no matching key is found', () => { - const bucket = buckets.forCheckpoint(mockCheckpoint) - expect(bucket).to.be.an.instanceOf(StatsPoint) - expect(buckets.size).to.equal(1) - const [key, value] = Array.from(buckets.entries())[0] - expect(key.toString()).to.equal(mockCheckpoint.hash.toString()) - expect(value).to.be.instanceOf(StatsPoint) - }) + it('should start empty', () => { + expect(buckets.checkpoints.size).to.equal(0) + }) - it('should not add a new entry if matching key is found', () => { - buckets.forCheckpoint(mockCheckpoint) - expect(buckets.size).to.equal(1) + it('should add a new entry when no matching key is found', () => { + const bucket = buckets.forCheckpoint(mockCheckpoint) + const checkpoints = buckets.checkpoints + expect(bucket).to.be.an.instanceOf(StatsPoint) + expect(checkpoints.size).to.equal(1) + const [key, value] = Array.from(checkpoints.entries())[0] + expect(key.toString()).to.equal(mockCheckpoint.hash.toString()) + expect(value).to.be.instanceOf(StatsPoint) + }) + + it('should not add a new entry if matching key is found', () => { + buckets.forCheckpoint(mockCheckpoint) + buckets.forCheckpoint(mockCheckpoint) + expect(buckets.checkpoints.size).to.equal(1) + }) + + it('should add a new entry when new checkpoint does not match existing agg keys', () => { + buckets.forCheckpoint(mockCheckpoint) + buckets.forCheckpoint(anotherMockCheckpoint) + expect(buckets.checkpoints.size).to.equal(2) + }) }) - it('should add a new entry when new checkpoint does not match existing agg keys', () => { - buckets.forCheckpoint(anotherMockCheckpoint) - expect(buckets.size).to.equal(2) + describe('Backlogs', () => { + let backlogBuckets + const mockBacklog = { + offset: 12, + type: 'kafka_consume', + consumer_group: 'test-consumer', + partition: 0, + topic: 'test-topic' + } + + beforeEach(() => { + backlogBuckets = new StatsBucket() + }) + + it('should start empty', () => { + expect(backlogBuckets.backlogs.size).to.equal(0) + }) + + it('should add a new entry when empty', () => { + const bucket = backlogBuckets.forBacklog(mockBacklog) + const backlogs = backlogBuckets.backlogs + expect(bucket).to.be.an.instanceOf(Backlog) + const [, value] = Array.from(backlogs.entries())[0] + expect(value).to.be.instanceOf(Backlog) + }) + + it('should add a new entry when given different tags', () => { + const otherMockBacklog = { + offset: 1, + type: 'kafka_consume', + consumer_group: 'test-consumer', + partition: 1, + topic: 'test-topic' + } + + backlogBuckets.forBacklog(mockBacklog) + backlogBuckets.forBacklog(otherMockBacklog) + expect(backlogBuckets.backlogs.size).to.equal(2) + }) + + it('should update the existing entry if offset is higher', () => { + const higherMockBacklog = { + offset: 16, + type: 'kafka_consume', + consumer_group: 'test-consumer', + partition: 0, + topic: 'test-topic' + } + + backlogBuckets.forBacklog(mockBacklog) + const backlog = backlogBuckets.forBacklog(higherMockBacklog) + expect(backlog.offset).to.equal(higherMockBacklog.offset) + expect(backlogBuckets.backlogs.size).to.equal(1) + }) + + it('should discard the passed backlog if offset is lower', () => { + const lowerMockBacklog = { + offset: 2, + type: 'kafka_consume', + consumer_group: 'test-consumer', + partition: 0, + topic: 'test-topic' + } + + backlogBuckets.forBacklog(mockBacklog) + const backlog = backlogBuckets.forBacklog(lowerMockBacklog) + expect(backlog.offset).to.equal(mockBacklog.offset) + expect(backlogBuckets.backlogs.size).to.equal(1) + }) }) }) @@ -125,9 +204,14 @@ describe('DataStreamsProcessor', () => { env: 'test', version: 'v1', service: 'service1', - tags: { tag: 'some tag' } + tags: { foo: 'foovalue', bar: 'barvalue' } } + beforeEach(() => { + processor = new DataStreamsProcessor(config) + clearTimeout(processor.timer) + }) + it('should construct', () => { processor = new DataStreamsProcessor(config) clearTimeout(processor.timer) @@ -144,6 +228,35 @@ describe('DataStreamsProcessor', () => { expect(processor.tags).to.deep.equal(config.tags) }) + it('should track backlogs', () => { + const mockBacklog = { + offset: 12, + type: 'kafka_consume', + consumer_group: 'test-consumer', + partition: 0, + topic: 'test-topic' + } + expect(processor.buckets.size).to.equal(0) + processor.recordOffset({ timestamp: DEFAULT_TIMESTAMP, ...mockBacklog }) + expect(processor.buckets.size).to.equal(1) + + const timeBucket = processor.buckets.values().next().value + expect(timeBucket).to.be.instanceOf(StatsBucket) + expect(timeBucket.backlogs.size).to.equal(1) + + const backlog = timeBucket.forBacklog(mockBacklog) + expect(timeBucket.backlogs.size).to.equal(1) + expect(backlog).to.be.instanceOf(Backlog) + + const encoded = backlog.encode() + expect(encoded).to.deep.equal({ + Tags: [ + 'consumer_group:test-consumer', 'partition:0', 'topic:test-topic', 'type:kafka_consume' + ], + Value: 12 + }) + }) + it('should track latency stats', () => { expect(processor.buckets.size).to.equal(0) processor.recordCheckpoint(mockCheckpoint) @@ -151,10 +264,10 @@ describe('DataStreamsProcessor', () => { const timeBucket = processor.buckets.values().next().value expect(timeBucket).to.be.instanceOf(StatsBucket) - expect(timeBucket.size).to.equal(1) + expect(timeBucket.checkpoints.size).to.equal(1) const checkpointBucket = timeBucket.forCheckpoint(mockCheckpoint) - expect(timeBucket.size).to.equal(1) + expect(timeBucket.checkpoints.size).to.equal(1) expect(checkpointBucket).to.be.instanceOf(StatsPoint) edgeLatency = new LogCollapsingLowestDenseDDSketch(0.00775) @@ -174,6 +287,7 @@ describe('DataStreamsProcessor', () => { }) it('should export on interval', () => { + processor.recordCheckpoint(mockCheckpoint) processor.onInterval() expect(writer.flush).to.be.calledWith({ Env: 'test', @@ -189,10 +303,12 @@ describe('DataStreamsProcessor', () => { EdgeLatency: edgeLatency.toProto(), PathwayLatency: pathwayLatency.toProto(), PayloadSize: payloadSize.toProto() - }] + }], + Backlogs: [] }], TracerVersion: pkg.version, - Lang: 'javascript' + Lang: 'javascript', + Tags: ['foo:foovalue', 'bar:barvalue'] }) }) }) diff --git a/packages/dd-trace/test/datastreams/schemas/schema_builder.spec.js b/packages/dd-trace/test/datastreams/schemas/schema_builder.spec.js new file mode 100644 index 00000000000..134724b593a --- /dev/null +++ b/packages/dd-trace/test/datastreams/schemas/schema_builder.spec.js @@ -0,0 +1,57 @@ +'use strict' + +require('../../setup/tap') + +const { SchemaBuilder } = require('../../../src/datastreams/schemas/schema_builder') +const { expect } = require('chai') + +class Iterator { + iterateOverSchema (builder) { + builder.addProperty('person', 'name', false, 'string', 'name of the person', null, null, null) + builder.addProperty('person', 'phone_numbers', true, 'string', null, null, null, null) + builder.addProperty('person', 'person_name', false, 'string', null, null, null, null) + builder.addProperty('person', 'address', false, 'object', null, '#/components/schemas/address', null, null) + builder.addProperty('address', 'zip', false, 'number', null, null, 'int', null) + builder.addProperty('address', 'street', false, 'string', null, null, null, null) + } +} + +describe('SchemaBuilder', () => { + it('should convert schema correctly to JSON', () => { + const builder = new SchemaBuilder(new Iterator()) + + const shouldExtractPerson = builder.shouldExtractSchema('person', 0) + const shouldExtractAddress = builder.shouldExtractSchema('address', 1) + const shouldExtractPerson2 = builder.shouldExtractSchema('person', 0) + const shouldExtractTooDeep = builder.shouldExtractSchema('city', 11) + const schema = SchemaBuilder.getSchemaDefinition(builder.build()) + + const expectedSchema = { + components: { + schemas: { + person: { + properties: { + name: { description: 'name of the person', type: 'string' }, + phone_numbers: { items: { type: 'string' }, type: 'array' }, + person_name: { type: 'string' }, + address: { $ref: '#/components/schemas/address', type: 'object' } + }, + type: 'object' + }, + address: { + properties: { zip: { format: 'int', type: 'number' }, street: { type: 'string' } }, + type: 'object' + } + } + }, + openapi: '3.0.0' + } + + expect(JSON.parse(schema.definition)).to.deep.equal(expectedSchema) + expect(schema.id).to.equal('9510078321201428652') + expect(shouldExtractPerson).to.be.true + expect(shouldExtractAddress).to.be.true + expect(shouldExtractPerson2).to.be.false + expect(shouldExtractTooDeep).to.be.false + }) +}) diff --git a/packages/dd-trace/test/datastreams/schemas/schema_sampler.spec.js b/packages/dd-trace/test/datastreams/schemas/schema_sampler.spec.js new file mode 100644 index 00000000000..80e288a66b6 --- /dev/null +++ b/packages/dd-trace/test/datastreams/schemas/schema_sampler.spec.js @@ -0,0 +1,39 @@ +'use strict' + +require('../../setup/tap') + +const { SchemaSampler } = require('../../../src/datastreams/schemas/schema_sampler') +const { expect } = require('chai') + +describe('SchemaSampler', () => { + it('samples with correct weights', () => { + const currentTimeMs = 100000 + const sampler = new SchemaSampler() + + const canSample1 = sampler.canSample(currentTimeMs) + const weight1 = sampler.trySample(currentTimeMs) + + const canSample2 = sampler.canSample(currentTimeMs + 1000) + const weight2 = sampler.trySample(currentTimeMs + 1000) + + const canSample3 = sampler.canSample(currentTimeMs + 2000) + const weight3 = sampler.trySample(currentTimeMs + 2000) + + const canSample4 = sampler.canSample(currentTimeMs + 30000) + const weight4 = sampler.trySample(currentTimeMs + 30000) + + const canSample5 = sampler.canSample(currentTimeMs + 30001) + const weight5 = sampler.trySample(currentTimeMs + 30001) + + expect(canSample1).to.be.true + expect(weight1).to.equal(1) + expect(canSample2).to.be.false + expect(weight2).to.equal(0) + expect(canSample3).to.be.false + expect(weight3).to.equal(0) + expect(canSample4).to.be.true + expect(weight4).to.equal(3) + expect(canSample5).to.be.false + expect(weight5).to.equal(0) + }) +}) diff --git a/packages/dd-trace/test/datastreams/writer.spec.js b/packages/dd-trace/test/datastreams/writer.spec.js new file mode 100644 index 00000000000..0d4d7875629 --- /dev/null +++ b/packages/dd-trace/test/datastreams/writer.spec.js @@ -0,0 +1,52 @@ +'use strict' +require('../setup/tap') +const pkg = require('../../../../package.json') +const stubRequest = sinon.stub() +const msgpack = require('msgpack-lite') +const codec = msgpack.createCodec({ int64: true }) + +const stubZlib = { + gzip: (payload, _opts, fn) => { + fn(undefined, payload) + } +} + +const { DataStreamsWriter } = proxyquire( + '../src/datastreams/writer', { + '../exporters/common/request': stubRequest, + zlib: stubZlib + }) + +describe('DataStreamWriter unix', () => { + let writer + const unixConfig = { + hostname: '', + url: new URL('unix:///var/run/datadog/apm.socket'), + port: '' + } + + it('should construct unix config', () => { + writer = new DataStreamsWriter(unixConfig) + expect(writer._url).to.equal(unixConfig.url) + }) + + it("should call 'request' through flush with correct options", () => { + writer = new DataStreamsWriter(unixConfig) + writer.flush({}) + const stubRequestCall = stubRequest.getCalls()[0] + const decodedPayload = msgpack.decode(stubRequestCall?.args[0], { codec }) + const requestOptions = stubRequestCall?.args[1] + expect(decodedPayload).to.deep.equal({}) + expect(requestOptions).to.deep.equal({ + path: '/v0.1/pipeline_stats', + method: 'POST', + headers: { + 'Datadog-Meta-Lang': 'javascript', + 'Datadog-Meta-Tracer-Version': pkg.version, + 'Content-Type': 'application/msgpack', + 'Content-Encoding': 'gzip' + }, + url: unixConfig.url + }) + }) +}) diff --git a/packages/dd-trace/test/debugger/devtools_client/snapshot/complex-types.spec.js b/packages/dd-trace/test/debugger/devtools_client/snapshot/complex-types.spec.js new file mode 100644 index 00000000000..22036e4c60a --- /dev/null +++ b/packages/dd-trace/test/debugger/devtools_client/snapshot/complex-types.spec.js @@ -0,0 +1,323 @@ +'use strict' + +require('../../../setup/mocha') + +const { session, getTargetCodePath, enable, teardown, setAndTriggerBreakpoint } = require('./utils') +const { getLocalStateForCallFrame } = require('../../../../src/debugger/devtools_client/snapshot') + +const NODE_20_PLUS = require('semver').gte(process.version, '20.0.0') +const target = getTargetCodePath(__filename) + +describe('debugger -> devtools client -> snapshot.getLocalStateForCallFrame', function () { + describe('complex types', function () { + let state + + beforeEach(enable(__filename)) + + afterEach(teardown) + + beforeEach(async function () { + let resolve + const localState = new Promise((_resolve) => { resolve = _resolve }) + + session.once('Debugger.paused', async ({ params }) => { + expect(params.hitBreakpoints.length).to.eq(1) + + resolve((await getLocalStateForCallFrame(params.callFrames[0]))()) + }) + + await setAndTriggerBreakpoint(target, 10) + + state = await localState + }) + + it('should contain expected properties from closure scope', function () { + expect(Object.keys(state).length).to.equal(28) + + // from block scope + // ... tested individually in the remaining it-blocks inside this describe-block + + // from closure scope + expect(state).to.have.deep.property('ref', { + type: 'Object', + fields: { + wmo1: { type: 'Object', fields: { a: { type: 'number', value: '1' } } }, + wmo2: { type: 'Object', fields: { b: { type: 'number', value: '3' } } }, + wso1: { type: 'Object', fields: { a: { type: 'number', value: '1' } } }, + wso2: { type: 'Object', fields: { a: { type: 'number', value: '2' } } }, + wso3: { type: 'Object', fields: { a: { type: 'number', value: '3' } } } + } + }) + expect(state).to.have.deep.property('get', { + type: 'Function', + fields: { + length: { type: 'number', value: '0' }, + name: { type: 'string', value: 'get' } + } + }) + }) + + it('object literal', function () { + expect(state).to.have.deep.property('oblit', { + type: 'Object', + fields: { + a: { type: 'number', value: '1' }, + b_b: { type: 'number', value: '2' }, + 'Symbol(c)': { type: 'number', value: '3' }, + d: { type: 'getter' }, + e: { type: 'getter' }, + f: { type: 'setter' }, + g: { type: 'getter/setter' } + } + }) + }) + + it('custom object from class', function () { + expect(state).to.have.deep.property('obnew', { + type: 'MyClass', + fields: { + foo: { type: 'number', value: '42' }, + '#secret': { type: 'number', value: '42' } + } + }) + }) + + it('Array', function () { + expect(state).to.have.deep.property('arr', { + type: 'Array', + elements: [ + { type: 'number', value: '1' }, + { type: 'number', value: '2' }, + { type: 'number', value: '3' } + ] + }) + }) + + it('RegExp', function () { + expect(state).to.have.deep.property('regex', { type: 'RegExp', value: '/foo/' }) + }) + + it('Date', function () { + expect(state).to.have.deep.property('date', { + type: 'Date', + value: '2024-09-20T07:22:59Z' // missing milliseconds due to API limitation (should have been `998`) + }) + }) + + it('Map', function () { + expect(state).to.have.deep.property('map', { + type: 'Map', + entries: [ + [{ type: 'number', value: '1' }, { type: 'number', value: '2' }], + [{ type: 'number', value: '3' }, { type: 'number', value: '4' }] + ] + }) + }) + + it('Set', function () { + expect(state).to.have.deep.property('set', { + type: 'Set', + elements: [ + { + type: 'Array', + elements: [ + { type: 'number', value: '1' }, + { type: 'number', value: '2' } + ] + }, + { type: 'number', value: '3' }, + { type: 'number', value: '4' } + ] + }) + }) + + it('WeakMap', function () { + expect(state).to.have.property('wmap') + expect(state.wmap).to.have.keys('type', 'entries') + expect(state.wmap.entries).to.be.an('array') + state.wmap.entries = state.wmap.entries.sort((a, b) => a[1].value - b[1].value) + expect(state).to.have.deep.property('wmap', { + type: 'WeakMap', + entries: [[ + { type: 'Object', fields: { a: { type: 'number', value: '1' } } }, + { type: 'number', value: '2' } + ], [ + { type: 'Object', fields: { b: { type: 'number', value: '3' } } }, + { type: 'number', value: '4' } + ]] + }) + }) + + it('WeakSet', function () { + expect(state).to.have.property('wset') + expect(state.wset).to.have.keys('type', 'elements') + expect(state.wset.elements).to.be.an('array') + state.wset.elements = state.wset.elements.sort((a, b) => a.fields.a.value - b.fields.a.value) + expect(state).to.have.deep.property('wset', { + type: 'WeakSet', + elements: [ + { type: 'Object', fields: { a: { type: 'number', value: '1' } } }, + { type: 'Object', fields: { a: { type: 'number', value: '2' } } }, + { type: 'Object', fields: { a: { type: 'number', value: '3' } } } + ] + }) + }) + + it('Generator', function () { + expect(state).to.have.deep.property('gen', { + type: 'generator', + fields: { foo: { type: 'number', value: '42' } } + }) + }) + + it('Error', function () { + expect(state).to.have.property('err') + expect(state.err).to.have.keys('type', 'fields') + expect(state.err).to.have.property('type', 'CustomError') + expect(state.err.fields).to.be.an('object') + expect(state.err.fields).to.have.keys('stack', 'message', 'foo') + expect(state.err.fields).to.deep.include({ + message: { type: 'string', value: 'boom!' }, + foo: { type: 'number', value: '42' } + }) + expect(state.err.fields.stack).to.have.keys('type', 'value', 'truncated', 'size') + expect(state.err.fields.stack.value).to.be.a('string') + expect(state.err.fields.stack.value).to.match(/^Error: boom!/) + expect(state.err.fields.stack.size).to.be.a('number') + expect(state.err.fields.stack.size).to.above(255) + expect(state.err.fields.stack).to.deep.include({ + type: 'string', + truncated: true + }) + }) + + it('Function', function () { + expect(state).to.have.deep.property('fn', { + type: 'Function', + fields: { + foo: { + type: 'Object', + fields: { bar: { type: 'number', value: '42' } } + }, + length: { type: 'number', value: '2' }, + name: { type: 'string', value: 'fnWithProperties' } + } + }) + }) + + it('Bound function', function () { + expect(state).to.have.deep.property('bfn', { + type: 'Function', + fields: { + length: { type: 'number', value: '0' }, + name: { type: 'string', value: 'bound fnWithProperties' } + } + }) + }) + + it('Arrow function', function () { + expect(state).to.have.deep.property('afn', { + type: 'Function', + fields: { + length: { type: 'number', value: '0' }, + name: { type: 'string', value: 'afn' } + } + }) + }) + + it('Class', function () { + expect(state).to.have.deep.property('cls', { type: 'class MyClass' }) + }) + + it('Anonymous class', function () { + expect(state).to.have.deep.property('acls', { type: 'class' }) + }) + + it('Proxy for object literal', function () { + expect(state).to.have.deep.property('prox', { + type: NODE_20_PLUS ? 'Proxy(Object)' : 'Proxy', + fields: { + target: { type: 'boolean', value: 'true' } + } + }) + }) + + it('Proxy for custom class', function () { + expect(state).to.have.deep.property('custProx', { + type: NODE_20_PLUS ? 'Proxy(MyClass)' : 'Proxy', + fields: { + foo: { type: 'number', value: '42' } + } + }) + }) + + it('Promise: Pending', function () { + expect(state).to.have.deep.property('pPen', { + type: 'Promise', + fields: { + '[[PromiseState]]': { type: 'string', value: 'pending' }, + '[[PromiseResult]]': { type: 'undefined' } + } + }) + }) + + it('Promise: Resolved', function () { + expect(state).to.have.deep.property('pRes', { + type: 'Promise', + fields: { + '[[PromiseState]]': { type: 'string', value: 'fulfilled' }, + '[[PromiseResult]]': { type: 'string', value: 'resolved value' } + } + }) + }) + + it('Promise: Rejected', function () { + expect(state).to.have.deep.property('pRej', { + type: 'Promise', + fields: { + '[[PromiseState]]': { type: 'string', value: 'rejected' }, + '[[PromiseResult]]': { type: 'string', value: 'rejected value' } + } + }) + }) + + it('TypedArray', function () { + expect(state).to.have.deep.property('tarr', { + type: 'Int8Array', + elements: [ + { type: 'number', value: '72' }, + { type: 'number', value: '65' }, + { type: 'number', value: '76' } + ] + }) + }) + + it('ArrayBuffer', function () { + expect(state).to.have.deep.property('ab', { + type: 'ArrayBuffer', + value: 'HAL' + }) + }) + + it('SharedArrayBuffer', function () { + expect(state).to.have.deep.property('sab', { + type: 'SharedArrayBuffer', + value: 'hello\x01\x02\x03world' + }) + }) + + it('circular reference in object', function () { + expect(state).to.have.property('circular') + expect(state.circular).to.have.property('type', 'Object') + expect(state.circular).to.have.property('fields') + // For the circular field, just check that at least one of the expected properties are present + expect(state.circular.fields).to.deep.include({ + regex: { type: 'RegExp', value: '/foo/' } + }) + }) + + it('non-enumerable property', function () { + expect(state).to.have.deep.property('hidden', { type: 'string', value: 'secret' }) + }) + }) +}) diff --git a/packages/dd-trace/test/debugger/devtools_client/snapshot/max-collection-size.spec.js b/packages/dd-trace/test/debugger/devtools_client/snapshot/max-collection-size.spec.js new file mode 100644 index 00000000000..6b63eec715e --- /dev/null +++ b/packages/dd-trace/test/debugger/devtools_client/snapshot/max-collection-size.spec.js @@ -0,0 +1,129 @@ +'use strict' + +require('../../../setup/mocha') + +const { getTargetCodePath, enable, teardown, assertOnBreakpoint, setAndTriggerBreakpoint } = require('./utils') + +const DEFAULT_MAX_COLLECTION_SIZE = 100 +const target = getTargetCodePath(__filename) + +describe('debugger -> devtools client -> snapshot.getLocalStateForCallFrame', function () { + describe('maxCollectionSize', function () { + const configs = [ + undefined, + { maxCollectionSize: 3 } + ] + + beforeEach(enable(__filename)) + + afterEach(teardown) + + for (const config of configs) { + const maxCollectionSize = config?.maxCollectionSize ?? DEFAULT_MAX_COLLECTION_SIZE + const postfix = config === undefined ? 'not set' : `set to ${config.maxCollectionSize}` + + describe(`shold respect the default maxCollectionSize if ${postfix}`, function () { + let state + + const expectedElements = [] + const expectedEntries = [] + for (let i = 1; i <= maxCollectionSize; i++) { + expectedElements.push({ type: 'number', value: i.toString() }) + expectedEntries.push([ + { type: 'number', value: i.toString() }, + { + type: 'Object', + fields: { i: { type: 'number', value: i.toString() } } + } + ]) + } + + beforeEach(function (done) { + assertOnBreakpoint(done, config, (_state) => { + state = _state + }) + setAndTriggerBreakpoint(target, 24) + }) + + it('should have expected number of elements in state', function () { + expect(state).to.have.keys(['arr', 'map', 'set', 'wmap', 'wset', 'typedArray']) + }) + + it('Array', function () { + expect(state).to.have.deep.property('arr', { + type: 'Array', + elements: expectedElements, + notCapturedReason: 'collectionSize', + size: 1000 + }) + }) + + it('Map', function () { + expect(state).to.have.deep.property('map', { + type: 'Map', + entries: expectedEntries, + notCapturedReason: 'collectionSize', + size: 1000 + }) + }) + + it('Set', function () { + expect(state).to.have.deep.property('set', { + type: 'Set', + elements: expectedElements, + notCapturedReason: 'collectionSize', + size: 1000 + }) + }) + + it('WeakMap', function () { + expect(state.wmap).to.include({ + type: 'WeakMap', + notCapturedReason: 'collectionSize', + size: 1000 + }) + + expect(state.wmap.entries).to.have.lengthOf(maxCollectionSize) + + // The order of the entries is not guaranteed, so we don't know which were removed + for (const entry of state.wmap.entries) { + expect(entry).to.have.lengthOf(2) + expect(entry[0]).to.have.property('type', 'Object') + expect(entry[0].fields).to.have.property('i') + expect(entry[0].fields.i).to.have.property('type', 'number') + expect(entry[0].fields.i).to.have.property('value').to.match(/^\d+$/) + expect(entry[1]).to.have.property('type', 'number') + expect(entry[1]).to.have.property('value', entry[0].fields.i.value) + } + }) + + it('WeakSet', function () { + expect(state.wset).to.include({ + type: 'WeakSet', + notCapturedReason: 'collectionSize', + size: 1000 + }) + + expect(state.wset.elements).to.have.lengthOf(maxCollectionSize) + + // The order of the elements is not guaranteed, so we don't know which were removed + for (const element of state.wset.elements) { + expect(element).to.have.property('type', 'Object') + expect(element.fields).to.have.property('i') + expect(element.fields.i).to.have.property('type', 'number') + expect(element.fields.i).to.have.property('value').to.match(/^\d+$/) + } + }) + + it('TypedArray', function () { + expect(state).to.have.deep.property('typedArray', { + type: 'Uint16Array', + elements: expectedElements, + notCapturedReason: 'collectionSize', + size: 1000 + }) + }) + }) + } + }) +}) diff --git a/packages/dd-trace/test/debugger/devtools_client/snapshot/max-reference-depth.spec.js b/packages/dd-trace/test/debugger/devtools_client/snapshot/max-reference-depth.spec.js new file mode 100644 index 00000000000..4c5971969fb --- /dev/null +++ b/packages/dd-trace/test/debugger/devtools_client/snapshot/max-reference-depth.spec.js @@ -0,0 +1,124 @@ +'use strict' + +require('../../../setup/mocha') + +const { getTargetCodePath, enable, teardown, assertOnBreakpoint, setAndTriggerBreakpoint } = require('./utils') + +const target = getTargetCodePath(__filename) + +describe('debugger -> devtools client -> snapshot.getLocalStateForCallFrame', function () { + describe('maxReferenceDepth', function () { + beforeEach(enable(__filename)) + + afterEach(teardown) + + it('should return expected object for nested objects with maxReferenceDepth: 1', function (done) { + assertOnBreakpoint(done, { maxReferenceDepth: 1 }, (state) => { + expect(Object.keys(state).length).to.equal(1) + + expect(state).to.have.property('myNestedObj') + expect(state.myNestedObj).to.have.property('type', 'Object') + expect(state.myNestedObj).to.have.property('fields') + expect(Object.keys(state.myNestedObj).length).to.equal(2) + + expect(state.myNestedObj.fields).to.have.deep.property('deepObj', { + type: 'Object', notCapturedReason: 'depth' + }) + + expect(state.myNestedObj.fields).to.have.deep.property('deepArr', { + type: 'Array', notCapturedReason: 'depth' + }) + }) + + setAndTriggerBreakpoint(target, 9) + }) + + it('should return expected object for nested objects with maxReferenceDepth: 5', function (done) { + assertOnBreakpoint(done, { maxReferenceDepth: 5 }, (state) => { + expect(Object.entries(state).length).to.equal(1) + + expect(state).to.have.property('myNestedObj') + expect(state.myNestedObj).to.have.property('type', 'Object') + expect(state.myNestedObj).to.have.property('fields') + expect(Object.entries(state.myNestedObj).length).to.equal(2) + + expect(state.myNestedObj.fields).to.have.deep.property('deepObj', { + type: 'Object', + fields: { + foo: { + type: 'Object', + fields: { + foo: { + type: 'Object', + fields: { + foo: { + type: 'Object', + fields: { + foo: { type: 'Object', notCapturedReason: 'depth' } + } + } + } + } + } + } + } + }) + + expect(state.myNestedObj.fields).to.have.deep.property('deepArr', { + type: 'Array', + elements: [{ + type: 'Array', + elements: [{ + type: 'Array', + elements: [{ + type: 'Array', + elements: [{ type: 'Array', notCapturedReason: 'depth' }] + }] + }] + }] + }) + }) + + setAndTriggerBreakpoint(target, 9) + }) + + it('should return expected object for nested objects if maxReferenceDepth is missing', function (done) { + assertOnBreakpoint(done, (state) => { + expect(Object.entries(state).length).to.equal(1) + + expect(state).to.have.property('myNestedObj') + expect(state.myNestedObj).to.have.property('type', 'Object') + expect(state.myNestedObj).to.have.property('fields') + expect(Object.entries(state.myNestedObj).length).to.equal(2) + + expect(state.myNestedObj.fields).to.have.deep.property('deepObj', { + type: 'Object', + fields: { + foo: { + type: 'Object', + fields: { + foo: { + type: 'Object', + notCapturedReason: 'depth' + } + } + } + } + }) + + expect(state.myNestedObj.fields).to.have.deep.property('deepArr', { + type: 'Array', + elements: [{ + type: 'Array', + elements: [{ + type: 'Array', + notCapturedReason: 'depth' + }] + }] + }) + }) + + setAndTriggerBreakpoint(target, 9) + }) + }) +}) diff --git a/packages/dd-trace/test/debugger/devtools_client/snapshot/primitives.spec.js b/packages/dd-trace/test/debugger/devtools_client/snapshot/primitives.spec.js new file mode 100644 index 00000000000..a01203fe48f --- /dev/null +++ b/packages/dd-trace/test/debugger/devtools_client/snapshot/primitives.spec.js @@ -0,0 +1,30 @@ +'use strict' + +require('../../../setup/mocha') + +const { getTargetCodePath, enable, teardown, assertOnBreakpoint, setAndTriggerBreakpoint } = require('./utils') + +const target = getTargetCodePath(__filename) + +describe('debugger -> devtools client -> snapshot.getLocalStateForCallFrame', function () { + describe('primitives', function () { + beforeEach(enable(__filename)) + + afterEach(teardown) + + it('should return expected object for primitives', function (done) { + assertOnBreakpoint(done, (state) => { + expect(Object.keys(state).length).to.equal(7) + expect(state).to.have.deep.property('undef', { type: 'undefined' }) + expect(state).to.have.deep.property('nil', { type: 'null', isNull: true }) + expect(state).to.have.deep.property('bool', { type: 'boolean', value: 'true' }) + expect(state).to.have.deep.property('num', { type: 'number', value: '42' }) + expect(state).to.have.deep.property('bigint', { type: 'bigint', value: '18014398509481982' }) + expect(state).to.have.deep.property('str', { type: 'string', value: 'foo' }) + expect(state).to.have.deep.property('sym', { type: 'symbol', value: 'Symbol(foo)' }) + }) + + setAndTriggerBreakpoint(target, 13) + }) + }) +}) diff --git a/packages/dd-trace/test/debugger/devtools_client/snapshot/scopes.spec.js b/packages/dd-trace/test/debugger/devtools_client/snapshot/scopes.spec.js new file mode 100644 index 00000000000..d02093a4b01 --- /dev/null +++ b/packages/dd-trace/test/debugger/devtools_client/snapshot/scopes.spec.js @@ -0,0 +1,29 @@ +'use strict' + +require('../../../setup/mocha') + +const { getTargetCodePath, enable, teardown, assertOnBreakpoint, setAndTriggerBreakpoint } = require('./utils') + +const target = getTargetCodePath(__filename) + +describe('debugger -> devtools client -> snapshot.getLocalStateForCallFrame', function () { + beforeEach(enable(__filename)) + + afterEach(teardown) + + describe('scopes', function () { + it('should capture expected scopes', function (done) { + assertOnBreakpoint(done, (state) => { + expect(Object.entries(state).length).to.equal(5) + + expect(state).to.have.deep.property('a1', { type: 'number', value: '1' }) + expect(state).to.have.deep.property('a2', { type: 'number', value: '2' }) + expect(state).to.have.deep.property('total', { type: 'number', value: '0' }) + expect(state).to.have.deep.property('i', { type: 'number', value: '0' }) + expect(state).to.have.deep.property('inc', { type: 'number', value: '2' }) + }) + + setAndTriggerBreakpoint(target, 13) + }) + }) +}) diff --git a/packages/dd-trace/test/debugger/devtools_client/snapshot/target-code/complex-types.js b/packages/dd-trace/test/debugger/devtools_client/snapshot/target-code/complex-types.js new file mode 100644 index 00000000000..65e3e7fac48 --- /dev/null +++ b/packages/dd-trace/test/debugger/devtools_client/snapshot/target-code/complex-types.js @@ -0,0 +1,127 @@ +'use strict' + +function run () { + /* eslint-disable no-unused-vars */ + const { + oblit, obnew, arr, regex, date, map, set, wmap, wset, gen, err, fn, bfn, afn, cls, acls, prox, custProx, pPen, + pRes, pRej, tarr, ab, sab, circular, hidden + } = get() + /* eslint-enable no-unused-vars */ + return 'my return value' // breakpoint at this line +} + +// WARNING: Breakpoints present above this line - Any changes to the lines above might influence tests! + +// References to objects used in WeakMap/WeakSet objects to ensure that they are not garbage collected during testing +const ref = { + wmo1: { a: 1 }, + wmo2: { b: 3 }, + wso1: { a: 1 }, + wso2: { a: 2 }, + wso3: { a: 3 } +} + +// warp it all in a single function to avoid spamming the closure scope with a lot of variables (makes testing simpler) +function get () { + let e, g + const oblit = { + a: 1, + 'b.b': 2, + [Symbol('c')]: 3, + // Has no side-effect + // TODO: At some point it would be great to detect this and get the value, + // though currently we can neither detect it, nor execute the getter. + get d () { + return 4 + }, + // Has side-effect: We should never try to fetch this! + get e () { + e = Math.random() + return e + }, + // Only setter + set f (v) {}, // eslint-disable-line accessor-pairs + // Both getter and setter + get g () { return g }, + set g (x) { g = x } + } + + function fnWithProperties (a, b) {} + fnWithProperties.foo = { bar: 42 } + + class MyClass { + #secret = 42 + constructor () { + this.foo = this.#secret + } + } + + function * makeIterator () { + yield 1 + yield 2 + } + const gen = makeIterator() + gen.foo = 42 + + class CustomError extends Error { + constructor (...args) { + super(...args) + this.foo = 42 + } + } + const err = new CustomError('boom!') + + const buf1 = Buffer.from('IBM') + const buf2 = Buffer.from('hello\x01\x02\x03world') + + const arrayBuffer = new ArrayBuffer(buf1.length) + const sharedArrayBuffer = new SharedArrayBuffer(buf2.length) + + const typedArray = new Int8Array(arrayBuffer) + for (let i = 0; i < buf1.length; i++) typedArray[i] = buf1[i] - 1 + + const sharedTypedArray = new Int8Array(sharedArrayBuffer) + for (let i = 0; i < buf2.length; i++) sharedTypedArray[i] = buf2[i] + + const complexTypes = { + oblit, + obnew: new MyClass(), + arr: [1, 2, 3], + regex: /foo/, + date: new Date('2024-09-20T07:22:59.998Z'), + map: new Map([[1, 2], [3, 4]]), + set: new Set([[1, 2], 3, 4]), + wmap: new WeakMap([[ref.wmo1, 2], [ref.wmo2, 4]]), + wset: new WeakSet([ref.wso1, ref.wso2, ref.wso3]), + gen, + err, + fn: fnWithProperties, + bfn: fnWithProperties.bind(new MyClass(), 1, 2), + afn: () => { return 42 }, + cls: MyClass, + acls: class + {}, // eslint-disable-line indent, brace-style + prox: new Proxy({ target: true }, { get () { return false } }), + custProx: new Proxy(new MyClass(), { get () { return false } }), + pPen: new Promise(() => {}), + pRes: Promise.resolve('resolved value'), + pRej: Promise.reject('rejected value'), // eslint-disable-line prefer-promise-reject-errors + tarr: typedArray, // TODO: Should we test other TypedArray's? + ab: arrayBuffer, + sab: sharedArrayBuffer + } + + complexTypes.circular = complexTypes + + Object.defineProperty(complexTypes, 'hidden', { + value: 'secret', + enumerable: false + }) + + // ensure we don't get an unhandled promise rejection error + complexTypes.pRej.catch(() => {}) + + return complexTypes +} + +module.exports = { run } diff --git a/packages/dd-trace/test/debugger/devtools_client/snapshot/target-code/max-collection-size.js b/packages/dd-trace/test/debugger/devtools_client/snapshot/target-code/max-collection-size.js new file mode 100644 index 00000000000..09c8ca81100 --- /dev/null +++ b/packages/dd-trace/test/debugger/devtools_client/snapshot/target-code/max-collection-size.js @@ -0,0 +1,27 @@ +'use stict' + +function run () { + const arr = [] + const map = new Map() + const set = new Set() + const wmap = new WeakMap() + const wset = new WeakSet() + const typedArray = new Uint16Array(new ArrayBuffer(2000)) + + // 1000 is larger the default maxCollectionSize of 100 + for (let i = 1; i <= 1000; i++) { + // A reference that can be used in WeakMap/WeakSet to avoid GC + const obj = { i } + + arr.push(i) + map.set(i, obj) + set.add(i) + wmap.set(obj, i) + wset.add(obj) + typedArray[i - 1] = i + } + + return 'my return value' // breakpoint at this line +} + +module.exports = { run } diff --git a/packages/dd-trace/test/debugger/devtools_client/snapshot/target-code/max-reference-depth.js b/packages/dd-trace/test/debugger/devtools_client/snapshot/target-code/max-reference-depth.js new file mode 100644 index 00000000000..4c80d2098f9 --- /dev/null +++ b/packages/dd-trace/test/debugger/devtools_client/snapshot/target-code/max-reference-depth.js @@ -0,0 +1,12 @@ +'use strict' + +function run () { + // eslint-disable-next-line no-unused-vars + const myNestedObj = { + deepObj: { foo: { foo: { foo: { foo: { foo: true } } } } }, + deepArr: [[[[[42]]]]] + } + return 'my return value' // breakpoint at this line +} + +module.exports = { run } diff --git a/packages/dd-trace/test/debugger/devtools_client/snapshot/target-code/primitives.js b/packages/dd-trace/test/debugger/devtools_client/snapshot/target-code/primitives.js new file mode 100644 index 00000000000..eba86269a4d --- /dev/null +++ b/packages/dd-trace/test/debugger/devtools_client/snapshot/target-code/primitives.js @@ -0,0 +1,16 @@ +'use strict' + +function run () { + /* eslint-disable no-unused-vars */ + const undef = undefined + const nil = null + const bool = true + const num = 42 + const bigint = BigInt(Number.MAX_SAFE_INTEGER) * 2n + const str = 'foo' + const sym = Symbol('foo') + /* eslint-enable no-unused-vars */ + return 'my return value' // breakpoint at this line +} + +module.exports = { run } diff --git a/packages/dd-trace/test/debugger/devtools_client/snapshot/target-code/scopes.js b/packages/dd-trace/test/debugger/devtools_client/snapshot/target-code/scopes.js new file mode 100644 index 00000000000..e9f771f7226 --- /dev/null +++ b/packages/dd-trace/test/debugger/devtools_client/snapshot/target-code/scopes.js @@ -0,0 +1,17 @@ +'use strict' + +/* eslint-disable no-unused-vars */ +const foo = 'foo' +const bar = 'bar' +/* eslint-enable no-unused-vars */ + +function run (a1 = 1, a2 = 2) { + let total = 0 + for (let i = 0; i < 3; i++) { + const inc = 2 + // eslint-disable-next-line no-unused-vars + total += inc // breakpoint at this line + } +} + +module.exports = { run } diff --git a/packages/dd-trace/test/debugger/devtools_client/snapshot/utils.js b/packages/dd-trace/test/debugger/devtools_client/snapshot/utils.js new file mode 100644 index 00000000000..215b93a4002 --- /dev/null +++ b/packages/dd-trace/test/debugger/devtools_client/snapshot/utils.js @@ -0,0 +1,92 @@ +'use strict' + +const { join, basename } = require('path') + +const inspector = require('../../../../src/debugger/devtools_client/inspector_promises_polyfill') +const session = new inspector.Session() +session.connect() + +session['@noCallThru'] = true +proxyquire('../src/debugger/devtools_client/snapshot/collector', { + '../session': session +}) + +const { getLocalStateForCallFrame } = require('../../../../src/debugger/devtools_client/snapshot') + +module.exports = { + session, + getTargetCodePath, + enable, + teardown, + setAndTriggerBreakpoint, + assertOnBreakpoint +} + +/** + * @param {string} caller - The filename of the calling spec file (hint: `__filename`) + */ +function getTargetCodePath (caller) { + // Convert /path/to/file.spec.js to /path/to/target-code/file.js + const filename = basename(caller) + return caller.replace(filename, join('target-code', filename.replace('.spec', ''))) +} + +/** + * @param {string} caller - The filename of the calling spec file (hint: `__filename`) + */ +function enable (caller) { + const path = getTargetCodePath(caller) + + // The beforeEach hook + return async () => { + // The scriptIds are resolved asynchronously, so to ensure we have an easy way to get them for each script, we + // store a promise on the script that will resolve to its id once it's emitted by Debugger.scriptParsed. + let pResolve = null + const p = new Promise((resolve) => { + pResolve = resolve + }) + p.resolve = pResolve + require(path).scriptId = p + + session.on('Debugger.scriptParsed', ({ params }) => { + if (params.url.endsWith(path)) { + require(path).scriptId.resolve(params.scriptId) + } + }) + + await session.post('Debugger.enable') + } +} + +async function teardown () { + session.removeAllListeners('Debugger.scriptParsed') + session.removeAllListeners('Debugger.paused') + await session.post('Debugger.disable') +} + +async function setAndTriggerBreakpoint (path, line) { + const { run, scriptId } = require(path) + await session.post('Debugger.setBreakpoint', { + location: { + scriptId: await scriptId, + lineNumber: line - 1 // Beware! lineNumber is zero-indexed + } + }) + run() +} + +function assertOnBreakpoint (done, config, callback) { + if (typeof config === 'function') { + callback = config + config = undefined + } + + session.once('Debugger.paused', ({ params }) => { + expect(params.hitBreakpoints.length).to.eq(1) + + getLocalStateForCallFrame(params.callFrames[0], config).then((process) => { + callback(process()) + done() + }).catch(done) + }) +} diff --git a/packages/dd-trace/test/debugger/devtools_client/status.spec.js b/packages/dd-trace/test/debugger/devtools_client/status.spec.js new file mode 100644 index 00000000000..41433f453c5 --- /dev/null +++ b/packages/dd-trace/test/debugger/devtools_client/status.spec.js @@ -0,0 +1,102 @@ +'use strict' + +require('../../setup/mocha') + +const ddsource = 'dd_debugger' +const service = 'my-service' +const runtimeId = 'my-runtime-id' + +describe('diagnostic message http request caching', function () { + let statusproxy, request + + const acks = [ + ['ackReceived', 'RECEIVED'], + ['ackInstalled', 'INSTALLED'], + ['ackEmitting', 'EMITTING'], + ['ackError', 'ERROR', new Error('boom')] + ] + + beforeEach(function () { + request = sinon.spy() + request['@noCallThru'] = true + + statusproxy = proxyquire('../src/debugger/devtools_client/status', { + './config': { service, runtimeId, '@noCallThru': true }, + '../../exporters/common/request': request + }) + }) + + for (const [ackFnName, status, err] of acks) { + describe(ackFnName, function () { + let ackFn, exception + + beforeEach(function () { + if (err) { + ackFn = statusproxy[ackFnName].bind(null, err) + // Use `JSON.stringify` to remove any fields that are `undefined` + exception = JSON.parse(JSON.stringify({ + type: err.code, + message: err.message, + stacktrace: err.stack + })) + } else { + ackFn = statusproxy[ackFnName] + exception = undefined + } + }) + + it('should only call once if no change', function () { + ackFn({ id: 'foo', version: 0 }) + expect(request).to.have.been.calledOnce + assertRequestData(request, { probeId: 'foo', version: 0, status, exception }) + + ackFn({ id: 'foo', version: 0 }) + expect(request).to.have.been.calledOnce + }) + + it('should call again if version changes', function () { + ackFn({ id: 'foo', version: 0 }) + expect(request).to.have.been.calledOnce + assertRequestData(request, { probeId: 'foo', version: 0, status, exception }) + + ackFn({ id: 'foo', version: 1 }) + expect(request).to.have.been.calledTwice + assertRequestData(request, { probeId: 'foo', version: 1, status, exception }) + }) + + it('should call again if probeId changes', function () { + ackFn({ id: 'foo', version: 0 }) + expect(request).to.have.been.calledOnce + assertRequestData(request, { probeId: 'foo', version: 0, status, exception }) + + ackFn({ id: 'bar', version: 0 }) + expect(request).to.have.been.calledTwice + assertRequestData(request, { probeId: 'bar', version: 0, status, exception }) + }) + }) + } +}) + +function assertRequestData (request, { probeId, version, status, exception }) { + const payload = getFormPayload(request) + const diagnostics = { probeId, runtimeId, version, status } + + // Error requests will also contain an `exception` property + if (exception) diagnostics.exception = exception + + expect(payload).to.deep.equal({ ddsource, service, debugger: { diagnostics } }) + + const opts = getRequestOptions(request) + expect(opts).to.have.property('method', 'POST') + expect(opts).to.have.property('path', '/debugger/v1/diagnostics') +} + +function getRequestOptions (request) { + return request.lastCall.args[1] +} + +function getFormPayload (request) { + const form = request.lastCall.args[0] + const payload = form._data[form._data.length - 2] // the last element is an empty line + return JSON.parse(payload) +} diff --git a/packages/dd-trace/test/dogstatsd.spec.js b/packages/dd-trace/test/dogstatsd.spec.js index 0666c559374..18a39d6d304 100644 --- a/packages/dd-trace/test/dogstatsd.spec.js +++ b/packages/dd-trace/test/dogstatsd.spec.js @@ -63,8 +63,8 @@ describe('dogstatsd', () => { }) const dogstatsd = proxyquire('../src/dogstatsd', { - 'dgram': dgram, - 'dns': dns + dgram, + dns }) DogStatsDClient = dogstatsd.DogStatsDClient CustomMetrics = dogstatsd.CustomMetrics @@ -125,6 +125,20 @@ describe('dogstatsd', () => { expect(udp4.send.firstCall.args[4]).to.equal('127.0.0.1') }) + it('should send histograms', () => { + client = new DogStatsDClient() + + client.histogram('test.histogram', 10) + client.flush() + + expect(udp4.send).to.have.been.called + expect(udp4.send.firstCall.args[0].toString()).to.equal('test.histogram:10|h\n') + expect(udp4.send.firstCall.args[1]).to.equal(0) + expect(udp4.send.firstCall.args[2]).to.equal(20) + expect(udp4.send.firstCall.args[3]).to.equal(8125) + expect(udp4.send.firstCall.args[4]).to.equal('127.0.0.1') + }) + it('should send counters', () => { client = new DogStatsDClient() @@ -296,7 +310,7 @@ describe('dogstatsd', () => { client.flush() }) - it('should fail over to UDP', (done) => { + it('should fail over to UDP when receiving HTTP 404 error from agent', (done) => { assertData = () => { setTimeout(() => { try { @@ -321,6 +335,32 @@ describe('dogstatsd', () => { client.flush() }) + it('should fail over to UDP when receiving network error from agent', (done) => { + udp4.send = sinon.stub().callsFake(() => { + try { + expect(udp4.send).to.have.been.called + expect(udp4.send.firstCall.args[0].toString()).to.equal('test.foo:10|c\n') + expect(udp4.send.firstCall.args[2]).to.equal(14) + done() + } catch (e) { + done(e) + } + }) + + statusCode = null + + // host exists but port does not, ECONNREFUSED + client = new DogStatsDClient({ + metricsProxyUrl: 'http://localhost:32700', + host: 'localhost', + port: 8125 + }) + + client.increment('test.foo', 10) + + client.flush() + }) + describe('CustomMetrics', () => { it('.gauge()', () => { client = new CustomMetrics({ dogstatsd: {} }) @@ -381,5 +421,15 @@ describe('dogstatsd', () => { expect(udp4.send).to.have.been.called expect(udp4.send.firstCall.args[0].toString()).to.equal('test.dist:10|d\n') }) + + it('.histogram()', () => { + client = new CustomMetrics({ dogstatsd: {} }) + + client.histogram('test.histogram', 10) + client.flush() + + expect(udp4.send).to.have.been.called + expect(udp4.send.firstCall.args[0].toString()).to.equal('test.histogram:10|h\n') + }) }) }) diff --git a/packages/dd-trace/test/encode/0.4.spec.js b/packages/dd-trace/test/encode/0.4.spec.js index e6db8af12f6..564daf8e92e 100644 --- a/packages/dd-trace/test/encode/0.4.spec.js +++ b/packages/dd-trace/test/encode/0.4.spec.js @@ -44,7 +44,8 @@ describe('encode', () => { example: 1 }, start: 123, - duration: 456 + duration: 456, + links: [] }] }) @@ -183,4 +184,282 @@ describe('encode', () => { expect(decodedData.parent_id.toString(16)).to.equal('1234abcd1234abcd') }) }) + + it('should encode span events', () => { + const encodedLink = '[{"name":"Something went so wrong","time_unix_nano":1000000},' + + '{"name":"I can sing!!! acbdefggnmdfsdv k 2e2ev;!|=xxx","time_unix_nano":1633023102000000,' + + '"attributes":{"emotion":"happy","rating":9.8,"other":[1,9.5,1],"idol":false}}]' + + data[0].meta.events = encodedLink + + encoder.encode(data) + + const buffer = encoder.makePayload() + const decoded = msgpack.decode(buffer, { codec }) + const trace = decoded[0] + expect(trace[0].meta.events).to.deep.equal(encodedLink) + }) + + it('should encode spanLinks', () => { + const traceIdHigh = id('10') + const traceId = id('1234abcd1234abcd') + const rootTid = traceIdHigh.toString(16).padStart(16, '0') + const rootT64 = traceId.toString(16).padStart(16, '0') + const traceIdVal = `${rootTid}${rootT64}` + + const encodedLink = `[{"trace_id":"${traceIdVal}","span_id":"1234abcd1234abcd",` + + '"attributes":{"foo":"bar"},"tracestate":"dd=s:-1;o:foo;t.dm:-4;t.usr.id:bar","flags":1}]' + + data[0].meta['_dd.span_links'] = encodedLink + + encoder.encode(data) + + const buffer = encoder.makePayload() + const decoded = msgpack.decode(buffer, { codec }) + const trace = decoded[0] + expect(trace).to.be.instanceof(Array) + expect(trace[0]).to.be.instanceof(Object) + expect(trace[0].trace_id.toString(16)).to.equal(data[0].trace_id.toString()) + expect(trace[0].span_id.toString(16)).to.equal(data[0].span_id.toString()) + expect(trace[0].parent_id.toString(16)).to.equal(data[0].parent_id.toString()) + expect(trace[0].start.toNumber()).to.equal(123) + expect(trace[0].duration.toNumber()).to.equal(456) + expect(trace[0].name).to.equal(data[0].name) + expect(trace[0].meta).to.deep.equal({ bar: 'baz', '_dd.span_links': encodedLink }) + expect(trace[0].metrics).to.deep.equal({ example: 1 }) + }) + + it('should encode spanLinks with just span and trace id', () => { + const traceId = '00000000000000001234abcd1234abcd' + const spanId = '1234abcd1234abcd' + const encodedLink = `[{"trace_id":"${traceId}","span_id":"${spanId}"}]` + data[0].meta['_dd.span_links'] = encodedLink + encoder.encode(data) + + const buffer = encoder.makePayload() + const decoded = msgpack.decode(buffer, { codec }) + const trace = decoded[0] + expect(trace).to.be.instanceof(Array) + expect(trace[0]).to.be.instanceof(Object) + expect(trace[0].trace_id.toString(16)).to.equal(data[0].trace_id.toString()) + expect(trace[0].span_id.toString(16)).to.equal(data[0].span_id.toString()) + expect(trace[0].parent_id.toString(16)).to.equal(data[0].parent_id.toString()) + expect(trace[0].start.toNumber()).to.equal(123) + expect(trace[0].duration.toNumber()).to.equal(456) + expect(trace[0].name).to.equal(data[0].name) + expect(trace[0].meta).to.deep.equal({ bar: 'baz', '_dd.span_links': encodedLink }) + expect(trace[0].metrics).to.deep.equal({ example: 1 }) + }) + + describe('meta_struct', () => { + it('should encode meta_struct with simple key value object', () => { + const metaStruct = { + foo: 'bar', + baz: 123 + } + data[0].meta_struct = metaStruct + encoder.encode(data) + + const buffer = encoder.makePayload() + + const decoded = msgpack.decode(buffer, { codec }) + const trace = decoded[0] + + expect(msgpack.decode(trace[0].meta_struct.foo)).to.be.equal(metaStruct.foo) + expect(msgpack.decode(trace[0].meta_struct.baz)).to.be.equal(metaStruct.baz) + }) + + it('should ignore array in meta_struct', () => { + const metaStruct = ['one', 2, 'three', 4, 5, 'six'] + data[0].meta_struct = metaStruct + encoder.encode(data) + + const buffer = encoder.makePayload() + + const decoded = msgpack.decode(buffer, { codec }) + const trace = decoded[0] + expect(trace[0].meta_struct).to.deep.equal({}) + }) + + it('should encode meta_struct with empty object and array', () => { + const metaStruct = { + foo: {}, + bar: [] + } + data[0].meta_struct = metaStruct + encoder.encode(data) + + const buffer = encoder.makePayload() + + const decoded = msgpack.decode(buffer, { codec }) + const trace = decoded[0] + expect(msgpack.decode(trace[0].meta_struct.foo)).to.deep.equal(metaStruct.foo) + expect(msgpack.decode(trace[0].meta_struct.bar)).to.deep.equal(metaStruct.bar) + }) + + it('should encode meta_struct with possible real use case', () => { + const metaStruct = { + '_dd.stack': { + exploit: [ + { + type: 'test', + language: 'nodejs', + id: 'someuuid', + message: 'Threat detected', + frames: [ + { + id: 0, + file: 'test.js', + line: 1, + column: 31, + function: 'test' + }, + { + id: 1, + file: 'test2.js', + line: 54, + column: 77, + function: 'test' + }, + { + id: 2, + file: 'test.js', + line: 1245, + column: 41, + function: 'test' + }, + { + id: 3, + file: 'test3.js', + line: 2024, + column: 32, + function: 'test' + } + ] + } + ] + } + } + data[0].meta_struct = metaStruct + + encoder.encode(data) + + const buffer = encoder.makePayload() + + const decoded = msgpack.decode(buffer, { codec }) + const trace = decoded[0] + expect(msgpack.decode(trace[0].meta_struct['_dd.stack'])).to.deep.equal(metaStruct['_dd.stack']) + }) + + it('should encode meta_struct ignoring circular references in objects', () => { + const circular = { + bar: 'baz', + deeper: { + foo: 'bar' + } + } + circular.deeper.circular = circular + const metaStruct = { + foo: circular + } + data[0].meta_struct = metaStruct + + encoder.encode(data) + + const buffer = encoder.makePayload() + + const decoded = msgpack.decode(buffer, { codec }) + const trace = decoded[0] + + const expectedMetaStruct = { + foo: { + bar: 'baz', + deeper: { + foo: 'bar' + } + } + } + expect(msgpack.decode(trace[0].meta_struct.foo)).to.deep.equal(expectedMetaStruct.foo) + }) + + it('should encode meta_struct ignoring circular references in arrays', () => { + const circular = [{ + bar: 'baz' + }] + circular.push(circular) + const metaStruct = { + foo: circular + } + data[0].meta_struct = metaStruct + + encoder.encode(data) + + const buffer = encoder.makePayload() + + const decoded = msgpack.decode(buffer, { codec }) + const trace = decoded[0] + + const expectedMetaStruct = { + foo: [{ + bar: 'baz' + }] + } + expect(msgpack.decode(trace[0].meta_struct.foo)).to.deep.equal(expectedMetaStruct.foo) + }) + + it('should encode meta_struct ignoring undefined properties', () => { + const metaStruct = { + foo: 'bar', + undefinedProperty: undefined + } + data[0].meta_struct = metaStruct + + encoder.encode(data) + + const buffer = encoder.makePayload() + + const decoded = msgpack.decode(buffer, { codec }) + const trace = decoded[0] + + const expectedMetaStruct = { + foo: 'bar' + } + expect(msgpack.decode(trace[0].meta_struct.foo)).to.deep.equal(expectedMetaStruct.foo) + expect(trace[0].meta_struct.undefinedProperty).to.be.undefined + }) + + it('should encode meta_struct ignoring null properties', () => { + const metaStruct = { + foo: 'bar', + nullProperty: null + } + data[0].meta_struct = metaStruct + + encoder.encode(data) + + const buffer = encoder.makePayload() + + const decoded = msgpack.decode(buffer, { codec }) + const trace = decoded[0] + + const expectedMetaStruct = { + foo: 'bar' + } + expect(msgpack.decode(trace[0].meta_struct.foo)).to.deep.equal(expectedMetaStruct.foo) + expect(trace[0].meta_struct.nullProperty).to.be.undefined + }) + + it('should not encode null meta_struct', () => { + data[0].meta_struct = null + + encoder.encode(data) + + const buffer = encoder.makePayload() + + const decoded = msgpack.decode(buffer, { codec }) + const trace = decoded[0] + + expect(trace[0].meta_struct).to.be.undefined + }) + }) }) diff --git a/packages/dd-trace/test/encode/0.5.spec.js b/packages/dd-trace/test/encode/0.5.spec.js index 4da755742ad..ec7b36af08b 100644 --- a/packages/dd-trace/test/encode/0.5.spec.js +++ b/packages/dd-trace/test/encode/0.5.spec.js @@ -36,7 +36,8 @@ describe('encode 0.5', () => { example: 1 }, start: 123123123123123120, - duration: 456456456456456456 + duration: 456456456456456456, + links: [] }] }) @@ -64,6 +65,101 @@ describe('encode 0.5', () => { expect(stringMap[trace[0][11]]).to.equal('') // unset }) + it('should encode span events', () => { + const encodedLink = '[{"name":"Something went so wrong","time_unix_nano":1000000},' + + '{"name":"I can sing!!! acbdefggnmdfsdv k 2e2ev;!|=xxx","time_unix_nano":1633023102000000,' + + '"attributes":{"emotion":"happy","rating":9.8,"other":[1,9.5,1],"idol":false}}]' + + data[0].meta.events = encodedLink + + encoder.encode(data) + + const buffer = encoder.makePayload() + const decoded = msgpack.decode(buffer, { codec }) + const stringMap = decoded[0] + const trace = decoded[1][0] + expect(stringMap).to.include('events') + expect(stringMap).to.include(encodedLink) + expect(trace[0][9]).to.include({ + [stringMap.indexOf('bar')]: stringMap.indexOf('baz'), + [stringMap.indexOf('events')]: stringMap.indexOf(encodedLink) + }) + }) + + it('should encode span links', () => { + const traceIdHigh = id('10') + const traceId = id('1234abcd1234abcd') + const rootTid = traceIdHigh.toString(16).padStart(16, '0') + const rootT64 = traceId.toString(16).padStart(16, '0') + const traceIdVal = `${rootTid}${rootT64}` + + const encodedLink = `[{"trace_id":"${traceIdVal}","span_id":"1234abcd1234abcd",` + + '"attributes":{"foo":"bar"},"tracestate":"dd=s:-1;o:foo;t.dm:-4;t.usr.id:bar","flags":1}]' + + data[0].meta['_dd.span_links'] = encodedLink + + encoder.encode(data) + + const buffer = encoder.makePayload() + const decoded = msgpack.decode(buffer, { codec }) + const stringMap = decoded[0] + const trace = decoded[1][0] + + expect(trace).to.be.instanceof(Array) + expect(trace[0]).to.be.instanceof(Array) + expect(stringMap[trace[0][0]]).to.equal(data[0].service) + expect(stringMap[trace[0][1]]).to.equal(data[0].name) + expect(stringMap[trace[0][2]]).to.equal(data[0].resource) + expect(stringMap).to.include('_dd.span_links') + expect(stringMap).to.include(encodedLink) + expect(trace[0][3].toString(16)).to.equal(data[0].trace_id.toString()) + expect(trace[0][4].toString(16)).to.equal(data[0].span_id.toString()) + expect(trace[0][5].toString(16)).to.equal(data[0].parent_id.toString()) + expect(trace[0][6].toNumber()).to.equal(data[0].start) + expect(trace[0][7].toNumber()).to.equal(data[0].duration) + expect(trace[0][8]).to.equal(0) + expect(trace[0][9]).to.deep.equal({ + [stringMap.indexOf('bar')]: stringMap.indexOf('baz'), + [stringMap.indexOf('_dd.span_links')]: stringMap.indexOf(encodedLink) + }) + expect(trace[0][10]).to.deep.equal({ [stringMap.indexOf('example')]: 1 }) + expect(stringMap[trace[0][11]]).to.equal('') // unset + }) + + it('should encode span link with just span and trace id', () => { + const traceId = '00000000000000001234abcd1234abcd' + const spanId = '1234abcd1234abcd' + const encodedLink = `[{"trace_id":"${traceId}","span_id":"${spanId}"}]` + data[0].meta['_dd.span_links'] = encodedLink + + encoder.encode(data) + + const buffer = encoder.makePayload() + const decoded = msgpack.decode(buffer, { codec }) + const stringMap = decoded[0] + const trace = decoded[1][0] + + expect(trace).to.be.instanceof(Array) + expect(trace[0]).to.be.instanceof(Array) + expect(stringMap[trace[0][0]]).to.equal(data[0].service) + expect(stringMap[trace[0][1]]).to.equal(data[0].name) + expect(stringMap[trace[0][2]]).to.equal(data[0].resource) + expect(stringMap).to.include('_dd.span_links') + expect(stringMap).to.include(encodedLink) + expect(trace[0][3].toString(16)).to.equal(data[0].trace_id.toString()) + expect(trace[0][4].toString(16)).to.equal(data[0].span_id.toString()) + expect(trace[0][5].toString(16)).to.equal(data[0].parent_id.toString()) + expect(trace[0][6].toNumber()).to.equal(data[0].start) + expect(trace[0][7].toNumber()).to.equal(data[0].duration) + expect(trace[0][8]).to.equal(0) + expect(trace[0][9]).to.deep.equal({ + [stringMap.indexOf('bar')]: stringMap.indexOf('baz'), + [stringMap.indexOf('_dd.span_links')]: stringMap.indexOf(encodedLink) + }) + expect(trace[0][10]).to.deep.equal({ [stringMap.indexOf('example')]: 1 }) + expect(stringMap[trace[0][11]]).to.equal('') // unset + }) + it('should truncate long IDs', () => { data[0].trace_id = id('ffffffffffffffff1234abcd1234abcd') data[0].span_id = id('ffffffffffffffff1234abcd1234abcd') @@ -114,4 +210,31 @@ describe('encode 0.5', () => { expect(payload[5]).to.equal(1) expect(payload[11]).to.equal(0) }) + + it('should ignore meta_struct property', () => { + data[0].meta_struct = { foo: 'bar' } + + encoder.encode(data) + + const buffer = encoder.makePayload() + const decoded = msgpack.decode(buffer, { codec }) + const stringMap = decoded[0] + const trace = decoded[1][0] + + expect(trace).to.be.instanceof(Array) + expect(trace[0]).to.be.instanceof(Array) + expect(stringMap[trace[0][0]]).to.equal(data[0].service) + expect(stringMap[trace[0][1]]).to.equal(data[0].name) + expect(stringMap[trace[0][2]]).to.equal(data[0].resource) + expect(trace[0][3].toString(16)).to.equal(data[0].trace_id.toString()) + expect(trace[0][4].toString(16)).to.equal(data[0].span_id.toString()) + expect(trace[0][5].toString(16)).to.equal(data[0].parent_id.toString()) + expect(trace[0][6].toNumber()).to.equal(data[0].start) + expect(trace[0][7].toNumber()).to.equal(data[0].duration) + expect(trace[0][8]).to.equal(0) + expect(trace[0][9]).to.deep.equal({ [stringMap.indexOf('bar')]: stringMap.indexOf('baz') }) + expect(trace[0][10]).to.deep.equal({ [stringMap.indexOf('example')]: 1 }) + expect(stringMap[trace[0][11]]).to.equal('') // unset + expect(trace[0][12]).to.be.undefined // Everything works the same as without meta_struct, and nothing else is added + }) }) diff --git a/packages/dd-trace/test/exporters/agent/exporter.spec.js b/packages/dd-trace/test/exporters/agent/exporter.spec.js index 9bfeb2ec04f..a3e402ba358 100644 --- a/packages/dd-trace/test/exporters/agent/exporter.spec.js +++ b/packages/dd-trace/test/exporters/agent/exporter.spec.js @@ -43,6 +43,18 @@ describe('Exporter', () => { }) }) + it('should pass computed stats header through to writer if standalone appsec is enabled', () => { + const stats = { enabled: false } + const appsec = { standalone: { enabled: true } } + exporter = new Exporter({ url, flushInterval, stats, appsec }, prioritySampler) + + expect(Writer).to.have.been.calledWithMatch({ + headers: { + 'Datadog-Client-Computed-Stats': 'yes' + } + }) + }) + it('should support IPv6', () => { const stats = { enabled: true } exporter = new Exporter({ hostname: '::1', flushInterval, stats }, prioritySampler) @@ -102,6 +114,7 @@ describe('Exporter', () => { beforeEach(() => { exporter = new Exporter({ url }) }) + it('should set the URL on self and writer', () => { exporter.setUrl('http://example2.com') const url = new URL('http://example2.com') diff --git a/packages/dd-trace/test/exporters/common/request.spec.js b/packages/dd-trace/test/exporters/common/request.spec.js index ca2935f63e9..55bcb603a27 100644 --- a/packages/dd-trace/test/exporters/common/request.spec.js +++ b/packages/dd-trace/test/exporters/common/request.spec.js @@ -5,6 +5,7 @@ require('../../setup/tap') const nock = require('nock') const getPort = require('get-port') const http = require('http') +const zlib = require('zlib') const FormData = require('../../../src/exporters/common/form-data') @@ -162,6 +163,7 @@ describe('request', function () { hostname: 'test', port: 123, path: '/' + // eslint-disable-next-line n/handle-callback-err }, (err, res) => { expect(res).to.equal('OK') }) @@ -177,6 +179,7 @@ describe('request', function () { request(Buffer.from(''), { path: '/path', method: 'PUT' + // eslint-disable-next-line n/handle-callback-err }, (err, res) => { expect(res).to.equal('OK') done() @@ -213,6 +216,7 @@ describe('request', function () { request(form, { path: '/path', method: 'PUT' + // eslint-disable-next-line n/handle-callback-err }, (err, res) => { expect(res).to.equal('OK') done() @@ -242,6 +246,7 @@ describe('request', function () { hostname: 'localhost', protocol: 'http:', port: port2 + // eslint-disable-next-line n/handle-callback-err }, (err, res) => { expect(res).to.equal('OK') shutdownFirst() @@ -306,6 +311,36 @@ describe('request', function () { }) }) + it('should calculate correct Content-Length header for multi-byte characters', (done) => { + const sandbox = sinon.createSandbox() + sandbox.spy(http, 'request') + + const body = 'æøå' + const charLength = body.length + const byteLength = Buffer.byteLength(body, 'utf-8') + + expect(charLength).to.be.below(byteLength) + + nock('http://test:123').post('/').reply(200, 'OK') + + request( + body, + { + host: 'test', + port: 123, + method: 'POST', + headers: { 'Content-Type': 'text/plain; charset=utf-8' } + }, + (err, res) => { + expect(res).to.equal('OK') + const { headers } = http.request.getCall(0).args[0] + sandbox.restore() + expect(headers['Content-Length']).to.equal(byteLength) + done(err) + } + ) + }) + describe('when intercepting http', () => { const sandbox = sinon.createSandbox() @@ -343,4 +378,61 @@ describe('request', function () { }) }) }) + + describe('with compressed responses', () => { + it('can decompress gzip responses', (done) => { + const compressedData = zlib.gzipSync(Buffer.from(JSON.stringify({ foo: 'bar' }))) + nock('http://test:123', { + reqheaders: { + 'content-type': 'application/json', + 'accept-encoding': 'gzip' + } + }) + .post('/path') + .reply(200, compressedData, { 'content-encoding': 'gzip' }) + + request(Buffer.from(''), { + protocol: 'http:', + hostname: 'test', + port: 123, + path: '/path', + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'accept-encoding': 'gzip' + } + }, (err, res) => { + expect(res).to.equal(JSON.stringify({ foo: 'bar' })) + done(err) + }) + }) + + it('should ignore badly compressed data and log an error', (done) => { + const badlyCompressedData = 'this is not actually compressed data' + nock('http://test:123', { + reqheaders: { + 'content-type': 'application/json', + 'accept-encoding': 'gzip' + } + }) + .post('/path') + .reply(200, badlyCompressedData, { 'content-encoding': 'gzip' }) + + request(Buffer.from(''), { + protocol: 'http:', + hostname: 'test', + port: 123, + path: '/path', + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'accept-encoding': 'gzip' + } + }, (err, res) => { + expect(log.error).to.have.been.calledWith('Could not gunzip response: unexpected end of file') + expect(res).to.equal('') + done(err) + }) + }) + }) }) diff --git a/packages/dd-trace/src/external-logger/test/index.spec.js b/packages/dd-trace/test/external-logger/index.spec.js similarity index 100% rename from packages/dd-trace/src/external-logger/test/index.spec.js rename to packages/dd-trace/test/external-logger/index.spec.js diff --git a/packages/dd-trace/test/fixtures/config/appsec-blocked-graphql-template.json b/packages/dd-trace/test/fixtures/config/appsec-blocked-graphql-template.json new file mode 100644 index 00000000000..e792d611dd7 --- /dev/null +++ b/packages/dd-trace/test/fixtures/config/appsec-blocked-graphql-template.json @@ -0,0 +1,5 @@ +{ + "errors": { + "message": "blocked" + } +} diff --git a/packages/dd-trace/test/flare.spec.js b/packages/dd-trace/test/flare.spec.js new file mode 100644 index 00000000000..ac4133cd9e9 --- /dev/null +++ b/packages/dd-trace/test/flare.spec.js @@ -0,0 +1,161 @@ +'use strict' + +const Config = require('../src/config') +const { channel } = require('dc-polyfill') +const express = require('express') +const getPort = require('get-port') +const http = require('http') +const upload = require('multer')() +const proxyquire = require('proxyquire').noCallThru() + +require('./setup/tap') + +const debugChannel = channel('datadog:log:debug') + +describe('Flare', () => { + let flare + let startupLog + let tracerConfig + let task + let port + let server + let listener + let socket + let handler + + const createServer = () => { + const app = express() + + app.post('/tracer_flare/v1', upload.any(), (req, res) => { + res.sendStatus(200) + handler(req) + }) + + server = http.createServer(app) + server.on('connection', socket_ => { + socket = socket_ + }) + + listener = server.listen(port) + } + + beforeEach(() => { + startupLog = { + tracerInfo: () => ({ + lang: 'nodejs' + }) + } + + flare = proxyquire('../src/flare', { + '../startup-log': startupLog + }) + + return getPort().then(port_ => { + port = port_ + }) + }) + + beforeEach(() => { + tracerConfig = new Config({ + url: `http://127.0.0.1:${port}` + }) + + task = { + case_id: '111', + hostname: 'myhostname', + user_handle: 'user.name@datadoghq.com' + } + + createServer() + }) + + afterEach(done => { + handler = null + flare.disable() + listener.close() + socket && socket.end() + server.on('close', () => { + server = null + listener = null + socket = null + + done() + }) + }) + + it('should send a flare', done => { + handler = req => { + try { + expect(req.body).to.include({ + case_id: task.case_id, + hostname: task.hostname, + email: task.user_handle, + source: 'tracer_nodejs' + }) + + done() + } catch (e) { + done(e) + } + } + + flare.enable(tracerConfig) + flare.send(task) + }) + + it('should send the tracer info', done => { + handler = req => { + try { + expect(req.files).to.have.length(1) + expect(req.files[0]).to.include({ + fieldname: 'flare_file', + originalname: 'tracer_info.txt', + mimetype: 'application/octet-stream' + }) + + const content = JSON.parse(req.files[0].buffer.toString()) + + expect(content).to.have.property('lang', 'nodejs') + + done() + } catch (e) { + done(e) + } + } + + flare.enable(tracerConfig) + flare.send(task) + }) + + it('should send the tracer logs', done => { + handler = req => { + try { + const file = req.files[0] + + if (file.originalname !== 'tracer_logs.txt') return + + expect(file).to.include({ + fieldname: 'flare_file', + originalname: 'tracer_logs.txt', + mimetype: 'application/octet-stream' + }) + + const content = file.buffer.toString() + + expect(content).to.equal('foo\nbar\n') + + done() + } catch (e) { + done(e) + } + } + + flare.enable(tracerConfig) + flare.prepare('debug') + + debugChannel.publish('foo') + debugChannel.publish('bar') + + flare.send(task) + }) +}) diff --git a/packages/dd-trace/test/format.spec.js b/packages/dd-trace/test/format.spec.js index d6f218156b4..846a02cd66a 100644 --- a/packages/dd-trace/test/format.spec.js +++ b/packages/dd-trace/test/format.spec.js @@ -24,14 +24,20 @@ const ERROR_STACK = constants.ERROR_STACK const ERROR_TYPE = constants.ERROR_TYPE const spanId = id('0234567812345678') +const spanId2 = id('0254567812345678') +const spanId3 = id('0264567812345678') describe('format', () => { let format let span let trace let spanContext + let spanContext2 + let spanContext3 + let TraceState beforeEach(() => { + TraceState = require('../src/opentracing/propagation/tracestate') spanContext = { _traceId: spanId, _spanId: spanId, @@ -40,9 +46,12 @@ describe('format', () => { _metrics: {}, _sampling: {}, _trace: { - started: [] + started: [], + tags: {} }, - _name: 'operation' + _name: 'operation', + toTraceId: sinon.stub().returns(spanId), + toSpanId: sinon.stub().returns(spanId) } span = { @@ -57,10 +66,49 @@ describe('format', () => { spanContext._trace.started.push(span) + spanContext2 = { + ...spanContext, + _traceId: spanId2, + _spanId: spanId2, + _parentId: spanId2, + toTraceId: sinon.stub().returns(spanId2.toString(16)), + toSpanId: sinon.stub().returns(spanId2.toString(16)) + } + spanContext3 = { + ...spanContext, + _traceId: spanId3, + _spanId: spanId3, + _parentId: spanId3, + toTraceId: sinon.stub().returns(spanId3.toString(16)), + toSpanId: sinon.stub().returns(spanId3.toString(16)) + } + format = require('../src/format') }) describe('format', () => { + it('should format span events', () => { + span._events = [ + { name: 'Something went so wrong', startTime: 1 }, + { + name: 'I can sing!!! acbdefggnmdfsdv k 2e2ev;!|=xxx', + attributes: { emotion: 'happy', rating: 9.8, other: [1, 9.5, 1], idol: false }, + startTime: 1633023102 + } + ] + + trace = format(span) + const spanEvents = JSON.parse(trace.meta.events) + expect(spanEvents).to.deep.equal([{ + name: 'Something went so wrong', + time_unix_nano: 1000000 + }, { + name: 'I can sing!!! acbdefggnmdfsdv k 2e2ev;!|=xxx', + time_unix_nano: 1633023102000000, + attributes: { emotion: 'happy', rating: 9.8, other: [1, 9.5, 1], idol: false } + }]) + }) + it('should convert a span to the correct trace format', () => { trace = format(span) @@ -116,14 +164,12 @@ describe('format', () => { }) it('should extract Datadog specific tags', () => { - spanContext._tags['operation.name'] = 'name' spanContext._tags['service.name'] = 'service' spanContext._tags['span.type'] = 'type' spanContext._tags['resource.name'] = 'resource' trace = format(span) - expect(trace.name).to.equal('name') expect(trace.service).to.equal('service') expect(trace.type).to.equal('type') expect(trace.resource).to.equal('resource') @@ -182,6 +228,60 @@ describe('format', () => { ) }) + it('should format span links', () => { + span._links = [ + { + context: spanContext2 + }, + { + context: spanContext3 + } + ] + + trace = format(span) + const spanLinks = JSON.parse(trace.meta['_dd.span_links']) + + expect(spanLinks).to.deep.equal([{ + trace_id: spanId2.toString(16), + span_id: spanId2.toString(16) + }, { + trace_id: spanId3.toString(16), + span_id: spanId3.toString(16) + }]) + }) + + it('creates a span link', () => { + const ts = TraceState.fromString('dd=s:-1;o:foo;t.dm:-4;t.usr.id:bar') + const traceIdHigh = '0000000000000010' + spanContext2._tracestate = ts + spanContext2._trace = { + started: [], + finished: [], + origin: 'synthetics', + tags: { + '_dd.p.tid': traceIdHigh + } + } + + spanContext2._sampling.priority = 0 + const link = { + context: spanContext2, + attributes: { foo: 'bar' } + } + span._links = [link] + + trace = format(span) + const spanLinks = JSON.parse(trace.meta['_dd.span_links']) + + expect(spanLinks).to.deep.equal([{ + trace_id: spanId2.toString(16), + span_id: spanId2.toString(16), + attributes: { foo: 'bar' }, + tracestate: ts.toString(), + flags: 0 + }]) + }) + it('should extract trace chunk tags', () => { spanContext._trace.tags = { chunk: 'test', @@ -264,7 +364,7 @@ describe('format', () => { it('should extract errors', () => { const error = new Error('boom') - spanContext._tags['error'] = error + spanContext._tags.error = error trace = format(span) expect(trace.meta[ERROR_MESSAGE]).to.equal(error.message) @@ -277,7 +377,7 @@ describe('format', () => { error.name = null error.stack = null - spanContext._tags['error'] = error + spanContext._tags.error = error trace = format(span) expect(trace.meta[ERROR_MESSAGE]).to.equal(error.message) @@ -296,12 +396,12 @@ describe('format', () => { it('should add the language tag for a basic span', () => { trace = format(span) - expect(trace.meta['language']).to.equal('javascript') + expect(trace.meta.language).to.equal('javascript') }) describe('when there is an `error` tag ', () => { it('should set the error flag when error tag is true', () => { - spanContext._tags['error'] = true + spanContext._tags.error = true trace = format(span) @@ -309,7 +409,7 @@ describe('format', () => { }) it('should not set the error flag when error is false', () => { - spanContext._tags['error'] = false + spanContext._tags.error = false trace = format(span) @@ -317,21 +417,38 @@ describe('format', () => { }) it('should not extract error to meta', () => { - spanContext._tags['error'] = true + spanContext._tags.error = true trace = format(span) - expect(trace.meta['error']).to.be.undefined + expect(trace.meta.error).to.be.undefined }) }) - it('should set the error flag when there is an error-related tag', () => { + it('should not set the error flag when there is an error-related tag without a set trace tag', () => { spanContext._tags[ERROR_TYPE] = 'Error' spanContext._tags[ERROR_MESSAGE] = 'boom' spanContext._tags[ERROR_STACK] = '' trace = format(span) + expect(trace.error).to.equal(0) + }) + + it('should set the error flag when there is an error-related tag with should setTrace', () => { + spanContext._tags[ERROR_TYPE] = 'Error' + spanContext._tags[ERROR_MESSAGE] = 'boom' + spanContext._tags[ERROR_STACK] = '' + spanContext._tags.setTraceError = 1 + + trace = format(span) + + expect(trace.error).to.equal(1) + + spanContext._tags[ERROR_TYPE] = 'foo' + spanContext._tags[ERROR_MESSAGE] = 'foo' + spanContext._tags[ERROR_STACK] = 'foo' + expect(trace.error).to.equal(1) }) @@ -347,7 +464,7 @@ describe('format', () => { }) it('should not set the error flag for internal spans with error tag', () => { - spanContext._tags['error'] = new Error('boom') + spanContext._tags.error = new Error('boom') spanContext._name = 'fs.operation' trace = format(span) @@ -389,7 +506,7 @@ describe('format', () => { num: '1' } - spanContext._tags['nested'] = tag + spanContext._tags.nested = tag trace = format(span) expect(trace.meta['nested.num']).to.equal('1') @@ -447,8 +564,8 @@ describe('format', () => { it('should not crash on prototype-free tags objects when nesting', () => { const tags = Object.create(null) - tags['nested'] = { foo: 'bar' } - spanContext._tags['nested'] = tags + tags.nested = { foo: 'bar' } + spanContext._tags.nested = tags format(span) }) diff --git a/packages/dd-trace/test/id.spec.js b/packages/dd-trace/test/id.spec.js index 4be8e752402..61d4dd5e054 100644 --- a/packages/dd-trace/test/id.spec.js +++ b/packages/dd-trace/test/id.spec.js @@ -25,7 +25,7 @@ describe('id', () => { sinon.stub(Math, 'random') id = proxyquire('../src/id', { - 'crypto': crypto + crypto }) }) diff --git a/packages/dd-trace/test/lambda/index.spec.js b/packages/dd-trace/test/lambda/index.spec.js index 8d22949dd76..db0e3836c3c 100644 --- a/packages/dd-trace/test/lambda/index.spec.js +++ b/packages/dd-trace/test/lambda/index.spec.js @@ -2,7 +2,6 @@ const path = require('path') -const ritm = require('../../src/lambda/runtime/ritm') const agent = require('../plugins/agent') const oldEnv = process.env @@ -33,7 +32,7 @@ const restoreEnv = () => { */ const loadAgent = ({ exporter = 'agent' } = {}) => { // Make sure the hook is re-registered - ritm.registerLambdaHook() + require('../../src/lambda') return agent.load(null, [], { experimental: { exporter @@ -50,6 +49,8 @@ const closeAgent = () => { // In testing, the patch needs to be deleted from the require cache, // in order to allow multiple handlers being patched properly. delete require.cache[require.resolve('../../src/lambda/runtime/patch.js')] + delete require.cache[require.resolve('../../src/lambda')] + agent.close({ ritmReset: true }) } @@ -199,6 +200,21 @@ describe('lambda', () => { }) await checkTraces }) + + it('doesnt patch lambda when instrumentation is disabled', async () => { + const _handlerPath = path.resolve(__dirname, './fixtures/handler.js') + const handlerBefore = require(_handlerPath).handler + + // Set the desired handler to patch + process.env.DD_TRACE_DISABLED_INSTRUMENTATIONS = 'lambda' + process.env.DD_LAMBDA_HANDLER = 'handler.handler' + // Register hook for patching + await loadAgent() + + // Mock `datadog-lambda` handler resolve and import. + const handlerAfter = require(_handlerPath).handler + expect(handlerBefore).to.equal(handlerAfter) + }) }) describe('timeout spans', () => { @@ -209,7 +225,7 @@ describe('lambda', () => { return closeAgent() }) - it(`doesnt crash when spans are finished early and reached impending timeout`, async () => { + it('doesnt crash when spans are finished early and reached impending timeout', async () => { process.env.DD_LAMBDA_HANDLER = 'handler.finishSpansEarlyTimeoutHandler' await loadAgent() diff --git a/packages/dd-trace/test/leak/tracer.js b/packages/dd-trace/test/leak/tracer.js deleted file mode 100644 index 4d2b408c178..00000000000 --- a/packages/dd-trace/test/leak/tracer.js +++ /dev/null @@ -1,15 +0,0 @@ -'use strict' - -const tracer = require('../..').init() - -const test = require('tape') -const profile = require('../profile') - -test('Tracer should not keep unfinished spans in memory if they are no longer needed', t => { - profile(t, operation) - - function operation (done) { - tracer.startSpan('test') - done() - } -}) diff --git a/packages/dd-trace/test/log.spec.js b/packages/dd-trace/test/log.spec.js index 69980da5fe6..f2ec9a02a1f 100644 --- a/packages/dd-trace/test/log.spec.js +++ b/packages/dd-trace/test/log.spec.js @@ -8,360 +8,431 @@ const { storage } = require('../../datadog-core') /* eslint-disable no-console */ describe('log', () => { - let log - let logger - let error + describe('config', () => { + let env - beforeEach(() => { - sinon.stub(console, 'info') - sinon.stub(console, 'error') - sinon.stub(console, 'warn') - sinon.stub(console, 'debug') - - error = new Error() - - logger = { - debug: sinon.spy(), - error: sinon.spy() - } - - log = require('../src/log') - log.toggle(true) - }) + beforeEach(() => { + env = process.env + process.env = {} + }) - afterEach(() => { - log.reset() - console.info.restore() - console.error.restore() - console.warn.restore() - console.debug.restore() - }) + afterEach(() => { + process.env = env + }) - it('should support chaining', () => { - expect(() => { - log - .use(logger) - .toggle(true) - .error('error') - .debug('debug') - .reset() - }).to.not.throw() - }) + it('should have getConfig function', () => { + const log = require('../src/log') + expect(log.getConfig).to.be.a('function') + }) - it('should call the logger in a noop context', () => { - logger.debug = () => { - expect(storage.getStore()).to.have.property('noop', true) - } + it('should be configured with default config if no environment variables are set', () => { + const log = require('../src/log') + expect(log.getConfig()).to.deep.equal({ + enabled: false, + logger: undefined, + logLevel: 'debug' + }) + }) - log.use(logger).debug('debug') - }) + it('should not be possbile to mutate config object returned by getConfig', () => { + const log = require('../src/log') + const config = log.getConfig() + config.enabled = 1 + config.logger = 1 + config.logLevel = 1 + expect(log.getConfig()).to.deep.equal({ + enabled: false, + logger: undefined, + logLevel: 'debug' + }) + }) - describe('debug', () => { - it('should log to console by default', () => { - log.debug('debug') + it('should initialize from environment variables with DD env vars taking precedence OTEL env vars', () => { + process.env.DD_TRACE_LOG_LEVEL = 'error' + process.env.DD_TRACE_DEBUG = 'false' + process.env.OTEL_LOG_LEVEL = 'debug' + const config = proxyquire('../src/log', {}).getConfig() + expect(config).to.have.property('enabled', false) + expect(config).to.have.property('logLevel', 'error') + }) - expect(console.debug).to.have.been.calledWith('debug') + it('should initialize with OTEL environment variables when DD env vars are not set', () => { + process.env.OTEL_LOG_LEVEL = 'debug' + const config = proxyquire('../src/log', {}).getConfig() + expect(config).to.have.property('enabled', true) + expect(config).to.have.property('logLevel', 'debug') }) - it('should support callbacks that return a message', () => { - log.debug(() => 'debug') + it('should initialize from environment variables', () => { + process.env.DD_TRACE_DEBUG = 'true' + const config = proxyquire('../src/log', {}).getConfig() + expect(config).to.have.property('enabled', true) + }) - expect(console.debug).to.have.been.calledWith('debug') + it('should read case-insensitive booleans from environment variables', () => { + process.env.DD_TRACE_DEBUG = 'TRUE' + const config = proxyquire('../src/log', {}).getConfig() + expect(config).to.have.property('enabled', true) }) }) - describe('error', () => { - it('should log to console by default', () => { - log.error(error) + describe('general usage', () => { + let log + let logger + let error - expect(console.error).to.have.been.calledWith(error) - }) + beforeEach(() => { + sinon.stub(console, 'info') + sinon.stub(console, 'error') + sinon.stub(console, 'warn') + sinon.stub(console, 'debug') + + error = new Error() - it('should support callbacks that return a error', () => { - log.error(() => error) + logger = { + debug: sinon.spy(), + error: sinon.spy() + } - expect(console.error).to.have.been.calledWith(error) + log = proxyquire('../src/log', {}) + log.toggle(true) }) - it('should convert strings to errors', () => { - log.error('error') + afterEach(() => { + log.reset() + console.info.restore() + console.error.restore() + console.warn.restore() + console.debug.restore() + }) - expect(console.error).to.have.been.called - expect(console.error.firstCall.args[0]).to.be.instanceof(Error) - expect(console.error.firstCall.args[0]).to.have.property('message', 'error') + it('should support chaining', () => { + expect(() => { + log + .use(logger) + .toggle(true) + .error('error') + .debug('debug') + .reset() + }).to.not.throw() }) - it('should convert empty values to errors', () => { - log.error() + it('should call the logger in a noop context', () => { + logger.debug = () => { + expect(storage.getStore()).to.have.property('noop', true) + } - expect(console.error).to.have.been.called - expect(console.error.firstCall.args[0]).to.be.instanceof(Error) - expect(console.error.firstCall.args[0]).to.have.property('message', 'undefined') + log.use(logger).debug('debug') }) - it('should convert invalid types to errors', () => { - log.error(123) + describe('debug', () => { + it('should log to console by default', () => { + log.debug('debug') - expect(console.error).to.have.been.called - expect(console.error.firstCall.args[0]).to.be.instanceof(Error) - expect(console.error.firstCall.args[0]).to.have.property('message', '123') - }) + expect(console.debug).to.have.been.calledWith('debug') + }) - it('should reuse error messages for non-errors', () => { - log.error({ message: 'test' }) + it('should support callbacks that return a message', () => { + log.debug(() => 'debug') - expect(console.error).to.have.been.called - expect(console.error.firstCall.args[0]).to.be.instanceof(Error) - expect(console.error.firstCall.args[0]).to.have.property('message', 'test') + expect(console.debug).to.have.been.calledWith('debug') + }) }) - it('should convert messages from callbacks to errors', () => { - log.error(() => 'error') + describe('error', () => { + it('should log to console by default', () => { + log.error(error) - expect(console.error).to.have.been.called - expect(console.error.firstCall.args[0]).to.be.instanceof(Error) - expect(console.error.firstCall.args[0]).to.have.property('message', 'error') - }) - }) + expect(console.error).to.have.been.calledWith(error) + }) - describe('toggle', () => { - it('should disable the logger', () => { - log.toggle(false) - log.debug('debug') - log.error(error) + it('should support callbacks that return a error', () => { + log.error(() => error) - expect(console.debug).to.not.have.been.called - expect(console.error).to.not.have.been.called - }) + expect(console.error).to.have.been.calledWith(error) + }) - it('should enable the logger', () => { - log.toggle(false) - log.toggle(true) - log.debug('debug') - log.error(error) + it('should convert strings to errors', () => { + log.error('error') - expect(console.debug).to.have.been.calledWith('debug') - expect(console.error).to.have.been.calledWith(error) - }) + expect(console.error).to.have.been.called + expect(console.error.firstCall.args[0]).to.be.instanceof(Error) + expect(console.error.firstCall.args[0]).to.have.property('message', 'error') + }) - it('should set minimum log level when enabled with logLevel argument set to a valid string', () => { - log.toggle(true, 'error') - log.debug('debug') - log.error(error) + it('should convert empty values to errors', () => { + log.error() - expect(console.debug).to.not.have.been.called - expect(console.error).to.have.been.calledWith(error) - }) + expect(console.error).to.have.been.called + expect(console.error.firstCall.args[0]).to.be.instanceof(Error) + expect(console.error.firstCall.args[0]).to.have.property('message', 'undefined') + }) - it('should set default log level when enabled with logLevel argument set to an invalid string', () => { - log.toggle(true, 'not a real log level') - log.debug('debug') - log.error(error) + it('should convert invalid types to errors', () => { + log.error(123) - expect(console.debug).to.have.been.calledWith('debug') - expect(console.error).to.have.been.calledWith(error) - }) + expect(console.error).to.have.been.called + expect(console.error.firstCall.args[0]).to.be.instanceof(Error) + expect(console.error.firstCall.args[0]).to.have.property('message', '123') + }) - it('should set min log level when enabled w/logLevel arg set to valid string w/wrong case or whitespace', () => { - log.toggle(true, ' ErRoR ') - log.debug('debug') - log.error(error) + it('should reuse error messages for non-errors', () => { + log.error({ message: 'test' }) - expect(console.debug).to.not.have.been.called - expect(console.error).to.have.been.calledWith(error) - }) + expect(console.error).to.have.been.called + expect(console.error.firstCall.args[0]).to.be.instanceof(Error) + expect(console.error.firstCall.args[0]).to.have.property('message', 'test') + }) - it('should log all log levels greater than or equal to minimum log level', () => { - log.toggle(true, 'debug') - log.debug('debug') - log.error(error) + it('should convert messages from callbacks to errors', () => { + log.error(() => 'error') - expect(console.debug).to.have.been.calledWith('debug') - expect(console.error).to.have.been.calledWith(error) + expect(console.error).to.have.been.called + expect(console.error.firstCall.args[0]).to.be.instanceof(Error) + expect(console.error.firstCall.args[0]).to.have.property('message', 'error') + }) }) - it('should enable default log level when enabled with logLevel argument set to invalid input', () => { - log.toggle(true, ['trace', 'info', 'eror']) - log.debug('debug') - log.error(error) + describe('toggle', () => { + it('should disable the logger', () => { + log.toggle(false) + log.debug('debug') + log.error(error) - expect(console.debug).to.have.been.calledWith('debug') - expect(console.error).to.have.been.calledWith(error) - }) + expect(console.debug).to.not.have.been.called + expect(console.error).to.not.have.been.called + }) - it('should enable default log level when enabled without logLevel argument', () => { - log.toggle(true) - log.debug('debug') - log.error(error) + it('should enable the logger', () => { + log.toggle(false) + log.toggle(true) + log.debug('debug') + log.error(error) - expect(console.debug).to.have.been.calledWith('debug') - expect(console.error).to.have.been.calledWith(error) - }) - }) + expect(console.debug).to.have.been.calledWith('debug') + expect(console.error).to.have.been.calledWith(error) + }) - describe('use', () => { - it('should set the underlying logger when valid', () => { - log.use(logger) - log.debug('debug') - log.error(error) + it('should set minimum log level when enabled with logLevel argument set to a valid string', () => { + log.toggle(true, 'error') + log.debug('debug') + log.error(error) - expect(logger.debug).to.have.been.calledWith('debug') - expect(logger.error).to.have.been.calledWith(error) - }) + expect(console.debug).to.not.have.been.called + expect(console.error).to.have.been.calledWith(error) + }) - it('be a no op with an empty logger', () => { - log.use(null) - log.debug('debug') - log.error(error) + it('should set default log level when enabled with logLevel argument set to an invalid string', () => { + log.toggle(true, 'not a real log level') + log.debug('debug') + log.error(error) - expect(console.debug).to.have.been.calledWith('debug') - expect(console.error).to.have.been.calledWith(error) - }) + expect(console.debug).to.have.been.calledWith('debug') + expect(console.error).to.have.been.calledWith(error) + }) - it('be a no op with an invalid logger', () => { - log.use('invalid') - log.debug('debug') - log.error(error) + it('should set min log level when enabled w/logLevel arg set to valid string w/wrong case or whitespace', () => { + log.toggle(true, ' ErRoR ') + log.debug('debug') + log.error(error) - expect(console.debug).to.have.been.calledWith('debug') - expect(console.error).to.have.been.calledWith(error) - }) - }) + expect(console.debug).to.not.have.been.called + expect(console.error).to.have.been.calledWith(error) + }) - describe('reset', () => { - it('should reset the logger', () => { - log.use(logger) - log.reset() - log.toggle(true) - log.debug('debug') - log.error(error) + it('should log all log levels greater than or equal to minimum log level', () => { + log.toggle(true, 'debug') + log.debug('debug') + log.error(error) - expect(console.debug).to.have.been.calledWith('debug') - expect(console.error).to.have.been.calledWith(error) - }) + expect(console.debug).to.have.been.calledWith('debug') + expect(console.error).to.have.been.calledWith(error) + }) - it('should reset the toggle', () => { - log.use(logger) - log.reset() - log.debug('debug') - log.error(error) + it('should enable default log level when enabled with logLevel argument set to invalid input', () => { + log.toggle(true, ['trace', 'info', 'eror']) + log.debug('debug') + log.error(error) - expect(console.debug).to.not.have.been.called - expect(console.error).to.not.have.been.called - }) + expect(console.debug).to.have.been.calledWith('debug') + expect(console.error).to.have.been.calledWith(error) + }) - it('should reset the minimum log level to defaults', () => { - log.use(logger) - log.toggle(true, 'error') - log.reset() - log.toggle(true) - log.debug('debug') - log.error(error) + it('should enable default log level when enabled without logLevel argument', () => { + log.toggle(true) + log.debug('debug') + log.error(error) - expect(console.debug).to.have.been.calledWith('debug') - expect(console.error).to.have.been.calledWith(error) + expect(console.debug).to.have.been.calledWith('debug') + expect(console.error).to.have.been.calledWith(error) + }) }) - }) - describe('deprecate', () => { - it('should log a deprecation warning', () => { - log.deprecate('test', 'message') + describe('use', () => { + it('should set the underlying logger when valid', () => { + log.use(logger) + log.debug('debug') + log.error(error) - expect(console.error).to.have.been.calledOnce - const consoleErrorArg = console.error.getCall(0).args[0] - expect(typeof consoleErrorArg).to.be.eq('object') - expect(consoleErrorArg.message).to.be.eq('message') - }) + expect(logger.debug).to.have.been.calledWith('debug') + expect(logger.error).to.have.been.calledWith(error) + }) - it('should only log once for a given code', () => { - log.deprecate('test', 'message') - log.deprecate('test', 'message') + it('be a no op with an empty logger', () => { + log.use(null) + log.debug('debug') + log.error(error) - expect(console.error).to.have.been.calledOnce - }) - }) + expect(console.debug).to.have.been.calledWith('debug') + expect(console.error).to.have.been.calledWith(error) + }) - describe('logWriter', () => { - let logWriter - beforeEach(() => { - logWriter = require('../src/log/writer') - }) + it('be a no op with an invalid logger', () => { + log.use('invalid') + log.debug('debug') + log.error(error) - afterEach(() => { - logWriter.reset() + expect(console.debug).to.have.been.calledWith('debug') + expect(console.error).to.have.been.calledWith(error) + }) }) - describe('error', () => { - it('should call logger error', () => { - logWriter.error(error) + describe('reset', () => { + it('should reset the logger', () => { + log.use(logger) + log.reset() + log.toggle(true) + log.debug('debug') + log.error(error) - expect(console.error).to.have.been.calledOnceWith(error) + expect(console.debug).to.have.been.calledWith('debug') + expect(console.error).to.have.been.calledWith(error) }) - it('should call console.error no matter enable flag value', () => { - logWriter.toggle(false) - logWriter.error(error) + it('should reset the toggle', () => { + log.use(logger) + log.reset() + log.debug('debug') + log.error(error) - expect(console.error).to.have.been.calledOnceWith(error) + expect(console.debug).to.not.have.been.called + expect(console.error).to.not.have.been.called }) - }) - describe('warn', () => { - it('should call logger warn', () => { - logWriter.warn('warn') + it('should reset the minimum log level to defaults', () => { + log.use(logger) + log.toggle(true, 'error') + log.reset() + log.toggle(true) + log.debug('debug') + log.error(error) - expect(console.warn).to.have.been.calledOnceWith('warn') + expect(console.debug).to.have.been.calledWith('debug') + expect(console.error).to.have.been.calledWith(error) }) + }) - it('should call logger debug if warn is not provided', () => { - logWriter.use(logger) - logWriter.warn('warn') + describe('deprecate', () => { + it('should log a deprecation warning', () => { + log.deprecate('test', 'message') - expect(logger.debug).to.have.been.calledOnceWith('warn') + expect(console.error).to.have.been.calledOnce + const consoleErrorArg = console.error.getCall(0).args[0] + expect(typeof consoleErrorArg).to.be.eq('object') + expect(consoleErrorArg.message).to.be.eq('message') }) - it('should call console.warn no matter enable flag value', () => { - logWriter.toggle(false) - logWriter.warn('warn') + it('should only log once for a given code', () => { + log.deprecate('test', 'message') + log.deprecate('test', 'message') - expect(console.warn).to.have.been.calledOnceWith('warn') + expect(console.error).to.have.been.calledOnce }) }) - describe('info', () => { - it('should call logger info', () => { - logWriter.info('info') + describe('logWriter', () => { + let logWriter + + beforeEach(() => { + logWriter = require('../src/log/writer') + }) - expect(console.info).to.have.been.calledOnceWith('info') + afterEach(() => { + logWriter.reset() }) - it('should call logger debug if info is not provided', () => { - logWriter.use(logger) - logWriter.info('info') + describe('error', () => { + it('should call logger error', () => { + logWriter.error(error) + + expect(console.error).to.have.been.calledOnceWith(error) + }) + + it('should call console.error no matter enable flag value', () => { + logWriter.toggle(false) + logWriter.error(error) - expect(logger.debug).to.have.been.calledOnceWith('info') + expect(console.error).to.have.been.calledOnceWith(error) + }) }) - it('should call console.info no matter enable flag value', () => { - logWriter.toggle(false) - logWriter.info('info') + describe('warn', () => { + it('should call logger warn', () => { + logWriter.warn('warn') - expect(console.info).to.have.been.calledOnceWith('info') + expect(console.warn).to.have.been.calledOnceWith('warn') + }) + + it('should call logger debug if warn is not provided', () => { + logWriter.use(logger) + logWriter.warn('warn') + + expect(logger.debug).to.have.been.calledOnceWith('warn') + }) + + it('should call console.warn no matter enable flag value', () => { + logWriter.toggle(false) + logWriter.warn('warn') + + expect(console.warn).to.have.been.calledOnceWith('warn') + }) }) - }) - describe('debug', () => { - it('should call logger debug', () => { - logWriter.debug('debug') + describe('info', () => { + it('should call logger info', () => { + logWriter.info('info') - expect(console.debug).to.have.been.calledOnceWith('debug') + expect(console.info).to.have.been.calledOnceWith('info') + }) + + it('should call logger debug if info is not provided', () => { + logWriter.use(logger) + logWriter.info('info') + + expect(logger.debug).to.have.been.calledOnceWith('info') + }) + + it('should call console.info no matter enable flag value', () => { + logWriter.toggle(false) + logWriter.info('info') + + expect(console.info).to.have.been.calledOnceWith('info') + }) }) - it('should call console.debug no matter enable flag value', () => { - logWriter.toggle(false) - logWriter.debug('debug') + describe('debug', () => { + it('should call logger debug', () => { + logWriter.debug('debug') + + expect(console.debug).to.have.been.calledOnceWith('debug') + }) + + it('should call console.debug no matter enable flag value', () => { + logWriter.toggle(false) + logWriter.debug('debug') - expect(console.debug).to.have.been.calledOnceWith('debug') + expect(console.debug).to.have.been.calledOnceWith('debug') + }) }) }) }) diff --git a/packages/dd-trace/test/opentelemetry/context_manager.spec.js b/packages/dd-trace/test/opentelemetry/context_manager.spec.js new file mode 100644 index 00000000000..ebf8f122d87 --- /dev/null +++ b/packages/dd-trace/test/opentelemetry/context_manager.spec.js @@ -0,0 +1,117 @@ +'use strict' + +require('../setup/tap') + +const { expect } = require('chai') +const ContextManager = require('../../src/opentelemetry/context_manager') +const { ROOT_CONTEXT } = require('@opentelemetry/api') +const api = require('@opentelemetry/api') + +describe('OTel Context Manager', () => { + let contextManager + let db + + beforeEach(() => { + contextManager = new ContextManager() + api.context.setGlobalContextManager(contextManager) + db = { + getSomeValue: async () => { + await new Promise(resolve => setTimeout(resolve, 100)) + return { name: 'Dummy Name' } + } + } + }) + + it('should create a new context', () => { + const key1 = api.createContextKey('My first key') + const key2 = api.createContextKey('My second key') + expect(key1.description).to.equal('My first key') + expect(key2.description).to.equal('My second key') + }) + + it('should delete a context', () => { + const key = api.createContextKey('some key') + const ctx = api.ROOT_CONTEXT + const ctx2 = ctx.setValue(key, 'context 2') + + // remove the entry + const ctx3 = ctx.deleteValue(key) + + expect(ctx3.getValue(key)).to.equal(undefined) + expect(ctx2.getValue(key)).to.equal('context 2') + expect(ctx.getValue(key)).to.equal(undefined) + }) + + it('should create a new root context', () => { + const key = api.createContextKey('some key') + const ctx = api.ROOT_CONTEXT + const ctx2 = ctx.setValue(key, 'context 2') + expect(ctx2.getValue(key)).to.equal('context 2') + expect(ctx.getValue(key)).to.equal(undefined) + }) + + it('should return root context', () => { + const ctx = api.context.active() + expect(ctx).to.be.an.instanceof(ROOT_CONTEXT.constructor) + }) + + it('should set active context', () => { + const key = api.createContextKey('Key to store a value') + const ctx = api.context.active() + + api.context.with(ctx.setValue(key, 'context 2'), async () => { + expect(api.context.active().getValue(key)).to.equal('context 2') + }) + }) + + it('should set active context on an asynchronous callback and return the result synchronously', async () => { + const name = await api.context.with(api.context.active(), async () => { + const row = await db.getSomeValue() + return row.name + }) + + expect(name).to.equal('Dummy Name') + }) + + it('should set active contexts in nested functions', async () => { + const key = api.createContextKey('Key to store a value') + const ctx = api.context.active() + expect(api.context.active().getValue(key)).to.equal(undefined) + api.context.with(ctx.setValue(key, 'context 2'), () => { + expect(api.context.active().getValue(key)).to.equal('context 2') + api.context.with(ctx.setValue(key, 'context 3'), () => { + expect(api.context.active().getValue(key)).to.equal('context 3') + }) + expect(api.context.active().getValue(key)).to.equal('context 2') + }) + expect(api.context.active().getValue(key)).to.equal(undefined) + }) + + it('should not modify contexts, instead it should create new context objects', async () => { + const key = api.createContextKey('Key to store a value') + + const ctx = api.context.active() + + const ctx2 = ctx.setValue(key, 'context 2') + expect(ctx.getValue(key)).to.equal(undefined) + expect(ctx).to.be.an.instanceof(ROOT_CONTEXT.constructor) + expect(ctx2.getValue(key)).to.equal('context 2') + + const ret = api.context.with(ctx2, () => { + const ctx3 = api.context.active().setValue(key, 'context 3') + + expect(api.context.active().getValue(key)).to.equal('context 2') + expect(ctx.getValue(key)).to.equal(undefined) + expect(ctx2.getValue(key)).to.equal('context 2') + expect(ctx3.getValue(key)).to.equal('context 3') + + api.context.with(ctx3, () => { + expect(api.context.active().getValue(key)).to.equal('context 3') + }) + expect(api.context.active().getValue(key)).to.equal('context 2') + + return 'return value' + }) + expect(ret).to.equal('return value') + }) +}) diff --git a/packages/dd-trace/test/opentelemetry/span.spec.js b/packages/dd-trace/test/opentelemetry/span.spec.js index 48dd3f6076f..578d92a6224 100644 --- a/packages/dd-trace/test/opentelemetry/span.spec.js +++ b/packages/dd-trace/test/opentelemetry/span.spec.js @@ -2,8 +2,12 @@ require('../setup/tap') -const { expect } = require('chai') +const sinon = require('sinon') +const { performance } = require('perf_hooks') +const { timeOrigin } = performance +const { timeInputToHrTime } = require('@opentelemetry/core') +const { expect } = require('chai') const tracer = require('../../').init() const api = require('@opentelemetry/api') @@ -42,7 +46,6 @@ describe('OTel Span', () => { it('should expose parent span id', () => { tracer.trace('outer', (outer) => { const span = makeSpan('name', {}) - expect(span.parentSpanId).to.equal(outer.context()._spanId.toString(16)) }) }) @@ -186,7 +189,7 @@ describe('OTel Span', () => { it(`should map span name when ${kindName} kind with network.protocol.name`, () => { const span = makeSpan(undefined, { - kind: kind, + kind, attributes: { 'network.protocol.name': 'protocol' } @@ -197,7 +200,7 @@ describe('OTel Span', () => { it(`should map span name when ${kindName} kind without network.protocol.name`, () => { const span = makeSpan(undefined, { - kind: kind + kind }) expect(span.name).to.equal(`${kindName}.request`) @@ -216,7 +219,8 @@ describe('OTel Span', () => { expect(span.name).to.equal(kindName) }) } - it(`should map span name with default span kind of internal`, () => { + + it('should map span name with default span kind of internal', () => { const span = makeSpan() expect(span.name).to.equal('internal') }) @@ -287,6 +291,40 @@ describe('OTel Span', () => { expect(_tags).to.have.property('baz', 'buz') }) + describe('should remap http.response.status_code', () => { + it('should remap when setting attributes', () => { + const span = makeSpan('name') + + const { _tags } = span._ddSpan.context() + + span.setAttributes({ 'http.response.status_code': 200 }) + expect(_tags).to.have.property('http.status_code', '200') + }) + + it('should remap when setting singular attribute', () => { + const span = makeSpan('name') + + const { _tags } = span._ddSpan.context() + + span.setAttribute('http.response.status_code', 200) + expect(_tags).to.have.property('http.status_code', '200') + }) + }) + + it('should set span links', () => { + const span = makeSpan('name') + const span2 = makeSpan('name2') + const span3 = makeSpan('name3') + + const { _links } = span._ddSpan + + span.addLink(span2.spanContext()) + expect(_links).to.have.lengthOf(1) + + span.addLink(span3.spanContext()) + expect(_links).to.have.lengthOf(2) + }) + it('should set status', () => { const unset = makeSpan('name') const unsetCtx = unset._ddSpan.context() @@ -313,6 +351,40 @@ describe('OTel Span', () => { } } + const error = new TestError() + const datenow = Date.now() + span.recordException(error, datenow) + + const { _tags } = span._ddSpan.context() + expect(_tags).to.have.property(ERROR_TYPE, error.name) + expect(_tags).to.have.property(ERROR_MESSAGE, error.message) + expect(_tags).to.have.property(ERROR_STACK, error.stack) + + const events = span._ddSpan._events + expect(events).to.have.lengthOf(1) + expect(events).to.deep.equal([{ + name: error.name, + attributes: { + 'exception.message': error.message, + 'exception.stacktrace': error.stack + }, + startTime: datenow + }]) + }) + + it('should record exception without passing in time', () => { + const stub = sinon.stub(performance, 'now').returns(60000) + const span = makeSpan('name') + + class TestError extends Error { + constructor () { + super('test message') + } + } + + const time = timeInputToHrTime(60000 + timeOrigin) + const timeInMilliseconds = time[0] * 1e3 + time[1] / 1e6 + const error = new TestError() span.recordException(error) @@ -320,6 +392,18 @@ describe('OTel Span', () => { expect(_tags).to.have.property(ERROR_TYPE, error.name) expect(_tags).to.have.property(ERROR_MESSAGE, error.message) expect(_tags).to.have.property(ERROR_STACK, error.stack) + + const events = span._ddSpan._events + expect(events).to.have.lengthOf(1) + expect(events).to.deep.equal([{ + name: error.name, + attributes: { + 'exception.message': error.message, + 'exception.stacktrace': error.stack + }, + startTime: timeInMilliseconds + }]) + stub.restore() }) it('should not set status on already ended spans', () => { @@ -368,4 +452,26 @@ describe('OTel Span', () => { expect(processor.onStart).to.have.been.calledWith(span, span._context) expect(processor.onEnd).to.have.been.calledWith(span) }) + + it('should add span events', () => { + const span1 = makeSpan('span1') + const span2 = makeSpan('span2') + const datenow = Date.now() + span1.addEvent('Web page unresponsive', + { 'error.code': '403', 'unknown values': [1, ['h', 'a', [false]]] }, datenow) + span2.addEvent('Web page loaded') + span2.addEvent('Button changed color', { colors: [112, 215, 70], 'response.time': 134.3, success: true }) + const events1 = span1._ddSpan._events + const events2 = span2._ddSpan._events + expect(events1).to.have.lengthOf(1) + expect(events1).to.deep.equal([{ + name: 'Web page unresponsive', + startTime: datenow, + attributes: { + 'error.code': '403', + 'unknown values': [1] + } + }]) + expect(events2).to.have.lengthOf(2) + }) }) diff --git a/packages/dd-trace/test/opentelemetry/span_context.spec.js b/packages/dd-trace/test/opentelemetry/span_context.spec.js index 2611e4baece..0be1fd7a3dd 100644 --- a/packages/dd-trace/test/opentelemetry/span_context.spec.js +++ b/packages/dd-trace/test/opentelemetry/span_context.spec.js @@ -42,7 +42,9 @@ describe('OTel Span Context', () => { const context = new SpanContext({ traceId }) - expect(context.traceId).to.equal(traceId.toString(16)) + // normalize to 128 bit since that is what otel expects + const normalizedTraceId = traceId.toString(16).padStart(32, '0') + expect(context.traceId).to.equal(normalizedTraceId) }) it('should get span id as hex', () => { diff --git a/packages/dd-trace/test/opentelemetry/tracer.spec.js b/packages/dd-trace/test/opentelemetry/tracer.spec.js index e74ddee72ba..169a3d20ed7 100644 --- a/packages/dd-trace/test/opentelemetry/tracer.spec.js +++ b/packages/dd-trace/test/opentelemetry/tracer.spec.js @@ -183,4 +183,63 @@ describe('OTel Tracer', () => { expect(childContext._parentId).to.not.eql(parentContext._spanId) }) }) + + it('test otel context span parenting', () => { + const tracerProvider = new TracerProvider() + tracerProvider.register() + const otelTracer = new Tracer({}, {}, tracerProvider) + otelTracer.startActiveSpan('otel-root', async (root) => { + await new Promise(resolve => setTimeout(resolve, 200)) + otelTracer.startActiveSpan('otel-parent1', async (parent1) => { + isChildOf(parent1._ddSpan, root._ddSpan) + await new Promise(resolve => setTimeout(resolve, 400)) + otelTracer.startActiveSpan('otel-child1', async (child) => { + isChildOf(child._ddSpan, parent1._ddSpan) + await new Promise(resolve => setTimeout(resolve, 600)) + }) + }) + const orphan1 = otelTracer.startSpan('orphan1') + isChildOf(orphan1._ddSpan, root._ddSpan) + const ctx = api.trace.setSpan(api.context.active(), root) + + otelTracer.startActiveSpan('otel-parent2', ctx, async (parent2) => { + isChildOf(parent2._ddSpan, root._ddSpan) + await new Promise(resolve => setTimeout(resolve, 400)) + const ctx = api.trace.setSpan(api.context.active(), root) + otelTracer.startActiveSpan('otel-child2', ctx, async (child) => { + isChildOf(child._ddSpan, parent2._ddSpan) + await new Promise(resolve => setTimeout(resolve, 600)) + }) + }) + orphan1.end() + }) + }) + + it('test otel context mixed span parenting', () => { + const tracerProvider = new TracerProvider() + tracerProvider.register() + const otelTracer = new Tracer({}, {}, tracerProvider) + otelTracer.startActiveSpan('otel-top-level', async (root) => { + tracer.trace('ddtrace-top-level', async (ddSpan) => { + isChildOf(ddSpan, root._ddSpan) + await new Promise(resolve => setTimeout(resolve, 200)) + tracer.trace('ddtrace-child', async (ddSpanChild) => { + isChildOf(ddSpanChild, ddSpan) + await new Promise(resolve => setTimeout(resolve, 400)) + }) + + otelTracer.startActiveSpan('otel-child', async (otelSpan) => { + isChildOf(otelSpan._ddSpan, ddSpan) + await new Promise(resolve => setTimeout(resolve, 200)) + tracer.trace('ddtrace-grandchild', async (ddSpanGrandchild) => { + isChildOf(ddSpanGrandchild, otelSpan._ddSpan) + otelTracer.startActiveSpan('otel-grandchild', async (otelGrandchild) => { + isChildOf(otelGrandchild._ddSpan, ddSpanGrandchild) + await new Promise(resolve => setTimeout(resolve, 200)) + }) + }) + }) + }) + }) + }) }) diff --git a/packages/dd-trace/test/opentracing/propagation/log.spec.js b/packages/dd-trace/test/opentracing/propagation/log.spec.js index 95ff1ce82ac..50c815f1b7c 100644 --- a/packages/dd-trace/test/opentracing/propagation/log.spec.js +++ b/packages/dd-trace/test/opentracing/propagation/log.spec.js @@ -76,6 +76,25 @@ describe('LogPropagator', () => { expect(carrier.dd).to.have.property('span_id', '456') }) + it('should correctly inject 128 bit trace ids when _dd.p.tid is present', () => { + config.traceId128BitLoggingEnabled = true + const carrier = {} + const traceId = id('4e2a9c1573d240b1a3b7e3c1d4c2f9a7', 16) + const traceIdTag = '8765432187654321' + const spanContext = new SpanContext({ + traceId, + spanId: id('456', 10) + }) + + spanContext._trace.tags['_dd.p.tid'] = traceIdTag + + propagator.inject(spanContext, carrier) + + expect(carrier).to.have.property('dd') + expect(carrier.dd).to.have.property('trace_id', '4e2a9c1573d240b1a3b7e3c1d4c2f9a7') + expect(carrier.dd).to.have.property('span_id', '456') + }) + it('should not inject 128-bit trace IDs when disabled', () => { const carrier = {} const traceId = id('123', 10) diff --git a/packages/dd-trace/test/opentracing/propagation/text_map.spec.js b/packages/dd-trace/test/opentracing/propagation/text_map.spec.js index 469126010f0..5b7fef68092 100644 --- a/packages/dd-trace/test/opentracing/propagation/text_map.spec.js +++ b/packages/dd-trace/test/opentracing/propagation/text_map.spec.js @@ -6,11 +6,15 @@ const Config = require('../../../src/config') const id = require('../../../src/id') const SpanContext = require('../../../src/opentracing/span_context') const TraceState = require('../../../src/opentracing/propagation/tracestate') +const { channel } = require('dc-polyfill') const { AUTO_KEEP, AUTO_REJECT, USER_KEEP } = require('../../../../../ext/priority') const { SAMPLING_MECHANISM_MANUAL } = require('../../../src/constants') const { expect } = require('chai') +const injectCh = channel('dd-trace:span:inject') +const extractCh = channel('dd-trace:span:extract') + describe('TextMapPropagator', () => { let TextMapPropagator let propagator @@ -23,6 +27,7 @@ describe('TextMapPropagator', () => { const spanContext = new SpanContext({ traceId: id('123', 10), spanId: id('456', 10), + isRemote: params.isRemote === undefined ? true : params.isRemote, baggageItems, ...params, trace: { @@ -53,6 +58,16 @@ describe('TextMapPropagator', () => { } }) + it('should not crash without spanContext', () => { + const carrier = {} + propagator.inject(null, carrier) + }) + + it('should not crash without carrier', () => { + const spanContext = createContext() + propagator.inject(spanContext, null) + }) + it('should inject the span context into the carrier', () => { const carrier = {} const spanContext = createContext() @@ -116,7 +131,7 @@ describe('TextMapPropagator', () => { trace: { tags: { '_dd.p.foo': 'foo', - 'bar': 'bar', + bar: 'bar', '_dd.p.baz': 'baz' } } @@ -162,7 +177,7 @@ describe('TextMapPropagator', () => { const spanContext = createContext({ trace: { tags: { - '_ddupefoo': 'value' + _ddupefoo: 'value' } } }) @@ -256,7 +271,8 @@ describe('TextMapPropagator', () => { priority: USER_KEEP, mechanism: SAMPLING_MECHANISM_MANUAL }, - tracestate: TraceState.fromString('other=bleh,dd=s:2;o:foo_bar_;t.dm:-4') + tracestate: TraceState.fromString('other=bleh,dd=s:2;o:foo_bar_;t.dm:-4'), + isRemote: false }) // Include invalid characters to verify underscore conversion spanContext._trace.origin = 'foo,bar=' @@ -265,11 +281,12 @@ describe('TextMapPropagator', () => { config.tracePropagationStyle.inject = ['tracecontext'] propagator.inject(spanContext, carrier) + expect(spanContext._isRemote).to.equal(false) expect(carrier).to.have.property('traceparent', '00-1111aaaa2222bbbb3333cccc4444dddd-5555eeee6666ffff-01') expect(carrier).to.have.property( 'tracestate', - 'dd=t.foo_bar_baz_:abc_!@#$%^&*()_+`-~;s:2;o:foo_bar~;t.dm:4,other=bleh' + 'dd=t.foo_bar_baz_:abc_!@#$%^&*()_+`-~;p:5555eeee6666ffff;s:2;o:foo_bar~;t.dm:-4,other=bleh' ) }) @@ -316,6 +333,26 @@ describe('TextMapPropagator', () => { expect(carrier).to.not.have.property('x-datadog-origin') expect(carrier).to.not.have.property('x-datadog-tags') }) + + it('should publish spanContext and carrier', () => { + const carrier = {} + const spanContext = createContext({ + traceId: id('0000000000000123'), + spanId: id('0000000000000456') + }) + + const onSpanInject = sinon.stub() + injectCh.subscribe(onSpanInject) + + propagator.inject(spanContext, carrier) + + try { + expect(onSpanInject).to.be.calledOnce + expect(onSpanInject.firstCall.args[0]).to.be.deep.equal({ spanContext, carrier }) + } finally { + injectCh.unsubscribe(onSpanInject) + } + }) }) describe('extract', () => { @@ -325,7 +362,8 @@ describe('TextMapPropagator', () => { expect(spanContext.toTraceId()).to.equal(carrier['x-datadog-trace-id']) expect(spanContext.toSpanId()).to.equal(carrier['x-datadog-parent-id']) - expect(spanContext._baggageItems['foo']).to.equal(carrier['ot-baggage-foo']) + expect(spanContext._baggageItems.foo).to.equal(carrier['ot-baggage-foo']) + expect(spanContext._isRemote).to.equal(true) }) it('should convert signed IDs to unsigned', () => { @@ -456,7 +494,7 @@ describe('TextMapPropagator', () => { const traceId = '1111aaaa2222bbbb3333cccc4444dddd' const spanId = '5555eeee6666ffff' - textMap['traceparent'] = `00-${traceId}-${spanId}-01` + textMap.traceparent = `00-${traceId}-${spanId}-01` const first = propagator.extract(textMap) @@ -464,9 +502,15 @@ describe('TextMapPropagator', () => { expect(first._spanId.toString(16)).to.equal(spanId) }) + it('should not crash with invalid traceparent', () => { + textMap.traceparent = 'invalid' + + propagator.extract(textMap) + }) + it('should always extract tracestate from tracecontext when trace IDs match', () => { - textMap['traceparent'] = '00-0000000000000000000000000000007B-0000000000000456-01' - textMap['tracestate'] = 'other=bleh,dd=t.foo_bar_baz_:abc_!@#$%^&*()_+`-~;s:2;o:foo;t.dm:-4' + textMap.traceparent = '00-0000000000000000000000000000007B-0000000000000456-01' + textMap.tracestate = 'other=bleh,dd=t.foo_bar_baz_:abc_!@#$%^&*()_+`-~;s:2;o:foo;t.dm:-4' config.tracePropagationStyle.extract = ['datadog', 'tracecontext'] const carrier = textMap @@ -475,9 +519,30 @@ describe('TextMapPropagator', () => { expect(spanContext._tracestate.get('other')).to.equal('bleh') }) - it(`should not extract tracestate from tracecontext when trace IDs don't match`, () => { - textMap['traceparent'] = '00-00000000000000000000000000000789-0000000000000456-01' - textMap['tracestate'] = 'other=bleh,dd=t.foo_bar_baz_:abc_!@#$%^&*()_+`-~;s:2;o:foo;t.dm:-4' + it('should extract the last datadog parent id from tracestate when p dd member is availible', () => { + textMap.traceparent = '00-0000000000000000000000000000007B-0000000000000456-01' + textMap.tracestate = 'other=bleh,dd=s:2;o:foo;p:2244eeee6666aaaa' + config.tracePropagationStyle.extract = ['tracecontext'] + + const carrier = textMap + const spanContext = propagator.extract(carrier) + + expect(spanContext._trace.tags).to.have.property('_dd.parent_id', '2244eeee6666aaaa') + }) + + it('should set the last datadog parent id to zero when p: is NOT in the tracestate', () => { + textMap.traceparent = '00-0000000000000000000000000000007B-0000000000000456-01' + textMap.tracestate = 'other=gg,dd=s:-1;' + config.tracePropagationStyle.extract = ['tracecontext'] + + const carrier = textMap + const spanContext = propagator.extract(carrier) + expect(spanContext._trace.tags).to.not.have.property('_dd.parent_id') + }) + + it('should not extract tracestate from tracecontext when trace IDs don\'t match', () => { + textMap.traceparent = '00-00000000000000000000000000000789-0000000000000456-01' + textMap.tracestate = 'other=bleh,dd=t.foo_bar_baz_:abc_!@#$%^&*()_+`-~;s:2;o:foo;t.dm:-4' config.tracePropagationStyle.extract = ['datadog', 'tracecontext'] const carrier = textMap @@ -486,9 +551,9 @@ describe('TextMapPropagator', () => { expect(spanContext._tracestate).to.be.undefined }) - it(`should not extract tracestate from tracecontext when configured to extract first`, () => { - textMap['traceparent'] = '00-0000000000000000000000000000007B-0000000000000456-01' - textMap['tracestate'] = 'other=bleh,dd=t.foo_bar_baz_:abc_!@#$%^&*()_+`-~;s:2;o:foo;t.dm:-4' + it('should not extract tracestate from tracecontext when configured to extract first', () => { + textMap.traceparent = '00-0000000000000000000000000000007B-0000000000000456-01' + textMap.tracestate = 'other=bleh,dd=t.foo_bar_baz_:abc_!@#$%^&*()_+`-~;s:2;o:foo;t.dm:-4' config.tracePropagationStyle.extract = ['datadog', 'tracecontext'] config.tracePropagationExtractFirst = true @@ -498,6 +563,49 @@ describe('TextMapPropagator', () => { expect(spanContext._tracestate).to.be.undefined }) + it('extracts span_id from tracecontext headers and stores datadog parent-id in trace_distributed_tags', () => { + textMap['x-datadog-trace-id'] = '61185' + textMap['x-datadog-parent-id'] = '15' + textMap.traceparent = '00-0000000000000000000000000000ef01-0000000000011ef0-01' + config.tracePropagationStyle.extract = ['datadog', 'tracecontext'] + + const carrier = textMap + const spanContext = propagator.extract(carrier) + expect(parseInt(spanContext._spanId.toString(), 16)).to.equal(73456) + expect(parseInt(spanContext._traceId.toString(), 16)).to.equal(61185) + expect(spanContext._trace.tags).to.have.property('_dd.parent_id', '000000000000000f') + }) + + it('extracts span_id from tracecontext headers and stores p value from tracestate in trace_distributed_tags', + () => { + textMap['x-datadog-trace-id'] = '61185' + textMap['x-datadog-parent-id'] = '15' + textMap.traceparent = '00-0000000000000000000000000000ef01-0000000000011ef0-01' + textMap.tracestate = 'other=bleh,dd=p:0000000000000001;s:2;o:foo;t.dm:-4' + config.tracePropagationStyle.extract = ['datadog', 'tracecontext'] + + const carrier = textMap + const spanContext = propagator.extract(carrier) + expect(parseInt(spanContext._spanId.toString(), 16)).to.equal(73456) + expect(parseInt(spanContext._traceId.toString(), 16)).to.equal(61185) + expect(spanContext._trace.tags).to.have.property('_dd.parent_id', '0000000000000001') + }) + + it('should publish spanContext and carrier', () => { + const onSpanExtract = sinon.stub() + extractCh.subscribe(onSpanExtract) + + const carrier = textMap + const spanContext = propagator.extract(carrier) + + try { + expect(onSpanExtract).to.be.calledOnce + expect(onSpanExtract.firstCall.args[0]).to.be.deep.equal({ spanContext, carrier }) + } finally { + extractCh.unsubscribe(onSpanExtract) + } + }) + describe('with B3 propagation as multiple headers', () => { beforeEach(() => { config.tracePropagationStyle.extract = ['b3multi'] @@ -584,7 +692,7 @@ describe('TextMapPropagator', () => { }) it('should extract the header', () => { - textMap['b3'] = '0000000000000123-0000000000000456' + textMap.b3 = '0000000000000123-0000000000000456' const carrier = textMap const spanContext = propagator.extract(carrier) @@ -596,7 +704,7 @@ describe('TextMapPropagator', () => { }) it('should extract sampling', () => { - textMap['b3'] = '0000000000000123-0000000000000456-1' + textMap.b3 = '0000000000000123-0000000000000456-1' const carrier = textMap const spanContext = propagator.extract(carrier) @@ -611,7 +719,7 @@ describe('TextMapPropagator', () => { }) it('should support the full syntax', () => { - textMap['b3'] = '0000000000000123-0000000000000456-1-0000000000000789' + textMap.b3 = '0000000000000123-0000000000000456-1-0000000000000789' const carrier = textMap const spanContext = propagator.extract(carrier) @@ -626,7 +734,7 @@ describe('TextMapPropagator', () => { }) it('should support unsampled traces', () => { - textMap['b3'] = '0' + textMap.b3 = '0' const carrier = textMap const spanContext = propagator.extract(carrier) @@ -639,7 +747,7 @@ describe('TextMapPropagator', () => { }) it('should support sampled traces', () => { - textMap['b3'] = '1' + textMap.b3 = '1' const carrier = textMap const spanContext = propagator.extract(carrier) @@ -652,7 +760,7 @@ describe('TextMapPropagator', () => { }) it('should support the debug flag', () => { - textMap['b3'] = 'd' + textMap.b3 = 'd' const carrier = textMap const spanContext = propagator.extract(carrier) @@ -665,7 +773,7 @@ describe('TextMapPropagator', () => { }) it('should skip extraction without the feature flag', () => { - textMap['b3'] = '0000000000000123-0000000000000456-1-0000000000000789' + textMap.b3 = '0000000000000123-0000000000000456-1-0000000000000789' config.tracePropagationStyle.extract = [] @@ -676,7 +784,7 @@ describe('TextMapPropagator', () => { }) it('should support 128-bit trace IDs', () => { - textMap['b3'] = '00000000000002340000000000000123-0000000000000456' + textMap.b3 = '00000000000002340000000000000123-0000000000000456' config.traceId128BitGenerationEnabled = true @@ -695,7 +803,7 @@ describe('TextMapPropagator', () => { }) it('should skip extracting upper bits for 64-bit trace IDs', () => { - textMap['b3'] = '00000000000000000000000000000123-0000000000000456' + textMap.b3 = '00000000000000000000000000000123-0000000000000456' config.traceId128BitGenerationEnabled = true @@ -716,7 +824,7 @@ describe('TextMapPropagator', () => { }) it('should skip extraction without the feature flag', () => { - textMap['traceparent'] = '00-000000000000000000000000000004d2-000000000000162e-01' + textMap.traceparent = '00-000000000000000000000000000004d2-000000000000162e-01' config.tracePropagationStyle.extract = [] const carrier = textMap @@ -725,8 +833,8 @@ describe('TextMapPropagator', () => { }) it('should extract the header', () => { - textMap['traceparent'] = '00-1111aaaa2222bbbb3333cccc4444dddd-5555eeee6666ffff-01' - textMap['tracestate'] = 'other=bleh,dd=t.foo_bar_baz_:abc_!@#$%^&*()_+`-~;s:2;o:foo;t.dm:-4' + textMap.traceparent = '00-1111aaaa2222bbbb3333cccc4444dddd-5555eeee6666ffff-01' + textMap.tracestate = 'other=bleh,dd=t.foo_bar_baz_:abc_!@#$%^&*()_+`-~;s:2;o:foo;t.dm:-4' config.tracePropagationStyle.extract = ['tracecontext'] const carrier = textMap @@ -743,7 +851,7 @@ describe('TextMapPropagator', () => { }) it('should extract a 128-bit trace ID', () => { - textMap['traceparent'] = '00-1111aaaa2222bbbb3333cccc4444dddd-5555eeee6666ffff-01' + textMap.traceparent = '00-1111aaaa2222bbbb3333cccc4444dddd-5555eeee6666ffff-01' config.tracePropagationStyle.extract = ['tracecontext'] config.traceId128BitGenerationEnabled = true @@ -754,7 +862,7 @@ describe('TextMapPropagator', () => { }) it('should skip extracting upper bits for 64-bit trace IDs', () => { - textMap['traceparent'] = '00-00000000000000003333cccc4444dddd-5555eeee6666ffff-01' + textMap.traceparent = '00-00000000000000003333cccc4444dddd-5555eeee6666ffff-01' config.tracePropagationStyle.extract = ['tracecontext'] config.traceId128BitGenerationEnabled = true @@ -766,8 +874,8 @@ describe('TextMapPropagator', () => { }) it('should propagate the version', () => { - textMap['traceparent'] = '01-1111aaaa2222bbbb3333cccc4444dddd-5555eeee6666ffff-01' - textMap['tracestate'] = 'other=bleh,dd=t.foo_bar_baz_:abc_!@#$%^&*()_+`-~;s:2;o:foo;t.dm:-4' + textMap.traceparent = '01-1111aaaa2222bbbb3333cccc4444dddd-5555eeee6666ffff-01' + textMap.tracestate = 'other=bleh,dd=t.foo_bar_baz_:abc_!@#$%^&*()_+`-~;s:2;o:foo;t.dm:-4' config.tracePropagationStyle.extract = ['tracecontext'] const carrier = {} @@ -781,21 +889,36 @@ describe('TextMapPropagator', () => { }) it('should propagate other vendors', () => { - textMap['traceparent'] = '01-1111aaaa2222bbbb3333cccc4444dddd-5555eeee6666ffff-01' - textMap['tracestate'] = 'other=bleh,dd=t.foo_bar_baz_:abc_!@#$%^&*()_+`-~;s:2;o:foo;t.dm:-4' + textMap.traceparent = '01-1111aaaa2222bbbb3333cccc4444dddd-5555eeee6666ffff-01' + textMap.tracestate = 'other=bleh,dd=t.foo_bar_baz_:abc_!@#$%^&*()_+`-~;s:2;o:foo;t.dm:-4' + config.tracePropagationStyle.extract = ['tracecontext'] + + const carrier = {} + const spanContext = propagator.extract(textMap) + + propagator.inject(spanContext, carrier) + + expect(carrier.tracestate).to.include('other=bleh') + }) + + it('should propagate last datadog id', () => { + textMap.traceparent = '01-1111aaaa2222bbbb3333cccc4444dddd-5555eeee6666ffff-01' + textMap.tracestate = 'other=bleh,dd=s:2;o:foo;t.dm:-4;p:4444eeee6666aaaa' config.tracePropagationStyle.extract = ['tracecontext'] const carrier = {} const spanContext = propagator.extract(textMap) + // Ensure the span context is marked as remote (i.e. not generated by the current process) + expect(spanContext._isRemote).to.equal(true) propagator.inject(spanContext, carrier) - expect(carrier['tracestate']).to.include('other=bleh') + expect(carrier.tracestate).to.include('p:4444eeee6666aaaa') }) it('should fix _dd.p.dm if invalid (non-hyphenated) input is received', () => { - textMap['traceparent'] = '01-1111aaaa2222bbbb3333cccc4444dddd-5555eeee6666ffff-01' - textMap['tracestate'] = 'other=bleh,dd=t.foo_bar_baz_:abc_!@#$%^&*()_+`-~;s:2;o:foo;t.dm:4' + textMap.traceparent = '01-1111aaaa2222bbbb3333cccc4444dddd-5555eeee6666ffff-01' + textMap.tracestate = 'other=bleh,dd=t.foo_bar_baz_:abc_!@#$%^&*()_+`-~;s:2;o:foo;t.dm:4' config.tracePropagationStyle.extract = ['tracecontext'] const carrier = {} @@ -808,8 +931,8 @@ describe('TextMapPropagator', () => { }) it('should maintain hyphen prefix when a default mechanism of 0 is received', () => { - textMap['traceparent'] = '01-1111aaaa2222bbbb3333cccc4444dddd-5555eeee6666ffff-01' - textMap['tracestate'] = 'other=bleh,dd=t.foo_bar_baz_:abc_!@#$%^&*()_+`-~;s:2;o:foo;t.dm:-0' + textMap.traceparent = '01-1111aaaa2222bbbb3333cccc4444dddd-5555eeee6666ffff-01' + textMap.tracestate = 'other=bleh,dd=t.foo_bar_baz_:abc_!@#$%^&*()_+`-~;s:2;o:foo;t.dm:-0' config.tracePropagationStyle.extract = ['tracecontext'] const carrier = {} diff --git a/packages/dd-trace/test/opentracing/span.spec.js b/packages/dd-trace/test/opentracing/span.spec.js index 6566faa053c..dbb248eb920 100644 --- a/packages/dd-trace/test/opentracing/span.spec.js +++ b/packages/dd-trace/test/opentracing/span.spec.js @@ -5,6 +5,9 @@ require('../setup/tap') const Config = require('../../src/config') const TextMapPropagator = require('../../src/opentracing/propagation/text_map') +const { channel } = require('dc-polyfill') +const startCh = channel('dd-trace:span:start') + describe('Span', () => { let Span let span @@ -46,7 +49,7 @@ describe('Span', () => { } Span = proxyquire('../src/opentracing/span', { - 'perf_hooks': { + perf_hooks: { performance: { now } @@ -66,6 +69,7 @@ describe('Span', () => { expect(span.context()._traceId).to.deep.equal('123') expect(span.context()._spanId).to.deep.equal('123') + expect(span.context()._isRemote).to.deep.equal(false) }) it('should add itself to the context trace started spans', () => { @@ -148,6 +152,7 @@ describe('Span', () => { expect(span.context()._parentId).to.deep.equal('456') expect(span.context()._baggageItems).to.deep.equal({ foo: 'bar' }) expect(span.context()._trace).to.equal(parent._trace) + expect(span.context()._isRemote).to.equal(false) }) it('should generate a 128-bit trace ID when configured', () => { @@ -161,6 +166,24 @@ describe('Span', () => { expect(span.context()._trace.tags['_dd.p.tid']).to.match(/^[a-f0-9]{8}0{8}$/) }) + it('should be published via dd-trace:span:start channel', () => { + const onSpan = sinon.stub() + startCh.subscribe(onSpan) + + const fields = { + operationName: 'operation' + } + + try { + span = new Span(tracer, processor, prioritySampler, fields) + + expect(onSpan).to.have.been.calledOnce + expect(onSpan.firstCall.args[0]).to.deep.equal({ span, fields }) + } finally { + startCh.unsubscribe(onSpan) + } + }) + describe('tracer', () => { it('should return its parent tracer', () => { span = new Span(tracer, processor, prioritySampler, { operationName: 'operation' }) @@ -202,7 +225,7 @@ describe('Span', () => { traceId: '123', spanId: '456', _baggageItems: { - 'foo': 'bar' + foo: 'bar' }, _trace: { started: ['span'], @@ -216,6 +239,104 @@ describe('Span', () => { }) }) + // TODO are these tests trivial? + describe('links', () => { + it('should allow links to be added', () => { + span = new Span(tracer, processor, prioritySampler, { operationName: 'operation' }) + const span2 = new Span(tracer, processor, prioritySampler, { operationName: 'operation' }) + + span.addLink(span2.context()) + expect(span).to.have.property('_links') + expect(span._links).to.have.lengthOf(1) + }) + + it('sanitizes attributes', () => { + span = new Span(tracer, processor, prioritySampler, { operationName: 'operation' }) + const span2 = new Span(tracer, processor, prioritySampler, { operationName: 'operation' }) + + const attributes = { + foo: 'bar', + baz: 'qux' + } + span.addLink(span2.context(), attributes) + expect(span._links[0].attributes).to.deep.equal(attributes) + }) + + it('sanitizes nested attributes', () => { + span = new Span(tracer, processor, prioritySampler, { operationName: 'operation' }) + const span2 = new Span(tracer, processor, prioritySampler, { operationName: 'operation' }) + + const attributes = { + foo: true, + bar: 'hi', + baz: 1, + qux: [1, 2, 3] + } + + span.addLink(span2.context(), attributes) + expect(span._links[0].attributes).to.deep.equal({ + foo: 'true', + bar: 'hi', + baz: '1', + 'qux.0': '1', + 'qux.1': '2', + 'qux.2': '3' + }) + }) + + it('sanitizes invalid attributes', () => { + span = new Span(tracer, processor, prioritySampler, { operationName: 'operation' }) + const span2 = new Span(tracer, processor, prioritySampler, { operationName: 'operation' }) + const attributes = { + foo: () => {}, + bar: Symbol('bar'), + baz: 'valid' + } + + span.addLink(span2.context(), attributes) + expect(span._links[0].attributes).to.deep.equal({ + baz: 'valid' + }) + }) + }) + + describe('events', () => { + it('should add span events', () => { + span = new Span(tracer, processor, prioritySampler, { operationName: 'operation' }) + + span.addEvent('Web page unresponsive', + { 'error.code': '403', 'unknown values': [1, ['h', 'a', [false]]] }, 1714536311886) + span.addEvent('Web page loaded') + span.addEvent('Button changed color', { colors: [112, 215, 70], 'response.time': 134.3, success: true }) + + const events = span._events + const expectedEvents = [ + { + name: 'Web page unresponsive', + startTime: 1714536311886, + attributes: { + 'error.code': '403', + 'unknown values': [1] + } + }, + { + name: 'Web page loaded', + startTime: 1500000000000 + }, + { + name: 'Button changed color', + attributes: { + colors: [112, 215, 70], + 'response.time': 134.3, + success: true + }, + startTime: 1500000000000 + } + ] + expect(events).to.deep.equal(expectedEvents) + }) + }) + describe('getBaggageItem', () => { it('should get a baggage item', () => { span = new Span(tracer, processor, prioritySampler, { operationName: 'operation' }) diff --git a/packages/dd-trace/test/opentracing/span_context.spec.js b/packages/dd-trace/test/opentracing/span_context.spec.js index d4de792d594..cfa184d433b 100644 --- a/packages/dd-trace/test/opentracing/span_context.spec.js +++ b/packages/dd-trace/test/opentracing/span_context.spec.js @@ -20,6 +20,7 @@ describe('SpanContext', () => { traceId: '123', spanId: '456', parentId: '789', + isRemote: false, name: 'test', isFinished: true, tags: {}, @@ -41,6 +42,7 @@ describe('SpanContext', () => { _traceId: '123', _spanId: '456', _parentId: '789', + _isRemote: false, _name: 'test', _isFinished: true, _tags: {}, @@ -54,7 +56,8 @@ describe('SpanContext', () => { tags: { foo: 'bar' } }, _traceparent: '00-1111aaaa2222bbbb3333cccc4444dddd-5555eeee6666ffff-01', - _tracestate: TraceState.fromString('dd=s:-1;o:foo;t.dm:-4;t.usr.id:bar') + _tracestate: TraceState.fromString('dd=s:-1;o:foo;t.dm:-4;t.usr.id:bar'), + _otelSpanContext: undefined }) }) @@ -68,6 +71,7 @@ describe('SpanContext', () => { _traceId: '123', _spanId: '456', _parentId: null, + _isRemote: true, _name: undefined, _isFinished: false, _tags: {}, @@ -81,7 +85,8 @@ describe('SpanContext', () => { tags: {} }, _traceparent: undefined, - _tracestate: undefined + _tracestate: undefined, + _otelSpanContext: undefined }) }) diff --git a/packages/dd-trace/test/opentracing/tracer.spec.js b/packages/dd-trace/test/opentracing/tracer.spec.js index 0c3fb37fbf3..31e3df79a33 100644 --- a/packages/dd-trace/test/opentracing/tracer.spec.js +++ b/packages/dd-trace/test/opentracing/tracer.spec.js @@ -103,6 +103,14 @@ describe('Tracer', () => { expect(SpanProcessor).to.have.been.calledWith(agentExporter, prioritySampler, config) }) + it('should allow to configure an alternative prioritySampler', () => { + const sampler = {} + tracer = new Tracer(config, sampler) + + expect(AgentExporter).to.have.been.calledWith(config, sampler) + expect(SpanProcessor).to.have.been.calledWith(agentExporter, sampler, config) + }) + describe('startSpan', () => { it('should start a span', () => { fields.tags = { foo: 'bar' } @@ -120,11 +128,12 @@ describe('Tracer', () => { startTime: fields.startTime, hostname: undefined, traceId128BitGenerationEnabled: undefined, - integrationName: undefined + integrationName: undefined, + links: undefined }, true) expect(span.addTags).to.have.been.calledWith({ - 'foo': 'bar' + foo: 'bar' }) expect(testSpan).to.equal(span) @@ -178,7 +187,8 @@ describe('Tracer', () => { startTime: fields.startTime, hostname: os.hostname(), traceId128BitGenerationEnabled: undefined, - integrationName: undefined + integrationName: undefined, + links: undefined }) expect(testSpan).to.equal(span) @@ -219,13 +229,13 @@ describe('Tracer', () => { it('should merge default tracer tags with span tags', () => { config.tags = { - 'foo': 'tracer', - 'bar': 'tracer' + foo: 'tracer', + bar: 'tracer' } fields.tags = { - 'bar': 'span', - 'baz': 'span' + bar: 'span', + baz: 'span' } tracer = new Tracer(config) @@ -235,6 +245,40 @@ describe('Tracer', () => { expect(span.addTags).to.have.been.calledWith(fields.tags) }) + it('If span is granted a service name that differs from the global service name' + + 'ensure spans `version` tag is undefined.', () => { + config.tags = { + foo: 'tracer', + bar: 'tracer' + } + + fields.tags = { + bar: 'span', + baz: 'span', + service: 'new-service' + + } + + tracer = new Tracer(config) + const testSpan = tracer.startSpan('name', fields) + + expect(span.addTags).to.have.been.calledWith(config.tags) + expect(span.addTags).to.have.been.calledWith({ ...fields.tags, version: undefined }) + expect(Span).to.have.been.calledWith(tracer, processor, prioritySampler, { + operationName: 'name', + parent: null, + tags: { + 'service.name': 'new-service' + }, + startTime: fields.startTime, + hostname: undefined, + traceId128BitGenerationEnabled: undefined, + integrationName: undefined, + links: undefined + }) + expect(testSpan).to.equal(span) + }) + it('should start a span with the trace ID generation configuration', () => { config.traceId128BitGenerationEnabled = true tracer = new Tracer(config) @@ -249,7 +293,30 @@ describe('Tracer', () => { startTime: fields.startTime, hostname: undefined, traceId128BitGenerationEnabled: true, - integrationName: undefined + integrationName: undefined, + links: undefined + }) + + expect(testSpan).to.equal(span) + }) + + it('should start a span with span links attached', () => { + const context = new SpanContext() + fields.links = [{ context }] + tracer = new Tracer(config) + const testSpan = tracer.startSpan('name', fields) + + expect(Span).to.have.been.calledWith(tracer, processor, prioritySampler, { + operationName: 'name', + parent: null, + tags: { + 'service.name': 'service' + }, + startTime: fields.startTime, + hostname: undefined, + traceId128BitGenerationEnabled: undefined, + integrationName: undefined, + links: [{ context }] }) expect(testSpan).to.equal(span) diff --git a/packages/dd-trace/test/payload-tagging/index.spec.js b/packages/dd-trace/test/payload-tagging/index.spec.js new file mode 100644 index 00000000000..a4f4da8108e --- /dev/null +++ b/packages/dd-trace/test/payload-tagging/index.spec.js @@ -0,0 +1,220 @@ +const { + PAYLOAD_TAG_REQUEST_PREFIX, + PAYLOAD_TAG_RESPONSE_PREFIX +} = require('../../src/constants') +const { tagsFromObject } = require('../../src/payload-tagging/tagging') +const { computeTags } = require('../../src/payload-tagging') + +const { expect } = require('chai') + +const defaultOpts = { maxDepth: 10, prefix: 'http.payload' } + +describe('Payload tagger', () => { + describe('tag count cutoff', () => { + it('should generate many tags when not reaching the cap', () => { + const belowCap = 200 + const input = { foo: Object.fromEntries([...Array(belowCap).keys()].map(i => [i, i])) } + const tagCount = Object.entries(tagsFromObject(input, defaultOpts)).length + expect(tagCount).to.equal(belowCap) + }) + + it('should stop generating tags once the cap is reached', () => { + const aboveCap = 759 + const input = { foo: Object.fromEntries([...Array(aboveCap).keys()].map(i => [i, i])) } + const tagCount = Object.entries(tagsFromObject(input, defaultOpts)).length + expect(tagCount).to.not.equal(aboveCap) + expect(tagCount).to.equal(758) + }) + }) + + describe('best-effort redacting of keys', () => { + it('should redact disallowed keys', () => { + const input = { + foo: { + bar: { + token: 'tokenpleaseredact', + authorization: 'pleaseredact', + valid: 'valid' + }, + baz: { + password: 'shouldgo', + 'x-authorization': 'shouldbegone', + data: 'shouldstay' + } + } + } + const tags = tagsFromObject(input, defaultOpts) + expect(tags).to.deep.equal({ + 'http.payload.foo.bar.token': 'redacted', + 'http.payload.foo.bar.authorization': 'redacted', + 'http.payload.foo.bar.valid': 'valid', + 'http.payload.foo.baz.password': 'redacted', + 'http.payload.foo.baz.x-authorization': 'redacted', + 'http.payload.foo.baz.data': 'shouldstay' + }) + }) + + it('should redact banned keys even if they are objects', () => { + const input = { + foo: { + authorization: { + token: 'tokenpleaseredact', + authorization: 'pleaseredact', + valid: 'valid' + }, + baz: { + password: 'shouldgo', + 'x-authorization': 'shouldbegone', + data: 'shouldstay' + } + } + } + const tags = tagsFromObject(input, defaultOpts) + expect(tags).to.deep.equal({ + 'http.payload.foo.authorization': 'redacted', + 'http.payload.foo.baz.password': 'redacted', + 'http.payload.foo.baz.x-authorization': 'redacted', + 'http.payload.foo.baz.data': 'shouldstay' + }) + }) + }) + + describe('escaping', () => { + it('should escape `.` characters in individual keys', () => { + const input = { 'foo.bar': { baz: 'quux' } } + const tags = tagsFromObject(input, defaultOpts) + expect(tags).to.deep.equal({ + 'http.payload.foo\\.bar.baz': 'quux' + }) + }) + }) + + describe('parsing', () => { + it('should transform null values to "null" string', () => { + const input = { foo: 'bar', baz: null } + const tags = tagsFromObject(input, defaultOpts) + expect(tags).to.deep.equal({ + 'http.payload.foo': 'bar', + 'http.payload.baz': 'null' + }) + }) + + it('should transform undefined values to "undefined" string', () => { + const input = { foo: 'bar', baz: undefined } + const tags = tagsFromObject(input, defaultOpts) + expect(tags).to.deep.equal({ + 'http.payload.foo': 'bar', + 'http.payload.baz': 'undefined' + }) + }) + + it('should transform boolean values to strings', () => { + const input = { foo: true, bar: false } + const tags = tagsFromObject(input, defaultOpts) + expect(tags).to.deep.equal({ + 'http.payload.foo': 'true', + 'http.payload.bar': 'false' + }) + }) + + it('should decode buffers as UTF-8', () => { + const input = { foo: Buffer.from('bar') } + const tags = tagsFromObject(input, defaultOpts) + expect(tags).to.deep.equal({ 'http.payload.foo': 'bar' }) + }) + + it('should provide tags from simple JSON objects, casting to strings where necessary', () => { + const input = { + foo: { bar: { baz: 1, quux: 2 } }, + asimplestring: 'isastring', + anullvalue: null, + anundefined: undefined + } + const tags = tagsFromObject(input, defaultOpts) + expect(tags).to.deep.equal({ + 'http.payload.foo.bar.baz': '1', + 'http.payload.foo.bar.quux': '2', + 'http.payload.asimplestring': 'isastring', + 'http.payload.anullvalue': 'null', + 'http.payload.anundefined': 'undefined' + }) + }) + + it('should index tags when encountering arrays', () => { + const input = { foo: { bar: { list: ['v0', 'v1', 'v2'] } } } + const tags = tagsFromObject(input, defaultOpts) + expect(tags).to.deep.equal({ + 'http.payload.foo.bar.list.0': 'v0', + 'http.payload.foo.bar.list.1': 'v1', + 'http.payload.foo.bar.list.2': 'v2' + }) + }) + + it('should not replace a real value at max depth', () => { + const input = { + 1: { 2: { 3: { 4: { 5: { 6: { 7: { 8: { 9: { 10: 11 } } } } } } } } } + } + const tags = tagsFromObject(input, defaultOpts) + expect(tags).to.deep.equal({ 'http.payload.1.2.3.4.5.6.7.8.9.10': '11' }) + }) + + it('should truncate paths beyond max depth', () => { + const input = { + 1: { 2: { 3: { 4: { 5: { 6: { 7: { 8: { 9: { 10: { 11: 'too much' } } } } } } } } } } + } + const tags = tagsFromObject(input, defaultOpts) + expect(tags).to.deep.equal({ 'http.payload.1.2.3.4.5.6.7.8.9.10': 'truncated' }) + }) + }) +}) + +describe('Tagging orchestration', () => { + it('should use the request config when given the request prefix', () => { + const config = { + request: ['$.request'], + response: ['$.response'], + expand: [] + } + const input = { + request: 'foo', + response: 'bar' + } + const tags = computeTags(config, input, { maxDepth: 10, prefix: PAYLOAD_TAG_REQUEST_PREFIX }) + expect(tags).to.have.property(`${PAYLOAD_TAG_REQUEST_PREFIX}.request`, 'redacted') + expect(tags).to.have.property(`${PAYLOAD_TAG_REQUEST_PREFIX}.response`, 'bar') + }) + + it('should use the response config when given the response prefix', () => { + const config = { + request: ['$.request'], + response: ['$.response'], + expand: [] + } + const input = { + request: 'foo', + response: 'bar' + } + const tags = computeTags(config, input, { maxDepth: 10, prefix: PAYLOAD_TAG_RESPONSE_PREFIX }) + expect(tags).to.have.property(`${PAYLOAD_TAG_RESPONSE_PREFIX}.response`, 'redacted') + expect(tags).to.have.property(`${PAYLOAD_TAG_RESPONSE_PREFIX}.request`, 'foo') + }) + + it('should apply expansion rules', () => { + const config = { + request: [], + response: [], + expand: ['$.request', '$.response', '$.invalid'] + } + const input = { + request: '{ "foo": "bar" }', + response: '{ "baz": "quux" }', + invalid: '{ invalid JSON }', + untargeted: '{ "foo": "bar" }' + } + const tags = computeTags(config, input, { maxDepth: 10, prefix: 'foo' }) + expect(tags).to.have.property('foo.request.foo', 'bar') + expect(tags).to.have.property('foo.response.baz', 'quux') + expect(tags).to.have.property('foo.invalid', '{ invalid JSON }') + expect(tags).to.have.property('foo.untargeted', '{ "foo": "bar" }') + }) +}) diff --git a/packages/dd-trace/test/payload_tagging.spec.js b/packages/dd-trace/test/payload_tagging.spec.js new file mode 100644 index 00000000000..630c773d567 --- /dev/null +++ b/packages/dd-trace/test/payload_tagging.spec.js @@ -0,0 +1,222 @@ +require('./setup/tap') + +const { + PAYLOAD_TAG_REQUEST_PREFIX, + PAYLOAD_TAG_RESPONSE_PREFIX +} = require('../src/constants') +const { tagsFromObject } = require('../src/payload-tagging/tagging') +const { computeTags } = require('../src/payload-tagging') + +const { expect } = require('chai') + +const defaultOpts = { maxDepth: 10, prefix: 'http.payload' } + +describe('Payload tagger', () => { + describe('tag count cutoff', () => { + it('should generate many tags when not reaching the cap', () => { + const belowCap = 200 + const input = { foo: Object.fromEntries([...Array(belowCap).keys()].map(i => [i, i])) } + const tagCount = Object.entries(tagsFromObject(input, defaultOpts)).length + expect(tagCount).to.equal(belowCap) + }) + + it('should stop generating tags once the cap is reached', () => { + const aboveCap = 759 + const input = { foo: Object.fromEntries([...Array(aboveCap).keys()].map(i => [i, i])) } + const tagCount = Object.entries(tagsFromObject(input, defaultOpts)).length + expect(tagCount).to.not.equal(aboveCap) + expect(tagCount).to.equal(758) + }) + }) + + describe('best-effort redacting of keys', () => { + it('should redact disallowed keys', () => { + const input = { + foo: { + bar: { + token: 'tokenpleaseredact', + authorization: 'pleaseredact', + valid: 'valid' + }, + baz: { + password: 'shouldgo', + 'x-authorization': 'shouldbegone', + data: 'shouldstay' + } + } + } + const tags = tagsFromObject(input, defaultOpts) + expect(tags).to.deep.equal({ + 'http.payload.foo.bar.token': 'redacted', + 'http.payload.foo.bar.authorization': 'redacted', + 'http.payload.foo.bar.valid': 'valid', + 'http.payload.foo.baz.password': 'redacted', + 'http.payload.foo.baz.x-authorization': 'redacted', + 'http.payload.foo.baz.data': 'shouldstay' + }) + }) + + it('should redact banned keys even if they are objects', () => { + const input = { + foo: { + authorization: { + token: 'tokenpleaseredact', + authorization: 'pleaseredact', + valid: 'valid' + }, + baz: { + password: 'shouldgo', + 'x-authorization': 'shouldbegone', + data: 'shouldstay' + } + } + } + const tags = tagsFromObject(input, defaultOpts) + expect(tags).to.deep.equal({ + 'http.payload.foo.authorization': 'redacted', + 'http.payload.foo.baz.password': 'redacted', + 'http.payload.foo.baz.x-authorization': 'redacted', + 'http.payload.foo.baz.data': 'shouldstay' + }) + }) + }) + + describe('escaping', () => { + it('should escape `.` characters in individual keys', () => { + const input = { 'foo.bar': { baz: 'quux' } } + const tags = tagsFromObject(input, defaultOpts) + expect(tags).to.deep.equal({ + 'http.payload.foo\\.bar.baz': 'quux' + }) + }) + }) + + describe('parsing', () => { + it('should transform null values to "null" string', () => { + const input = { foo: 'bar', baz: null } + const tags = tagsFromObject(input, defaultOpts) + expect(tags).to.deep.equal({ + 'http.payload.foo': 'bar', + 'http.payload.baz': 'null' + }) + }) + + it('should transform undefined values to "undefined" string', () => { + const input = { foo: 'bar', baz: undefined } + const tags = tagsFromObject(input, defaultOpts) + expect(tags).to.deep.equal({ + 'http.payload.foo': 'bar', + 'http.payload.baz': 'undefined' + }) + }) + + it('should transform boolean values to strings', () => { + const input = { foo: true, bar: false } + const tags = tagsFromObject(input, defaultOpts) + expect(tags).to.deep.equal({ + 'http.payload.foo': 'true', + 'http.payload.bar': 'false' + }) + }) + + it('should decode buffers as UTF-8', () => { + const input = { foo: Buffer.from('bar') } + const tags = tagsFromObject(input, defaultOpts) + expect(tags).to.deep.equal({ 'http.payload.foo': 'bar' }) + }) + + it('should provide tags from simple JSON objects, casting to strings where necessary', () => { + const input = { + foo: { bar: { baz: 1, quux: 2 } }, + asimplestring: 'isastring', + anullvalue: null, + anundefined: undefined + } + const tags = tagsFromObject(input, defaultOpts) + expect(tags).to.deep.equal({ + 'http.payload.foo.bar.baz': '1', + 'http.payload.foo.bar.quux': '2', + 'http.payload.asimplestring': 'isastring', + 'http.payload.anullvalue': 'null', + 'http.payload.anundefined': 'undefined' + }) + }) + + it('should index tags when encountering arrays', () => { + const input = { foo: { bar: { list: ['v0', 'v1', 'v2'] } } } + const tags = tagsFromObject(input, defaultOpts) + expect(tags).to.deep.equal({ + 'http.payload.foo.bar.list.0': 'v0', + 'http.payload.foo.bar.list.1': 'v1', + 'http.payload.foo.bar.list.2': 'v2' + }) + }) + + it('should not replace a real value at max depth', () => { + const input = { + 1: { 2: { 3: { 4: { 5: { 6: { 7: { 8: { 9: { 10: 11 } } } } } } } } } + } + const tags = tagsFromObject(input, defaultOpts) + expect(tags).to.deep.equal({ 'http.payload.1.2.3.4.5.6.7.8.9.10': '11' }) + }) + + it('should truncate paths beyond max depth', () => { + const input = { + 1: { 2: { 3: { 4: { 5: { 6: { 7: { 8: { 9: { 10: { 11: 'too much' } } } } } } } } } } + } + const tags = tagsFromObject(input, defaultOpts) + expect(tags).to.deep.equal({ 'http.payload.1.2.3.4.5.6.7.8.9.10': 'truncated' }) + }) + }) +}) + +describe('Tagging orchestration', () => { + it('should use the request config when given the request prefix', () => { + const config = { + request: ['$.request'], + response: ['$.response'], + expand: [] + } + const input = { + request: 'foo', + response: 'bar' + } + const tags = computeTags(config, input, { maxDepth: 10, prefix: PAYLOAD_TAG_REQUEST_PREFIX }) + expect(tags).to.have.property(`${PAYLOAD_TAG_REQUEST_PREFIX}.request`, 'redacted') + expect(tags).to.have.property(`${PAYLOAD_TAG_REQUEST_PREFIX}.response`, 'bar') + }) + + it('should use the response config when given the response prefix', () => { + const config = { + request: ['$.request'], + response: ['$.response'], + expand: [] + } + const input = { + request: 'foo', + response: 'bar' + } + const tags = computeTags(config, input, { maxDepth: 10, prefix: PAYLOAD_TAG_RESPONSE_PREFIX }) + expect(tags).to.have.property(`${PAYLOAD_TAG_RESPONSE_PREFIX}.response`, 'redacted') + expect(tags).to.have.property(`${PAYLOAD_TAG_RESPONSE_PREFIX}.request`, 'foo') + }) + + it('should apply expansion rules', () => { + const config = { + request: [], + response: [], + expand: ['$.request', '$.response', '$.invalid'] + } + const input = { + request: '{ "foo": "bar" }', + response: '{ "baz": "quux" }', + invalid: '{ invalid JSON }', + untargeted: '{ "foo": "bar" }' + } + const tags = computeTags(config, input, { maxDepth: 10, prefix: 'foo' }) + expect(tags).to.have.property('foo.request.foo', 'bar') + expect(tags).to.have.property('foo.response.baz', 'quux') + expect(tags).to.have.property('foo.invalid', '{ invalid JSON }') + expect(tags).to.have.property('foo.untargeted', '{ "foo": "bar" }') + }) +}) diff --git a/packages/dd-trace/test/pkg.spec.js b/packages/dd-trace/test/pkg.spec.js index 7eb65f0fe73..0248da089de 100644 --- a/packages/dd-trace/test/pkg.spec.js +++ b/packages/dd-trace/test/pkg.spec.js @@ -12,7 +12,7 @@ describe('pkg', () => { if (os.platform() !== 'win32') { describe('in pre-require', () => { it('should load the package.json correctly', () => { - const pkg = JSON.parse(execSync(`node --require ./pkg-loader.js -e ""`, { + const pkg = JSON.parse(execSync('node --require ./pkg-loader.js -e ""', { cwd: __dirname }).toString()) expect(pkg.name).to.equal('dd-trace') @@ -39,6 +39,6 @@ describe('load', () => { pathStub.parse = function () { return undefined } - proxyquire('../src/pkg', { 'path': pathStub }) + proxyquire('../src/pkg', { path: pathStub }) }) }) diff --git a/packages/dd-trace/test/plugin_manager.spec.js b/packages/dd-trace/test/plugin_manager.spec.js index 62a5b0a1bc9..e497efce321 100644 --- a/packages/dd-trace/test/plugin_manager.spec.js +++ b/packages/dd-trace/test/plugin_manager.spec.js @@ -6,7 +6,7 @@ const { channel } = require('dc-polyfill') const proxyquire = require('proxyquire') const loadChannel = channel('dd-trace:instrumentation:load') -const Nomenclature = require('../../dd-trace/src/service-naming') +const nomenclature = require('../../dd-trace/src/service-naming') describe('Plugin Manager', () => { let tracer @@ -19,7 +19,9 @@ describe('Plugin Manager', () => { let pm beforeEach(() => { - tracer = {} + tracer = { + _nomenclature: nomenclature + } instantiated = [] class FakePlugin { constructor (aTracer) { @@ -83,12 +85,14 @@ describe('Plugin Manager', () => { it('does not throw for old-style plugins', () => { expect(() => pm.configurePlugin('one', false)).to.not.throw() }) + describe('without configure', () => { it('should not configure plugins', () => { pm.configurePlugin('two') loadChannel.publish({ name: 'two' }) expect(Two.prototype.configure).to.not.have.been.called }) + it('should keep the config for future configure calls', () => { pm.configurePlugin('two', { foo: 'bar' }) pm.configure() @@ -99,46 +103,56 @@ describe('Plugin Manager', () => { }) }) }) + describe('without env vars', () => { beforeEach(() => pm.configure()) + it('works with no config param', () => { pm.configurePlugin('two') loadChannel.publish({ name: 'two' }) expect(Two.prototype.configure).to.have.been.calledWithMatch({ enabled: true }) }) + it('works with empty object config', () => { pm.configurePlugin('two', {}) loadChannel.publish({ name: 'two' }) expect(Two.prototype.configure).to.have.been.calledWithMatch({ enabled: true }) }) + it('works with "enabled: false" object config', () => { pm.configurePlugin('two', { enabled: false }) loadChannel.publish({ name: 'two' }) expect(Two.prototype.configure).to.have.been.calledWithMatch({ enabled: false }) }) + it('works with "enabled: true" object config', () => { pm.configurePlugin('two', { enabled: true }) loadChannel.publish({ name: 'two' }) expect(Two.prototype.configure).to.have.been.calledWithMatch({ enabled: true }) }) + it('works with boolean false', () => { pm.configurePlugin('two', false) loadChannel.publish({ name: 'two' }) expect(Two.prototype.configure).to.have.been.calledWithMatch({ enabled: false }) }) + it('works with boolean true', () => { pm.configurePlugin('two', true) loadChannel.publish({ name: 'two' }) expect(Two.prototype.configure).to.have.been.calledWithMatch({ enabled: true }) }) }) + describe('with disabled plugins', () => { beforeEach(() => pm.configure()) + it('should not call configure on individual enable override', () => { pm.configurePlugin('five', { enabled: true }) loadChannel.publish({ name: 'five' }) expect(Five.prototype.configure).to.not.have.been.called }) + it('should not configure all disabled plugins', () => { pm.configure({}) loadChannel.publish({ name: 'five' }) @@ -146,78 +160,96 @@ describe('Plugin Manager', () => { expect(Six.prototype.configure).to.not.have.been.called }) }) + describe('with env var true', () => { beforeEach(() => pm.configure()) + beforeEach(() => { process.env.DD_TRACE_TWO_ENABLED = '1' }) + afterEach(() => { delete process.env.DD_TRACE_TWO_ENABLED }) + it('works with no config param', () => { pm.configurePlugin('two') loadChannel.publish({ name: 'two' }) expect(Two.prototype.configure).to.have.been.calledWithMatch({ enabled: true }) }) + it('works with empty object config', () => { pm.configurePlugin('two', {}) loadChannel.publish({ name: 'two' }) expect(Two.prototype.configure).to.have.been.calledWithMatch({ enabled: true }) }) + it('works with "enabled: false" object config', () => { pm.configurePlugin('two', { enabled: false }) loadChannel.publish({ name: 'two' }) expect(Two.prototype.configure).to.have.been.calledWithMatch({ enabled: false }) }) + it('works with "enabled: true" object config', () => { pm.configurePlugin('two', { enabled: true }) loadChannel.publish({ name: 'two' }) expect(Two.prototype.configure).to.have.been.calledWithMatch({ enabled: true }) }) + it('works with boolean false', () => { pm.configurePlugin('two', false) loadChannel.publish({ name: 'two' }) expect(Two.prototype.configure).to.have.been.calledWithMatch({ enabled: false }) }) + it('works with boolean true', () => { pm.configurePlugin('two', true) loadChannel.publish({ name: 'two' }) expect(Two.prototype.configure).to.have.been.calledWithMatch({ enabled: true }) }) }) + describe('with env var false', () => { beforeEach(() => pm.configure()) + beforeEach(() => { process.env.DD_TRACE_TWO_ENABLED = '0' }) + afterEach(() => { delete process.env.DD_TRACE_TWO_ENABLED }) + it('works with no config param', () => { pm.configurePlugin('two') loadChannel.publish({ name: 'two' }) expect(Two.prototype.configure).to.not.have.been.called }) + it('works with empty object config', () => { pm.configurePlugin('two', {}) loadChannel.publish({ name: 'two' }) expect(Two.prototype.configure).to.not.have.been.called }) + it('works with "enabled: false" object config', () => { pm.configurePlugin('two', { enabled: false }) loadChannel.publish({ name: 'two' }) expect(Two.prototype.configure).to.not.have.been.called }) + it('works with "enabled: true" object config', () => { pm.configurePlugin('two', { enabled: true }) loadChannel.publish({ name: 'two' }) expect(Two.prototype.configure).to.not.have.been.called }) + it('works with boolean false', () => { pm.configurePlugin('two', false) loadChannel.publish({ name: 'two' }) expect(Two.prototype.configure).to.not.have.been.called }) + it('works with boolean true', () => { pm.configurePlugin('two', true) loadChannel.publish({ name: 'two' }) @@ -235,21 +267,23 @@ describe('Plugin Manager', () => { expect(Two.prototype.configure).to.not.have.been.called }) }) + it('instantiates plugin classes', () => { pm.configure() loadChannel.publish({ name: 'two' }) loadChannel.publish({ name: 'four' }) expect(instantiated).to.deep.equal(['two', 'four']) }) + describe('service naming schema manager', () => { const config = { - foo: { 'bar': 1 }, + foo: { bar: 1 }, baz: 2 } let configureSpy beforeEach(() => { - configureSpy = sinon.spy(Nomenclature, 'configure') + configureSpy = sinon.spy(tracer._nomenclature, 'configure') }) afterEach(() => { @@ -261,11 +295,13 @@ describe('Plugin Manager', () => { expect(configureSpy).to.have.been.calledWith(config) }) }) + it('skips configuring plugins entirely when plugins is false', () => { pm.configurePlugin = sinon.spy() pm.configure({ plugins: false }) expect(pm.configurePlugin).not.to.have.been.called }) + it('observes configuration options', () => { pm.configure({ serviceMapping: { two: 'deux' }, @@ -293,6 +329,7 @@ describe('Plugin Manager', () => { describe('destroy', () => { beforeEach(() => pm.configure()) + it('should disable the plugins', () => { loadChannel.publish({ name: 'two' }) loadChannel.publish({ name: 'four' }) diff --git a/packages/dd-trace/test/plugins/agent.js b/packages/dd-trace/test/plugins/agent.js index 06c41d77731..cb6f241e7d3 100644 --- a/packages/dd-trace/test/plugins/agent.js +++ b/packages/dd-trace/test/plugins/agent.js @@ -4,19 +4,20 @@ const http = require('http') const bodyParser = require('body-parser') const msgpack = require('msgpack-lite') const codec = msgpack.createCodec({ int64: true }) -const getPort = require('get-port') const express = require('express') const path = require('path') const ritm = require('../../src/ritm') const { storage } = require('../../../datadog-core') -const handlers = new Set() +const traceHandlers = new Set() +const statsHandlers = new Set() let sockets = [] let agent = null let listener = null let tracer = null let plugins = [] const testedPlugins = [] +let dsmStats = [] function isMatchingTrace (spans, spanResourceMatch) { if (!spanResourceMatch) { @@ -27,7 +28,7 @@ function isMatchingTrace (spans, spanResourceMatch) { function ciVisRequestHandler (request, response) { response.status(200).send('OK') - handlers.forEach(({ handler, spanResourceMatch }) => { + traceHandlers.forEach(({ handler, spanResourceMatch }) => { const { events } = request.body const spans = events.map(event => event.content) if (isMatchingTrace(spans, spanResourceMatch)) { @@ -36,6 +37,38 @@ function ciVisRequestHandler (request, response) { }) } +function dsmStatsExist (agent, expectedHash, expectedEdgeTags) { + const dsmStats = agent.getDsmStats() + let hashFound = false + if (dsmStats.length !== 0) { + for (const statsTimeBucket of dsmStats) { + for (const statsBucket of statsTimeBucket.Stats) { + for (const stats of statsBucket.Stats) { + if (stats.Hash.toString() === expectedHash) { + if (expectedEdgeTags) { + if (expectedEdgeTags.length !== stats.EdgeTags.length) { + return false + } + + const expected = expectedEdgeTags.slice().sort() + const actual = stats.EdgeTags.slice().sort() + + for (let i = 0; i < expected.length; i++) { + if (expected[i] !== actual[i]) { + return false + } + } + } + hashFound = true + return hashFound + } + } + } + } + } + return hashFound +} + function addEnvironmentVariablesToHeaders (headers) { // get all environment variables that start with "DD_" const ddEnvVars = new Map( @@ -73,7 +106,7 @@ function handleTraceRequest (req, res, sendToTestAgent) { const testAgentUrl = process.env.DD_TEST_AGENT_URL || 'http://127.0.0.1:9126' // remove incorrect headers - delete req.headers['host'] + delete req.headers.host delete req.headers['content-type'] delete req.headers['content-length'] @@ -107,7 +140,7 @@ function handleTraceRequest (req, res, sendToTestAgent) { } res.status(200).send({ rate_by_service: { 'service:,env:': 1 } }) - handlers.forEach(({ handler, spanResourceMatch }) => { + traceHandlers.forEach(({ handler, spanResourceMatch }) => { const trace = req.body const spans = trace.flatMap(span => span) if (isMatchingTrace(spans, spanResourceMatch)) { @@ -136,10 +169,67 @@ function checkAgentStatus () { }) } -const DEFAULT_AVAILABLE_ENDPOINTS = ['/evp_proxy/v2'] +function getDsmStats () { + return dsmStats +} +const DEFAULT_AVAILABLE_ENDPOINTS = ['/evp_proxy/v2'] let availableEndpoints = DEFAULT_AVAILABLE_ENDPOINTS +/** + * Register a callback with expectations to be run on every tracing or stats payload sent to the agent depending + * on the handlers inputted. If the callback does not throw, the returned promise resolves. If it does, + * then the agent will wait for additional payloads up until the timeout + * (default 1000 ms) and if any of them succeed, the promise will resolve. + * Otherwise, it will reject. + * + * @param {(traces: Array>) => void} callback - A function that tests a payload as it's received. + * @param {Object} [options] - An options object + * @param {number} [options.timeoutMs=1000] - The timeout in ms. + * @param {boolean} [options.rejectFirst=false] - If true, reject the first time the callback throws. + * @param {Set} [handlers] - Set of handlers to add the callback to. + * @returns {Promise} A promise resolving if expectations are met + */ +function runCallback (callback, options, handlers) { + const deferred = {} + const promise = new Promise((resolve, reject) => { + deferred.resolve = resolve + deferred.reject = reject + }) + + const timeoutMs = options !== null && typeof options === 'object' && options.timeoutMs ? options.timeoutMs : 1000 + + const timeout = setTimeout(() => { + if (error) { + deferred.reject(error) + } + }, timeoutMs) + + let error + const handlerPayload = { handler, spanResourceMatch: options && options.spanResourceMatch } + + function handler () { + try { + const result = callback.apply(null, arguments) + handlers.delete(handlerPayload) + clearTimeout(timeout) + deferred.resolve(result) + } catch (e) { + if (options && options.rejectFirst) { + clearTimeout(timeout) + deferred.reject(e) + } else { + error = error || e + } + } + } + + handler.promise = promise + handlers.add(handlerPayload) + + return promise +} + module.exports = { // Load the plugin on the tracer with an optional config and start a mock agent. async load (pluginName, config, tracerConfig = {}) { @@ -182,7 +272,15 @@ module.exports = { // EVP proxy endpoint agent.post('/evp_proxy/v2/api/v2/citestcycle', ciVisRequestHandler) - const port = await getPort() + // DSM Checkpoint endpoint + dsmStats = [] + agent.post('/v0.1/pipeline_stats', (req, res) => { + dsmStats.push(req.body) + statsHandlers.forEach(({ handler, spanResourceMatch }) => { + handler(dsmStats) + }) + res.status(200).send() + }) const server = this.server = http.createServer(agent) const emit = server.emit @@ -195,7 +293,25 @@ module.exports = { server.on('connection', socket => sockets.push(socket)) const promise = new Promise((resolve, reject) => { - listener = server.listen(port, () => resolve()) + listener = server.listen(0, () => { + const port = listener.address().port + + tracer.init(Object.assign({}, { + service: 'test', + env: 'tester', + port, + flushInterval: 0, + plugins: false + }, tracerConfig)) + + tracer.setUrl(`http://127.0.0.1:${port}`) + + for (let i = 0, l = pluginName.length; i < l; i++) { + tracer.use(pluginName[i], config[i]) + } + + resolve() + }) }) pluginName = [].concat(pluginName) @@ -204,22 +320,9 @@ module.exports = { server.on('close', () => { tracer = null + dsmStats = [] }) - tracer.init(Object.assign({}, { - service: 'test', - env: 'tester', - port, - flushInterval: 0, - plugins: false - }, tracerConfig)) - - tracer.setUrl(`http://127.0.0.1:${port}`) - - for (let i = 0, l = pluginName.length; i < l; i++) { - tracer.use(pluginName[i], config[i]) - } - return promise }, @@ -227,6 +330,7 @@ module.exports = { pluginName = [].concat(pluginName) plugins = pluginName config = [].concat(config) + dsmStats = [] for (let i = 0, l = pluginName.length; i < l; i++) { tracer.use(pluginName[i], config[i]) @@ -235,70 +339,32 @@ module.exports = { // Register handler to be executed each agent call, multiple times subscribe (handler) { - handlers.add({ handler }) + traceHandlers.add({ handler }) }, // Remove a handler unsubscribe (handler) { - handlers.delete(handler) + traceHandlers.delete(handler) }, /** * Register a callback with expectations to be run on every tracing payload sent to the agent. - * If the callback does not throw, the returned promise resolves. If it does, - * then the agent will wait for additional payloads up until the timeout - * (default 1000 ms) and if any of them succeed, the promise will resolve. - * Otherwise, it will reject. - * - * @param {(traces: Array>) => void} callback - A function that tests trace data as it's received. - * @param {Object} [options] - An options object - * @param {number} [options.timeoutMs=1000] - The timeout in ms. - * @param {boolean} [options.rejectFirst=false] - If true, reject the first time the callback throws. - * @returns {Promise} A promise resolving if expectations are met */ use (callback, options) { - const deferred = {} - const promise = new Promise((resolve, reject) => { - deferred.resolve = resolve - deferred.reject = reject - }) - - const timeoutMs = options && typeof options === 'object' && options.timeoutMs ? options.timeoutMs : 1000 - - const timeout = setTimeout(() => { - if (error) { - deferred.reject(error) - } - }, timeoutMs) - - let error - const handlerPayload = { handler, spanResourceMatch: options && options.spanResourceMatch } - - function handler () { - try { - callback.apply(null, arguments) - handlers.delete(handlerPayload) - clearTimeout(timeout) - deferred.resolve() - } catch (e) { - if (options && options.rejectFirst) { - clearTimeout(timeout) - deferred.reject(e) - } else { - error = error || e - } - } - } - - handler.promise = promise - handlers.add(handlerPayload) + return runCallback(callback, options, traceHandlers) + }, - return promise + /** + * Register a callback with expectations to be run on every stats payload sent to the agent. + */ + expectPipelineStats (callback, options) { + return runCallback(callback, options, statsHandlers) }, // Unregister any outstanding expectation callbacks. reset () { - handlers.clear() + traceHandlers.clear() + statsHandlers.clear() }, // Stop the mock agent, reset all expectations and wipe the require cache. @@ -310,7 +376,8 @@ module.exports = { sockets.forEach(socket => socket.end()) sockets = [] agent = null - handlers.clear() + traceHandlers.clear() + statsHandlers.clear() for (const plugin of plugins) { tracer.use(plugin, { enabled: false }) } @@ -354,5 +421,8 @@ module.exports = { }) }, - testedPlugins + tracer, + testedPlugins, + getDsmStats, + dsmStatsExist } diff --git a/packages/dd-trace/test/plugins/externals.json b/packages/dd-trace/test/plugins/externals.json index 01f65e0551e..0f98a05409b 100644 --- a/packages/dd-trace/test/plugins/externals.json +++ b/packages/dd-trace/test/plugins/externals.json @@ -1,4 +1,22 @@ { + "apollo": [ + { + "name": "@apollo/subgraph", + "versions": [">=2.3.0"] + }, + { + "name": "graphql", + "versions": ["^16.6.0"] + }, + { + "name": "graphql-tag", + "versions": ["^2.12.6"] + }, + { + "name": "@apollo/server", + "versions": ["^4.0.0"] + } + ], "aws-sdk": [ { "name": "@aws-sdk/client-lambda", @@ -12,6 +30,10 @@ "name": "@aws-sdk/client-s3", "versions": [">=3"] }, + { + "name": "@aws-sdk/client-sfn", + "versions": [">=3"] + }, { "name": "@aws-sdk/client-sns", "versions": [">=3"] @@ -25,6 +47,18 @@ "versions": [">=3"] } ], + "body-parser": [ + { + "name": "express", + "versions": ["^4"] + } + ], + "cookie-parser": [ + { + "name": "express", + "versions": ["^4"] + } + ], "cypress": [ { "name": "cypress", @@ -32,6 +66,10 @@ } ], "express": [ + { + "name": "axios", + "versions": [">=1.0.0"] + }, { "name": "loopback", "versions": [">=2.38.1"] @@ -39,6 +77,10 @@ { "name": "cookie-parser", "versions": [">=1.4.6"] + }, + { + "name": "request", + "versions": ["2.88.2"] } ], "express-mongo-sanitize": [ @@ -59,6 +101,26 @@ "versions": ["1.20.1"] } ], + "mquery": [ + { + "name": "mquery", + "versions": [">=5.0.0"] + }, + { + "name": "mongodb", + "versions": ["5", ">=6"] + } + ], + "mysql2": [ + { + "name": "mysql2", + "versions": ["1.3.3"] + }, + { + "name": "express", + "versions": [">=4"] + } + ], "fastify": [ { "name": "fastify", @@ -86,6 +148,22 @@ "name": "apollo-server-core", "versions": ["1.3.6"] }, + { + "name": "express", + "versions": [">=4"] + }, + { + "name": "apollo-server-express", + "versions": [">=3"] + }, + { + "name": "fastify", + "versions": [">=3"] + }, + { + "name": "apollo-server-fastify", + "versions": [">=3"] + }, { "name": "graphql-tools", "versions": ["3.1.1"] @@ -99,6 +177,42 @@ "versions": ["^15.2.0"] } ], + "apollo-server-core": [ + { + "name": "fastify", + "versions": [">=3"] + }, + { + "name": "express", + "versions": [">=4"] + }, + { + "name": "apollo-server-fastify", + "versions": [">=3"] + }, + { + "name": "apollo-server-express", + "versions": [">=3"] + }, + { + "name": "graphql", + "versions": ["^15.2.0"] + } + ], + "apollo-server": [ + { + "name": "express", + "versions": [">=4"] + }, + { + "name": "@apollo/server", + "versions": [">=4"] + }, + { + "name": "graphql", + "versions": ["^16.6.0"] + } + ], "grpc": [ { "name": "@grpc/proto-loader", @@ -163,6 +277,12 @@ "versions": [">=2"] } ], + "lodash": [ + { + "name": "lodash", + "versions": [">=4"] + } + ], "mariadb": [ { "name": "mariadb", @@ -172,7 +292,7 @@ "mocha": [ { "name": "mocha", - "versions": [">=5.2.0"] + "versions": [">=5.2.0", ">=8.0.0"] }, { "name": "mocha-each", @@ -189,6 +309,10 @@ { "name": "bson", "versions": ["4.0.0"] + }, + { + "name": "mongodb", + "versions": ["6.3.0"] } ], "mongoose": [ @@ -233,12 +357,20 @@ { "name": "express", "versions": [">=4.16.2"] + }, + { + "name": "body-parser", + "versions": ["1.20.1"] } ], "pg": [ { "name": "pg-native", "versions": ["3.0.0"] + }, + { + "name": "express", + "versions": [">=4"] } ], "pino": [ @@ -263,5 +395,17 @@ "name": "redis", "versions": ["^4"] } + ], + "rhea": [ + { + "name": "amqp10", + "versions": ["^3"] + } + ], + "sequelize": [ + { + "name": "express", + "versions": [">=4"] + } ] } diff --git a/packages/dd-trace/test/plugins/helpers.js b/packages/dd-trace/test/plugins/helpers.js index 8d1f329de3f..b35793b6664 100644 --- a/packages/dd-trace/test/plugins/helpers.js +++ b/packages/dd-trace/test/plugins/helpers.js @@ -81,8 +81,8 @@ function compare (expected, actual) { } function isObject (obj) { - // `null` is also typeof 'object', so check for that with truthiness. - return obj && typeof obj === 'object' + // `null` is also typeof 'object' + return obj !== null && typeof obj === 'object' } function withDefaults (defaults, obj) { diff --git a/packages/dd-trace/test/plugins/outbound.spec.js b/packages/dd-trace/test/plugins/outbound.spec.js index 16ed6300c09..5709c789575 100644 --- a/packages/dd-trace/test/plugins/outbound.spec.js +++ b/packages/dd-trace/test/plugins/outbound.spec.js @@ -50,6 +50,7 @@ describe('OuboundPlugin', () => { expect(getRemapStub).to.not.be.called }) }) + describe('peer.service computation', () => { let instance = null @@ -88,7 +89,7 @@ describe('OuboundPlugin', () => { it('should use specific tags in order of precedence if they are available', () => { class WithPrecursors extends OutboundPlugin { - static get peerServicePrecursors () { return [ 'foo', 'bar' ] } + static get peerServicePrecursors () { return ['foo', 'bar'] } } const res = new WithPrecursors().getPeerService({ fooIsNotAPrecursor: 'bar', @@ -101,6 +102,7 @@ describe('OuboundPlugin', () => { }) }) }) + describe('remapping computation', () => { let instance = null let mappingStub = null @@ -119,8 +121,8 @@ describe('OuboundPlugin', () => { it('should return peer data unchanged if there is no peer service', () => { mappingStub = sinon.stub(instance, '_tracerConfig').value({}) - const mappingData = instance.getPeerServiceRemap({ 'foo': 'bar' }) - expect(mappingData).to.deep.equal({ 'foo': 'bar' }) + const mappingData = instance.getPeerServiceRemap({ foo: 'bar' }) + expect(mappingData).to.deep.equal({ foo: 'bar' }) }) it('should return peer data unchanged if no mapping is available', () => { diff --git a/packages/dd-trace/test/plugins/plugin.spec.js b/packages/dd-trace/test/plugins/plugin.spec.js new file mode 100644 index 00000000000..a5986b9bba3 --- /dev/null +++ b/packages/dd-trace/test/plugins/plugin.spec.js @@ -0,0 +1,69 @@ +'use strict' + +require('../setup/tap') + +const Plugin = require('../../src/plugins/plugin') +const plugins = require('../../src/plugins') +const { channel } = require('dc-polyfill') + +describe('Plugin', () => { + class BadPlugin extends Plugin { + static get id () { return 'badPlugin' } + + constructor () { + super() + this.addSub('apm:badPlugin:start', this.start) + } + + start () { + throw new Error('this is one bad plugin') + } + } + + class GoodPlugin extends Plugin { + static get id () { return 'goodPlugin' } + + constructor () { + super() + this.addSub('apm:goodPlugin:start', this.start) + } + + start () { + // + } + } + + const testPlugins = { badPlugin: BadPlugin, goodPlugin: GoodPlugin } + const loadChannel = channel('dd-trace:instrumentation:load') + + before(() => { + for (const [name, cls] of Object.entries(testPlugins)) { + plugins[name] = cls + loadChannel.publish({ name }) + } + }) + + after(() => { Object.keys(testPlugins).forEach(name => delete plugins[name]) }) + + it('should disable upon error', () => { + const plugin = new BadPlugin() + plugin.configure({ enabled: true }) + + expect(plugin._enabled).to.be.true + + channel('apm:badPlugin:start').publish({ foo: 'bar' }) + + expect(plugin._enabled).to.be.false + }) + + it('should not disable with no error', () => { + const plugin = new GoodPlugin() + plugin.configure({ enabled: true }) + + expect(plugin._enabled).to.be.true + + channel('apm:goodPlugin:start').publish({ foo: 'bar' }) + + expect(plugin._enabled).to.be.true + }) +}) diff --git a/packages/dd-trace/test/plugins/suite.js b/packages/dd-trace/test/plugins/suite.js index 65d6e6ccb78..a0cb20845b4 100644 --- a/packages/dd-trace/test/plugins/suite.js +++ b/packages/dd-trace/test/plugins/suite.js @@ -35,6 +35,7 @@ async function getLatest (modName, repoUrl) { function get (theUrl) { return new Promise((resolve, reject) => { + // eslint-disable-next-line n/no-deprecated-api const options = url.parse(theUrl) options.headers = { 'user-agent': 'dd-trace plugin test suites' @@ -131,7 +132,7 @@ async function setup (modName, repoName, commitish) { const repoUrl = `https://github.com/${repoName}.git` const cwd = await getTmpDir() await execOrError(`git clone ${repoUrl} --branch ${commitish} --single-branch ${cwd}`) - await execOrError(`npm install --legacy-peer-deps`, { cwd }) + await execOrError('npm install --legacy-peer-deps', { cwd }) } async function cleanup () { @@ -195,7 +196,7 @@ ${withTracer.stderr} function getOpts (args) { args = Array.from(args) - const [ modName, repoUrl, commitish, runner, timeout, testCmd ] = args + const [modName, repoUrl, commitish, runner, timeout, testCmd] = args const options = { modName, repoUrl, diff --git a/packages/dd-trace/test/plugins/tracing.spec.js b/packages/dd-trace/test/plugins/tracing.spec.js index b8331f7915e..9c24ce37ab5 100644 --- a/packages/dd-trace/test/plugins/tracing.spec.js +++ b/packages/dd-trace/test/plugins/tracing.spec.js @@ -32,6 +32,7 @@ describe('TracingPlugin', () => { describe('common Plugin behaviour', () => { before(() => agent.load()) + after(() => agent.close({ ritmReset: false })) class CommonPlugin extends TracingPlugin { static get id () { return 'commonPlugin' } @@ -54,7 +55,7 @@ describe('common Plugin behaviour', () => { } } - const testPlugins = { 'commonPlugin': CommonPlugin, 'suffixPlugin': SuffixPlugin } + const testPlugins = { commonPlugin: CommonPlugin, suffixPlugin: SuffixPlugin } const loadChannel = channel('dd-trace:instrumentation:load') before(() => { @@ -97,6 +98,7 @@ describe('common Plugin behaviour', () => { } ) }) + it('should tag when plugin impl does not match tracer service', done => { makeSpan( done, 'suffixPlugin', {}, @@ -106,6 +108,7 @@ describe('common Plugin behaviour', () => { } ) }) + it('should not tag when service matches tracer service', done => { makeSpan( done, 'commonPlugin', {}, diff --git a/packages/dd-trace/test/plugins/util/ci-env/azurepipelines.json b/packages/dd-trace/test/plugins/util/ci-env/azurepipelines.json index 594da6d147b..904e3fe9b26 100644 --- a/packages/dd-trace/test/plugins/util/ci-env/azurepipelines.json +++ b/packages/dd-trace/test/plugins/util/ci-env/azurepipelines.json @@ -650,6 +650,64 @@ "git.tag": "0.0.2" } ], + [ + { + "BUILD_BUILDID": "azure-pipelines-build-id", + "BUILD_DEFINITIONNAME": "azure-pipelines-name", + "BUILD_REPOSITORY_URI": "https://dev.azure.com/fabrikamfiber/repo", + "BUILD_REQUESTEDFOREMAIL": "azure-pipelines-commit-author-email@datadoghq.com", + "BUILD_REQUESTEDFORID": "azure-pipelines-commit-author", + "BUILD_SOURCEVERSIONMESSAGE": "azure-pipelines-commit-message", + "DD_TEST_CASE_NAME": "http-repository-url-no-git-suffix", + "SYSTEM_JOBID": "azure-pipelines-job-id", + "SYSTEM_TASKINSTANCEID": "azure-pipelines-task-id", + "SYSTEM_TEAMFOUNDATIONSERVERURI": "https://azure-pipelines-server-uri.com/", + "SYSTEM_TEAMPROJECTID": "azure-pipelines-project-id", + "TF_BUILD": "True" + }, + { + "_dd.ci.env_vars": "{\"SYSTEM_TEAMPROJECTID\":\"azure-pipelines-project-id\",\"BUILD_BUILDID\":\"azure-pipelines-build-id\",\"SYSTEM_JOBID\":\"azure-pipelines-job-id\"}", + "ci.job.url": "https://azure-pipelines-server-uri.com/azure-pipelines-project-id/_build/results?buildId=azure-pipelines-build-id&view=logs&j=azure-pipelines-job-id&t=azure-pipelines-task-id", + "ci.pipeline.id": "azure-pipelines-build-id", + "ci.pipeline.name": "azure-pipelines-name", + "ci.pipeline.number": "azure-pipelines-build-id", + "ci.pipeline.url": "https://azure-pipelines-server-uri.com/azure-pipelines-project-id/_build/results?buildId=azure-pipelines-build-id", + "ci.provider.name": "azurepipelines", + "git.commit.author.email": "azure-pipelines-commit-author-email@datadoghq.com", + "git.commit.author.name": "azure-pipelines-commit-author", + "git.commit.message": "azure-pipelines-commit-message", + "git.repository_url": "https://dev.azure.com/fabrikamfiber/repo" + } + ], + [ + { + "BUILD_BUILDID": "azure-pipelines-build-id", + "BUILD_DEFINITIONNAME": "azure-pipelines-name", + "BUILD_REPOSITORY_URI": "ssh://host.xz:port/path/to/repo/", + "BUILD_REQUESTEDFOREMAIL": "azure-pipelines-commit-author-email@datadoghq.com", + "BUILD_REQUESTEDFORID": "azure-pipelines-commit-author", + "BUILD_SOURCEVERSIONMESSAGE": "azure-pipelines-commit-message", + "DD_TEST_CASE_NAME": "ssh-repository-url-no-git-suffix", + "SYSTEM_JOBID": "azure-pipelines-job-id", + "SYSTEM_TASKINSTANCEID": "azure-pipelines-task-id", + "SYSTEM_TEAMFOUNDATIONSERVERURI": "https://azure-pipelines-server-uri.com/", + "SYSTEM_TEAMPROJECTID": "azure-pipelines-project-id", + "TF_BUILD": "True" + }, + { + "_dd.ci.env_vars": "{\"SYSTEM_TEAMPROJECTID\":\"azure-pipelines-project-id\",\"BUILD_BUILDID\":\"azure-pipelines-build-id\",\"SYSTEM_JOBID\":\"azure-pipelines-job-id\"}", + "ci.job.url": "https://azure-pipelines-server-uri.com/azure-pipelines-project-id/_build/results?buildId=azure-pipelines-build-id&view=logs&j=azure-pipelines-job-id&t=azure-pipelines-task-id", + "ci.pipeline.id": "azure-pipelines-build-id", + "ci.pipeline.name": "azure-pipelines-name", + "ci.pipeline.number": "azure-pipelines-build-id", + "ci.pipeline.url": "https://azure-pipelines-server-uri.com/azure-pipelines-project-id/_build/results?buildId=azure-pipelines-build-id", + "ci.provider.name": "azurepipelines", + "git.commit.author.email": "azure-pipelines-commit-author-email@datadoghq.com", + "git.commit.author.name": "azure-pipelines-commit-author", + "git.commit.message": "azure-pipelines-commit-message", + "git.repository_url": "ssh://host.xz:port/path/to/repo/" + } + ], [ { "BUILD_BUILDID": "azure-pipelines-build-id", diff --git a/packages/dd-trace/test/plugins/util/ci-env/bitbucket.json b/packages/dd-trace/test/plugins/util/ci-env/bitbucket.json index 72d47cdff00..019621c8999 100644 --- a/packages/dd-trace/test/plugins/util/ci-env/bitbucket.json +++ b/packages/dd-trace/test/plugins/util/ci-env/bitbucket.json @@ -400,6 +400,46 @@ "git.tag": "0.0.2" } ], + [ + { + "BITBUCKET_BUILD_NUMBER": "bitbucket-build-num", + "BITBUCKET_COMMIT": "b9f0fb3fdbb94c9d24b2c75b49663122a529e123", + "BITBUCKET_GIT_HTTP_ORIGIN": "https://bitbucket.org/DataDog/dogweb", + "BITBUCKET_PIPELINE_UUID": "{bitbucket-uuid}", + "BITBUCKET_REPO_FULL_NAME": "bitbucket-repo", + "DD_TEST_CASE_NAME": "http-repository-url-no-git-suffix" + }, + { + "ci.job.url": "https://bitbucket.org/bitbucket-repo/addon/pipelines/home#!/results/bitbucket-build-num", + "ci.pipeline.id": "bitbucket-uuid", + "ci.pipeline.name": "bitbucket-repo", + "ci.pipeline.number": "bitbucket-build-num", + "ci.pipeline.url": "https://bitbucket.org/bitbucket-repo/addon/pipelines/home#!/results/bitbucket-build-num", + "ci.provider.name": "bitbucket", + "git.commit.sha": "b9f0fb3fdbb94c9d24b2c75b49663122a529e123", + "git.repository_url": "https://bitbucket.org/DataDog/dogweb" + } + ], + [ + { + "BITBUCKET_BUILD_NUMBER": "bitbucket-build-num", + "BITBUCKET_COMMIT": "b9f0fb3fdbb94c9d24b2c75b49663122a529e123", + "BITBUCKET_GIT_HTTP_ORIGIN": "ssh://host.xz:port/path/to/repo/", + "BITBUCKET_PIPELINE_UUID": "{bitbucket-uuid}", + "BITBUCKET_REPO_FULL_NAME": "bitbucket-repo", + "DD_TEST_CASE_NAME": "ssh-repository-url-no-git-suffix" + }, + { + "ci.job.url": "https://bitbucket.org/bitbucket-repo/addon/pipelines/home#!/results/bitbucket-build-num", + "ci.pipeline.id": "bitbucket-uuid", + "ci.pipeline.name": "bitbucket-repo", + "ci.pipeline.number": "bitbucket-build-num", + "ci.pipeline.url": "https://bitbucket.org/bitbucket-repo/addon/pipelines/home#!/results/bitbucket-build-num", + "ci.provider.name": "bitbucket", + "git.commit.sha": "b9f0fb3fdbb94c9d24b2c75b49663122a529e123", + "git.repository_url": "ssh://host.xz:port/path/to/repo/" + } + ], [ { "BITBUCKET_BUILD_NUMBER": "bitbucket-build-num", diff --git a/packages/dd-trace/test/plugins/util/ci-env/bitrise.json b/packages/dd-trace/test/plugins/util/ci-env/bitrise.json index 6f5b52cdf90..73b753340cd 100644 --- a/packages/dd-trace/test/plugins/util/ci-env/bitrise.json +++ b/packages/dd-trace/test/plugins/util/ci-env/bitrise.json @@ -479,6 +479,50 @@ "git.tag": "0.0.2" } ], + [ + { + "BITRISE_BUILD_NUMBER": "bitrise-pipeline-number", + "BITRISE_BUILD_SLUG": "bitrise-pipeline-id", + "BITRISE_BUILD_URL": "https://bitrise-build-url.com//", + "BITRISE_GIT_MESSAGE": "bitrise-git-commit-message", + "BITRISE_TRIGGERED_WORKFLOW_ID": "bitrise-pipeline-name", + "DD_TEST_CASE_NAME": "http-repository-url-no-git-suffix", + "GIT_CLONE_COMMIT_HASH": "b9f0fb3fdbb94c9d24b2c75b49663122a529e123", + "GIT_REPOSITORY_URL": "https://github.com/DataDog/dogweb" + }, + { + "ci.pipeline.id": "bitrise-pipeline-id", + "ci.pipeline.name": "bitrise-pipeline-name", + "ci.pipeline.number": "bitrise-pipeline-number", + "ci.pipeline.url": "https://bitrise-build-url.com//", + "ci.provider.name": "bitrise", + "git.commit.message": "bitrise-git-commit-message", + "git.commit.sha": "b9f0fb3fdbb94c9d24b2c75b49663122a529e123", + "git.repository_url": "https://github.com/DataDog/dogweb" + } + ], + [ + { + "BITRISE_BUILD_NUMBER": "bitrise-pipeline-number", + "BITRISE_BUILD_SLUG": "bitrise-pipeline-id", + "BITRISE_BUILD_URL": "https://bitrise-build-url.com//", + "BITRISE_GIT_MESSAGE": "bitrise-git-commit-message", + "BITRISE_TRIGGERED_WORKFLOW_ID": "bitrise-pipeline-name", + "DD_TEST_CASE_NAME": "ssh-repository-url-no-git-suffix", + "GIT_CLONE_COMMIT_HASH": "b9f0fb3fdbb94c9d24b2c75b49663122a529e123", + "GIT_REPOSITORY_URL": "ssh://host.xz:port/path/to/repo/" + }, + { + "ci.pipeline.id": "bitrise-pipeline-id", + "ci.pipeline.name": "bitrise-pipeline-name", + "ci.pipeline.number": "bitrise-pipeline-number", + "ci.pipeline.url": "https://bitrise-build-url.com//", + "ci.provider.name": "bitrise", + "git.commit.message": "bitrise-git-commit-message", + "git.commit.sha": "b9f0fb3fdbb94c9d24b2c75b49663122a529e123", + "git.repository_url": "ssh://host.xz:port/path/to/repo/" + } + ], [ { "BITRISE_BUILD_NUMBER": "bitrise-pipeline-number", diff --git a/packages/dd-trace/test/plugins/util/ci-env/buddy.json b/packages/dd-trace/test/plugins/util/ci-env/buddy.json index 007cc196652..3a43ab27a6c 100644 --- a/packages/dd-trace/test/plugins/util/ci-env/buddy.json +++ b/packages/dd-trace/test/plugins/util/ci-env/buddy.json @@ -178,6 +178,68 @@ "git.tag": "v1.0" } ], + [ + { + "BUDDY": "true", + "BUDDY_EXECUTION_BRANCH": "master", + "BUDDY_EXECUTION_ID": "buddy-execution-id", + "BUDDY_EXECUTION_REVISION": "b9f0fb3fdbb94c9d24b2c75b49663122a529e123", + "BUDDY_EXECUTION_REVISION_COMMITTER_EMAIL": "mikebenson@buddy.works", + "BUDDY_EXECUTION_REVISION_COMMITTER_NAME": "Mike Benson", + "BUDDY_EXECUTION_REVISION_MESSAGE": "Create buddy.yml", + "BUDDY_EXECUTION_TAG": "v1.0", + "BUDDY_EXECUTION_URL": "https://app.buddy.works/myworkspace/my-project/pipelines/pipeline/456/execution/5d9dc42c422f5a268b389d08", + "BUDDY_PIPELINE_ID": "456", + "BUDDY_PIPELINE_NAME": "Deploy to Production", + "BUDDY_SCM_URL": "https://github.com/buddyworks/my-project", + "DD_TEST_CASE_NAME": "http-repository-url-no-git-suffix" + }, + { + "ci.pipeline.id": "456/buddy-execution-id", + "ci.pipeline.name": "Deploy to Production", + "ci.pipeline.number": "buddy-execution-id", + "ci.pipeline.url": "https://app.buddy.works/myworkspace/my-project/pipelines/pipeline/456/execution/5d9dc42c422f5a268b389d08", + "ci.provider.name": "buddy", + "git.branch": "master", + "git.commit.committer.email": "mikebenson@buddy.works", + "git.commit.committer.name": "Mike Benson", + "git.commit.message": "Create buddy.yml", + "git.commit.sha": "b9f0fb3fdbb94c9d24b2c75b49663122a529e123", + "git.repository_url": "https://github.com/buddyworks/my-project", + "git.tag": "v1.0" + } + ], + [ + { + "BUDDY": "true", + "BUDDY_EXECUTION_BRANCH": "master", + "BUDDY_EXECUTION_ID": "buddy-execution-id", + "BUDDY_EXECUTION_REVISION": "b9f0fb3fdbb94c9d24b2c75b49663122a529e123", + "BUDDY_EXECUTION_REVISION_COMMITTER_EMAIL": "mikebenson@buddy.works", + "BUDDY_EXECUTION_REVISION_COMMITTER_NAME": "Mike Benson", + "BUDDY_EXECUTION_REVISION_MESSAGE": "Create buddy.yml", + "BUDDY_EXECUTION_TAG": "v1.0", + "BUDDY_EXECUTION_URL": "https://app.buddy.works/myworkspace/my-project/pipelines/pipeline/456/execution/5d9dc42c422f5a268b389d08", + "BUDDY_PIPELINE_ID": "456", + "BUDDY_PIPELINE_NAME": "Deploy to Production", + "BUDDY_SCM_URL": "ssh://host.xz:port/path/to/repo/", + "DD_TEST_CASE_NAME": "ssh-repository-url-no-git-suffix" + }, + { + "ci.pipeline.id": "456/buddy-execution-id", + "ci.pipeline.name": "Deploy to Production", + "ci.pipeline.number": "buddy-execution-id", + "ci.pipeline.url": "https://app.buddy.works/myworkspace/my-project/pipelines/pipeline/456/execution/5d9dc42c422f5a268b389d08", + "ci.provider.name": "buddy", + "git.branch": "master", + "git.commit.committer.email": "mikebenson@buddy.works", + "git.commit.committer.name": "Mike Benson", + "git.commit.message": "Create buddy.yml", + "git.commit.sha": "b9f0fb3fdbb94c9d24b2c75b49663122a529e123", + "git.repository_url": "ssh://host.xz:port/path/to/repo/", + "git.tag": "v1.0" + } + ], [ { "BUDDY": "true", diff --git a/packages/dd-trace/test/plugins/util/ci-env/buildkite.json b/packages/dd-trace/test/plugins/util/ci-env/buildkite.json index 421904b20e6..b3bc32975e3 100644 --- a/packages/dd-trace/test/plugins/util/ci-env/buildkite.json +++ b/packages/dd-trace/test/plugins/util/ci-env/buildkite.json @@ -673,6 +673,70 @@ "git.tag": "0.0.2" } ], + [ + { + "BUILDKITE": "true", + "BUILDKITE_BRANCH": "", + "BUILDKITE_BUILD_AUTHOR": "buildkite-git-commit-author-name", + "BUILDKITE_BUILD_AUTHOR_EMAIL": "buildkite-git-commit-author-email@datadoghq.com", + "BUILDKITE_BUILD_ID": "buildkite-pipeline-id", + "BUILDKITE_BUILD_NUMBER": "buildkite-pipeline-number", + "BUILDKITE_BUILD_URL": "https://buildkite-build-url.com", + "BUILDKITE_COMMIT": "b9f0fb3fdbb94c9d24b2c75b49663122a529e123", + "BUILDKITE_JOB_ID": "buildkite-job-id", + "BUILDKITE_MESSAGE": "buildkite-git-commit-message", + "BUILDKITE_PIPELINE_SLUG": "buildkite-pipeline-name", + "BUILDKITE_REPO": "https://github.com/DataDog/dogweb", + "BUILDKITE_TAG": "", + "DD_TEST_CASE_NAME": "http-repository-url-no-git-suffix" + }, + { + "_dd.ci.env_vars": "{\"BUILDKITE_BUILD_ID\":\"buildkite-pipeline-id\",\"BUILDKITE_JOB_ID\":\"buildkite-job-id\"}", + "ci.job.url": "https://buildkite-build-url.com#buildkite-job-id", + "ci.pipeline.id": "buildkite-pipeline-id", + "ci.pipeline.name": "buildkite-pipeline-name", + "ci.pipeline.number": "buildkite-pipeline-number", + "ci.pipeline.url": "https://buildkite-build-url.com", + "ci.provider.name": "buildkite", + "git.commit.author.email": "buildkite-git-commit-author-email@datadoghq.com", + "git.commit.author.name": "buildkite-git-commit-author-name", + "git.commit.message": "buildkite-git-commit-message", + "git.commit.sha": "b9f0fb3fdbb94c9d24b2c75b49663122a529e123", + "git.repository_url": "https://github.com/DataDog/dogweb" + } + ], + [ + { + "BUILDKITE": "true", + "BUILDKITE_BRANCH": "", + "BUILDKITE_BUILD_AUTHOR": "buildkite-git-commit-author-name", + "BUILDKITE_BUILD_AUTHOR_EMAIL": "buildkite-git-commit-author-email@datadoghq.com", + "BUILDKITE_BUILD_ID": "buildkite-pipeline-id", + "BUILDKITE_BUILD_NUMBER": "buildkite-pipeline-number", + "BUILDKITE_BUILD_URL": "https://buildkite-build-url.com", + "BUILDKITE_COMMIT": "b9f0fb3fdbb94c9d24b2c75b49663122a529e123", + "BUILDKITE_JOB_ID": "buildkite-job-id", + "BUILDKITE_MESSAGE": "buildkite-git-commit-message", + "BUILDKITE_PIPELINE_SLUG": "buildkite-pipeline-name", + "BUILDKITE_REPO": "ssh://host.xz:port/path/to/repo/", + "BUILDKITE_TAG": "", + "DD_TEST_CASE_NAME": "ssh-repository-url-no-git-suffix" + }, + { + "_dd.ci.env_vars": "{\"BUILDKITE_BUILD_ID\":\"buildkite-pipeline-id\",\"BUILDKITE_JOB_ID\":\"buildkite-job-id\"}", + "ci.job.url": "https://buildkite-build-url.com#buildkite-job-id", + "ci.pipeline.id": "buildkite-pipeline-id", + "ci.pipeline.name": "buildkite-pipeline-name", + "ci.pipeline.number": "buildkite-pipeline-number", + "ci.pipeline.url": "https://buildkite-build-url.com", + "ci.provider.name": "buildkite", + "git.commit.author.email": "buildkite-git-commit-author-email@datadoghq.com", + "git.commit.author.name": "buildkite-git-commit-author-name", + "git.commit.message": "buildkite-git-commit-message", + "git.commit.sha": "b9f0fb3fdbb94c9d24b2c75b49663122a529e123", + "git.repository_url": "ssh://host.xz:port/path/to/repo/" + } + ], [ { "BUILDKITE": "true", diff --git a/packages/dd-trace/test/plugins/util/ci-env/circleci.json b/packages/dd-trace/test/plugins/util/ci-env/circleci.json index b9065be3bd6..ec61ea8205f 100644 --- a/packages/dd-trace/test/plugins/util/ci-env/circleci.json +++ b/packages/dd-trace/test/plugins/util/ci-env/circleci.json @@ -519,6 +519,54 @@ "git.tag": "0.0.2" } ], + [ + { + "CIRCLECI": "circleCI", + "CIRCLE_BUILD_NUM": "circleci-pipeline-number", + "CIRCLE_BUILD_URL": "https://circleci-build-url.com/", + "CIRCLE_JOB": "circleci-job-name", + "CIRCLE_PROJECT_REPONAME": "circleci-pipeline-name", + "CIRCLE_REPOSITORY_URL": "https://github.com/DataDog/dogweb", + "CIRCLE_SHA1": "b9f0fb3fdbb94c9d24b2c75b49663122a529e123", + "CIRCLE_WORKFLOW_ID": "circleci-pipeline-id", + "DD_TEST_CASE_NAME": "http-repository-url-no-git-suffix" + }, + { + "_dd.ci.env_vars": "{\"CIRCLE_WORKFLOW_ID\":\"circleci-pipeline-id\",\"CIRCLE_BUILD_NUM\":\"circleci-pipeline-number\"}", + "ci.job.name": "circleci-job-name", + "ci.job.url": "https://circleci-build-url.com/", + "ci.pipeline.id": "circleci-pipeline-id", + "ci.pipeline.name": "circleci-pipeline-name", + "ci.pipeline.url": "https://app.circleci.com/pipelines/workflows/circleci-pipeline-id", + "ci.provider.name": "circleci", + "git.commit.sha": "b9f0fb3fdbb94c9d24b2c75b49663122a529e123", + "git.repository_url": "https://github.com/DataDog/dogweb" + } + ], + [ + { + "CIRCLECI": "circleCI", + "CIRCLE_BUILD_NUM": "circleci-pipeline-number", + "CIRCLE_BUILD_URL": "https://circleci-build-url.com/", + "CIRCLE_JOB": "circleci-job-name", + "CIRCLE_PROJECT_REPONAME": "circleci-pipeline-name", + "CIRCLE_REPOSITORY_URL": "ssh://host.xz:port/path/to/repo/", + "CIRCLE_SHA1": "b9f0fb3fdbb94c9d24b2c75b49663122a529e123", + "CIRCLE_WORKFLOW_ID": "circleci-pipeline-id", + "DD_TEST_CASE_NAME": "ssh-repository-url-no-git-suffix" + }, + { + "_dd.ci.env_vars": "{\"CIRCLE_WORKFLOW_ID\":\"circleci-pipeline-id\",\"CIRCLE_BUILD_NUM\":\"circleci-pipeline-number\"}", + "ci.job.name": "circleci-job-name", + "ci.job.url": "https://circleci-build-url.com/", + "ci.pipeline.id": "circleci-pipeline-id", + "ci.pipeline.name": "circleci-pipeline-name", + "ci.pipeline.url": "https://app.circleci.com/pipelines/workflows/circleci-pipeline-id", + "ci.provider.name": "circleci", + "git.commit.sha": "b9f0fb3fdbb94c9d24b2c75b49663122a529e123", + "git.repository_url": "ssh://host.xz:port/path/to/repo/" + } + ], [ { "CIRCLECI": "circleCI", diff --git a/packages/dd-trace/test/plugins/util/ci-env/codefresh.json b/packages/dd-trace/test/plugins/util/ci-env/codefresh.json index 7b1367b4f09..d719df10592 100644 --- a/packages/dd-trace/test/plugins/util/ci-env/codefresh.json +++ b/packages/dd-trace/test/plugins/util/ci-env/codefresh.json @@ -158,167 +158,5 @@ "git.repository_url": "git@github.com:DataDog/userrepo.git", "git.tag": "0.0.2" } - ], - [ - { - "CF_BUILD_ID": "6410367cee516146a4c4c66e", - "CF_BUILD_URL": "https://g.codefresh.io/build/6410367cee516146a4c4c66e", - "CF_PIPELINE_NAME": "My simple project/Example Java Project Pipeline", - "CF_STEP_NAME": "mah-job-name", - "DD_GIT_REPOSITORY_URL": "https://user:password@github.com/DataDog/dogweb.git" - }, - { - "_dd.ci.env_vars": "{\"CF_BUILD_ID\":\"6410367cee516146a4c4c66e\"}", - "ci.job.name": "mah-job-name", - "ci.pipeline.id": "6410367cee516146a4c4c66e", - "ci.pipeline.name": "My simple project/Example Java Project Pipeline", - "ci.pipeline.url": "https://g.codefresh.io/build/6410367cee516146a4c4c66e", - "ci.provider.name": "codefresh", - "git.repository_url": "https://github.com/DataDog/dogweb.git" - } - ], - [ - { - "CF_BUILD_ID": "6410367cee516146a4c4c66e", - "CF_BUILD_URL": "https://g.codefresh.io/build/6410367cee516146a4c4c66e", - "CF_PIPELINE_NAME": "My simple project/Example Java Project Pipeline", - "CF_STEP_NAME": "mah-job-name", - "DD_GIT_REPOSITORY_URL": "https://user@github.com/DataDog/dogweb.git" - }, - { - "_dd.ci.env_vars": "{\"CF_BUILD_ID\":\"6410367cee516146a4c4c66e\"}", - "ci.job.name": "mah-job-name", - "ci.pipeline.id": "6410367cee516146a4c4c66e", - "ci.pipeline.name": "My simple project/Example Java Project Pipeline", - "ci.pipeline.url": "https://g.codefresh.io/build/6410367cee516146a4c4c66e", - "ci.provider.name": "codefresh", - "git.repository_url": "https://github.com/DataDog/dogweb.git" - } - ], - [ - { - "CF_BUILD_ID": "6410367cee516146a4c4c66e", - "CF_BUILD_URL": "https://g.codefresh.io/build/6410367cee516146a4c4c66e", - "CF_PIPELINE_NAME": "My simple project/Example Java Project Pipeline", - "CF_STEP_NAME": "mah-job-name", - "DD_GIT_REPOSITORY_URL": "https://user:password@github.com:1234/DataDog/dogweb.git" - }, - { - "_dd.ci.env_vars": "{\"CF_BUILD_ID\":\"6410367cee516146a4c4c66e\"}", - "ci.job.name": "mah-job-name", - "ci.pipeline.id": "6410367cee516146a4c4c66e", - "ci.pipeline.name": "My simple project/Example Java Project Pipeline", - "ci.pipeline.url": "https://g.codefresh.io/build/6410367cee516146a4c4c66e", - "ci.provider.name": "codefresh", - "git.repository_url": "https://github.com:1234/DataDog/dogweb.git" - } - ], - [ - { - "CF_BUILD_ID": "6410367cee516146a4c4c66e", - "CF_BUILD_URL": "https://g.codefresh.io/build/6410367cee516146a4c4c66e", - "CF_PIPELINE_NAME": "My simple project/Example Java Project Pipeline", - "CF_STEP_NAME": "mah-job-name", - "DD_GIT_REPOSITORY_URL": "https://user:password@1.1.1.1/DataDog/dogweb.git" - }, - { - "_dd.ci.env_vars": "{\"CF_BUILD_ID\":\"6410367cee516146a4c4c66e\"}", - "ci.job.name": "mah-job-name", - "ci.pipeline.id": "6410367cee516146a4c4c66e", - "ci.pipeline.name": "My simple project/Example Java Project Pipeline", - "ci.pipeline.url": "https://g.codefresh.io/build/6410367cee516146a4c4c66e", - "ci.provider.name": "codefresh", - "git.repository_url": "https://1.1.1.1/DataDog/dogweb.git" - } - ], - [ - { - "CF_BUILD_ID": "6410367cee516146a4c4c66e", - "CF_BUILD_URL": "https://g.codefresh.io/build/6410367cee516146a4c4c66e", - "CF_PIPELINE_NAME": "My simple project/Example Java Project Pipeline", - "CF_STEP_NAME": "mah-job-name", - "DD_GIT_REPOSITORY_URL": "https://user:password@1.1.1.1/DataDog/dogweb.git" - }, - { - "_dd.ci.env_vars": "{\"CF_BUILD_ID\":\"6410367cee516146a4c4c66e\"}", - "ci.job.name": "mah-job-name", - "ci.pipeline.id": "6410367cee516146a4c4c66e", - "ci.pipeline.name": "My simple project/Example Java Project Pipeline", - "ci.pipeline.url": "https://g.codefresh.io/build/6410367cee516146a4c4c66e", - "ci.provider.name": "codefresh", - "git.repository_url": "https://1.1.1.1/DataDog/dogweb.git" - } - ], - [ - { - "CF_BUILD_ID": "6410367cee516146a4c4c66e", - "CF_BUILD_URL": "https://g.codefresh.io/build/6410367cee516146a4c4c66e", - "CF_PIPELINE_NAME": "My simple project/Example Java Project Pipeline", - "CF_STEP_NAME": "mah-job-name", - "DD_GIT_REPOSITORY_URL": "https://user:password@1.1.1.1:1234/DataDog/dogweb.git" - }, - { - "_dd.ci.env_vars": "{\"CF_BUILD_ID\":\"6410367cee516146a4c4c66e\"}", - "ci.job.name": "mah-job-name", - "ci.pipeline.id": "6410367cee516146a4c4c66e", - "ci.pipeline.name": "My simple project/Example Java Project Pipeline", - "ci.pipeline.url": "https://g.codefresh.io/build/6410367cee516146a4c4c66e", - "ci.provider.name": "codefresh", - "git.repository_url": "https://1.1.1.1:1234/DataDog/dogweb.git" - } - ], - [ - { - "CF_BUILD_ID": "6410367cee516146a4c4c66e", - "CF_BUILD_URL": "https://g.codefresh.io/build/6410367cee516146a4c4c66e", - "CF_PIPELINE_NAME": "My simple project/Example Java Project Pipeline", - "CF_STEP_NAME": "mah-job-name", - "DD_GIT_REPOSITORY_URL": "https://user:password@1.1.1.1:1234/DataDog/dogweb_with_@_yeah.git" - }, - { - "_dd.ci.env_vars": "{\"CF_BUILD_ID\":\"6410367cee516146a4c4c66e\"}", - "ci.job.name": "mah-job-name", - "ci.pipeline.id": "6410367cee516146a4c4c66e", - "ci.pipeline.name": "My simple project/Example Java Project Pipeline", - "ci.pipeline.url": "https://g.codefresh.io/build/6410367cee516146a4c4c66e", - "ci.provider.name": "codefresh", - "git.repository_url": "https://1.1.1.1:1234/DataDog/dogweb_with_@_yeah.git" - } - ], - [ - { - "CF_BUILD_ID": "6410367cee516146a4c4c66e", - "CF_BUILD_URL": "https://g.codefresh.io/build/6410367cee516146a4c4c66e", - "CF_PIPELINE_NAME": "My simple project/Example Java Project Pipeline", - "CF_STEP_NAME": "mah-job-name", - "DD_GIT_REPOSITORY_URL": "ssh://user@host.xz:port/path/to/repo.git/" - }, - { - "_dd.ci.env_vars": "{\"CF_BUILD_ID\":\"6410367cee516146a4c4c66e\"}", - "ci.job.name": "mah-job-name", - "ci.pipeline.id": "6410367cee516146a4c4c66e", - "ci.pipeline.name": "My simple project/Example Java Project Pipeline", - "ci.pipeline.url": "https://g.codefresh.io/build/6410367cee516146a4c4c66e", - "ci.provider.name": "codefresh", - "git.repository_url": "ssh://host.xz:port/path/to/repo.git/" - } - ], - [ - { - "CF_BUILD_ID": "6410367cee516146a4c4c66e", - "CF_BUILD_URL": "https://g.codefresh.io/build/6410367cee516146a4c4c66e", - "CF_PIPELINE_NAME": "My simple project/Example Java Project Pipeline", - "CF_STEP_NAME": "mah-job-name", - "DD_GIT_REPOSITORY_URL": "ssh://user:password@host.xz:port/path/to/repo.git/" - }, - { - "_dd.ci.env_vars": "{\"CF_BUILD_ID\":\"6410367cee516146a4c4c66e\"}", - "ci.job.name": "mah-job-name", - "ci.pipeline.id": "6410367cee516146a4c4c66e", - "ci.pipeline.name": "My simple project/Example Java Project Pipeline", - "ci.pipeline.url": "https://g.codefresh.io/build/6410367cee516146a4c4c66e", - "ci.provider.name": "codefresh", - "git.repository_url": "ssh://host.xz:port/path/to/repo.git/" - } ] ] diff --git a/packages/dd-trace/test/plugins/util/ci-env/gitlab.json b/packages/dd-trace/test/plugins/util/ci-env/gitlab.json index 400d99c977d..7556df309e6 100644 --- a/packages/dd-trace/test/plugins/util/ci-env/gitlab.json +++ b/packages/dd-trace/test/plugins/util/ci-env/gitlab.json @@ -451,6 +451,88 @@ "git.tag": "0.1.0" } ], + [ + { + "CI_COMMIT_AUTHOR": "John Doe ", + "CI_COMMIT_MESSAGE": "gitlab-git-commit-message", + "CI_COMMIT_REF_NAME": "origin/master", + "CI_COMMIT_SHA": "b9f0fb3fdbb94c9d24b2c75b49663122a529e123", + "CI_COMMIT_TIMESTAMP": "2021-07-21T11:43:07-04:00", + "CI_JOB_ID": "gitlab-job-id", + "CI_JOB_NAME": "gitlab-job-name", + "CI_JOB_STAGE": "gitlab-stage-name", + "CI_JOB_URL": "https://gitlab.com/job", + "CI_PIPELINE_ID": "gitlab-pipeline-id", + "CI_PIPELINE_IID": "gitlab-pipeline-number", + "CI_PIPELINE_URL": "https://foo/repo/-/pipelines/1234", + "CI_PROJECT_DIR": "/foo/bar", + "CI_PROJECT_PATH": "gitlab-pipeline-name", + "CI_PROJECT_URL": "https://gitlab.com/repo", + "CI_REPOSITORY_URL": "http://hostname.com/repo", + "DD_TEST_CASE_NAME": "http-repository-url-no-git-suffix", + "GITLAB_CI": "gitlab" + }, + { + "_dd.ci.env_vars": "{\"CI_PROJECT_URL\":\"https://gitlab.com/repo\",\"CI_PIPELINE_ID\":\"gitlab-pipeline-id\",\"CI_JOB_ID\":\"gitlab-job-id\"}", + "ci.job.name": "gitlab-job-name", + "ci.job.url": "https://gitlab.com/job", + "ci.pipeline.id": "gitlab-pipeline-id", + "ci.pipeline.name": "gitlab-pipeline-name", + "ci.pipeline.number": "gitlab-pipeline-number", + "ci.pipeline.url": "https://foo/repo/-/pipelines/1234", + "ci.provider.name": "gitlab", + "ci.stage.name": "gitlab-stage-name", + "ci.workspace_path": "/foo/bar", + "git.branch": "master", + "git.commit.author.date": "2021-07-21T11:43:07-04:00", + "git.commit.author.email": "john@doe.com", + "git.commit.author.name": "John Doe", + "git.commit.message": "gitlab-git-commit-message", + "git.commit.sha": "b9f0fb3fdbb94c9d24b2c75b49663122a529e123", + "git.repository_url": "http://hostname.com/repo" + } + ], + [ + { + "CI_COMMIT_AUTHOR": "John Doe ", + "CI_COMMIT_MESSAGE": "gitlab-git-commit-message", + "CI_COMMIT_REF_NAME": "origin/master", + "CI_COMMIT_SHA": "b9f0fb3fdbb94c9d24b2c75b49663122a529e123", + "CI_COMMIT_TIMESTAMP": "2021-07-21T11:43:07-04:00", + "CI_JOB_ID": "gitlab-job-id", + "CI_JOB_NAME": "gitlab-job-name", + "CI_JOB_STAGE": "gitlab-stage-name", + "CI_JOB_URL": "https://gitlab.com/job", + "CI_PIPELINE_ID": "gitlab-pipeline-id", + "CI_PIPELINE_IID": "gitlab-pipeline-number", + "CI_PIPELINE_URL": "https://foo/repo/-/pipelines/1234", + "CI_PROJECT_DIR": "/foo/bar", + "CI_PROJECT_PATH": "gitlab-pipeline-name", + "CI_PROJECT_URL": "https://gitlab.com/repo", + "CI_REPOSITORY_URL": "ssh://host.xz:port/path/to/repo/", + "DD_TEST_CASE_NAME": "ssh-repository-url-no-git-suffix", + "GITLAB_CI": "gitlab" + }, + { + "_dd.ci.env_vars": "{\"CI_PROJECT_URL\":\"https://gitlab.com/repo\",\"CI_PIPELINE_ID\":\"gitlab-pipeline-id\",\"CI_JOB_ID\":\"gitlab-job-id\"}", + "ci.job.name": "gitlab-job-name", + "ci.job.url": "https://gitlab.com/job", + "ci.pipeline.id": "gitlab-pipeline-id", + "ci.pipeline.name": "gitlab-pipeline-name", + "ci.pipeline.number": "gitlab-pipeline-number", + "ci.pipeline.url": "https://foo/repo/-/pipelines/1234", + "ci.provider.name": "gitlab", + "ci.stage.name": "gitlab-stage-name", + "ci.workspace_path": "/foo/bar", + "git.branch": "master", + "git.commit.author.date": "2021-07-21T11:43:07-04:00", + "git.commit.author.email": "john@doe.com", + "git.commit.author.name": "John Doe", + "git.commit.message": "gitlab-git-commit-message", + "git.commit.sha": "b9f0fb3fdbb94c9d24b2c75b49663122a529e123", + "git.repository_url": "ssh://host.xz:port/path/to/repo/" + } + ], [ { "CI_COMMIT_AUTHOR": "John Doe ", diff --git a/packages/dd-trace/test/plugins/util/ci-env/jenkins.json b/packages/dd-trace/test/plugins/util/ci-env/jenkins.json index f87cdbd2a36..045c27270aa 100644 --- a/packages/dd-trace/test/plugins/util/ci-env/jenkins.json +++ b/packages/dd-trace/test/plugins/util/ci-env/jenkins.json @@ -666,6 +666,50 @@ "git.tag": "0.0.2" } ], + [ + { + "BUILD_NUMBER": "jenkins-pipeline-number", + "BUILD_TAG": "jenkins-pipeline-id", + "BUILD_URL": "https://jenkins.com/pipeline", + "DD_CUSTOM_TRACE_ID": "jenkins-custom-trace-id", + "DD_TEST_CASE_NAME": "http-repository-url-no-git-suffix", + "GIT_COMMIT": "b9f0fb3fdbb94c9d24b2c75b49663122a529e123", + "GIT_URL_1": "https://github.com/DataDog/dogweb", + "JENKINS_URL": "jenkins", + "JOB_URL": "https://jenkins.com/job" + }, + { + "_dd.ci.env_vars": "{\"DD_CUSTOM_TRACE_ID\":\"jenkins-custom-trace-id\"}", + "ci.pipeline.id": "jenkins-pipeline-id", + "ci.pipeline.number": "jenkins-pipeline-number", + "ci.pipeline.url": "https://jenkins.com/pipeline", + "ci.provider.name": "jenkins", + "git.commit.sha": "b9f0fb3fdbb94c9d24b2c75b49663122a529e123", + "git.repository_url": "https://github.com/DataDog/dogweb" + } + ], + [ + { + "BUILD_NUMBER": "jenkins-pipeline-number", + "BUILD_TAG": "jenkins-pipeline-id", + "BUILD_URL": "https://jenkins.com/pipeline", + "DD_CUSTOM_TRACE_ID": "jenkins-custom-trace-id", + "DD_TEST_CASE_NAME": "ssh-repository-url-no-git-suffix", + "GIT_COMMIT": "b9f0fb3fdbb94c9d24b2c75b49663122a529e123", + "GIT_URL_1": "ssh://host.xz:port/path/to/repo/", + "JENKINS_URL": "jenkins", + "JOB_URL": "https://jenkins.com/job" + }, + { + "_dd.ci.env_vars": "{\"DD_CUSTOM_TRACE_ID\":\"jenkins-custom-trace-id\"}", + "ci.pipeline.id": "jenkins-pipeline-id", + "ci.pipeline.number": "jenkins-pipeline-number", + "ci.pipeline.url": "https://jenkins.com/pipeline", + "ci.provider.name": "jenkins", + "git.commit.sha": "b9f0fb3fdbb94c9d24b2c75b49663122a529e123", + "git.repository_url": "ssh://host.xz:port/path/to/repo/" + } + ], [ { "BUILD_NUMBER": "jenkins-pipeline-number", diff --git a/packages/dd-trace/test/plugins/util/ci-env/teamcity.json b/packages/dd-trace/test/plugins/util/ci-env/teamcity.json index 037887c4ae0..086c1c16de1 100644 --- a/packages/dd-trace/test/plugins/util/ci-env/teamcity.json +++ b/packages/dd-trace/test/plugins/util/ci-env/teamcity.json @@ -74,117 +74,5 @@ "git.repository_url": "git@github.com:DataDog/userrepo.git", "git.tag": "0.0.2" } - ], - [ - { - "BUILD_URL": "https://teamcity.com/repo", - "DD_GIT_REPOSITORY_URL": "https://user:password@github.com/DataDog/dogweb.git", - "TEAMCITY_BUILDCONF_NAME": "Test 1", - "TEAMCITY_VERSION": "2022.10 (build 116751)" - }, - { - "ci.job.name": "Test 1", - "ci.job.url": "https://teamcity.com/repo", - "ci.provider.name": "teamcity", - "git.repository_url": "https://github.com/DataDog/dogweb.git" - } - ], - [ - { - "BUILD_URL": "https://teamcity.com/repo", - "DD_GIT_REPOSITORY_URL": "https://user@github.com/DataDog/dogweb.git", - "TEAMCITY_BUILDCONF_NAME": "Test 1", - "TEAMCITY_VERSION": "2022.10 (build 116751)" - }, - { - "ci.job.name": "Test 1", - "ci.job.url": "https://teamcity.com/repo", - "ci.provider.name": "teamcity", - "git.repository_url": "https://github.com/DataDog/dogweb.git" - } - ], - [ - { - "BUILD_URL": "https://teamcity.com/repo", - "DD_GIT_REPOSITORY_URL": "https://user:password@github.com:1234/DataDog/dogweb.git", - "TEAMCITY_BUILDCONF_NAME": "Test 1", - "TEAMCITY_VERSION": "2022.10 (build 116751)" - }, - { - "ci.job.name": "Test 1", - "ci.job.url": "https://teamcity.com/repo", - "ci.provider.name": "teamcity", - "git.repository_url": "https://github.com:1234/DataDog/dogweb.git" - } - ], - [ - { - "BUILD_URL": "https://teamcity.com/repo", - "DD_GIT_REPOSITORY_URL": "https://user:password@1.1.1.1/DataDog/dogweb.git", - "TEAMCITY_BUILDCONF_NAME": "Test 1", - "TEAMCITY_VERSION": "2022.10 (build 116751)" - }, - { - "ci.job.name": "Test 1", - "ci.job.url": "https://teamcity.com/repo", - "ci.provider.name": "teamcity", - "git.repository_url": "https://1.1.1.1/DataDog/dogweb.git" - } - ], - [ - { - "BUILD_URL": "https://teamcity.com/repo", - "DD_GIT_REPOSITORY_URL": "https://user:password@1.1.1.1:1234/DataDog/dogweb.git", - "TEAMCITY_BUILDCONF_NAME": "Test 1", - "TEAMCITY_VERSION": "2022.10 (build 116751)" - }, - { - "ci.job.name": "Test 1", - "ci.job.url": "https://teamcity.com/repo", - "ci.provider.name": "teamcity", - "git.repository_url": "https://1.1.1.1:1234/DataDog/dogweb.git" - } - ], - [ - { - "BUILD_URL": "https://teamcity.com/repo", - "DD_GIT_REPOSITORY_URL": "https://user:password@1.1.1.1:1234/DataDog/dogweb_with_@_yeah.git", - "TEAMCITY_BUILDCONF_NAME": "Test 1", - "TEAMCITY_VERSION": "2022.10 (build 116751)" - }, - { - "ci.job.name": "Test 1", - "ci.job.url": "https://teamcity.com/repo", - "ci.provider.name": "teamcity", - "git.repository_url": "https://1.1.1.1:1234/DataDog/dogweb_with_@_yeah.git" - } - ], - [ - { - "BUILD_URL": "https://teamcity.com/repo", - "DD_GIT_REPOSITORY_URL": "ssh://user@host.xz:port/path/to/repo.git/", - "TEAMCITY_BUILDCONF_NAME": "Test 1", - "TEAMCITY_VERSION": "2022.10 (build 116751)" - }, - { - "ci.job.name": "Test 1", - "ci.job.url": "https://teamcity.com/repo", - "ci.provider.name": "teamcity", - "git.repository_url": "ssh://host.xz:port/path/to/repo.git/" - } - ], - [ - { - "BUILD_URL": "https://teamcity.com/repo", - "DD_GIT_REPOSITORY_URL": "ssh://user:password@host.xz:port/path/to/repo.git/", - "TEAMCITY_BUILDCONF_NAME": "Test 1", - "TEAMCITY_VERSION": "2022.10 (build 116751)" - }, - { - "ci.job.name": "Test 1", - "ci.job.url": "https://teamcity.com/repo", - "ci.provider.name": "teamcity", - "git.repository_url": "ssh://host.xz:port/path/to/repo.git/" - } ] ] diff --git a/packages/dd-trace/test/plugins/util/ci-env/usersupplied.json b/packages/dd-trace/test/plugins/util/ci-env/usersupplied.json index 464c4158558..9a151c6c00e 100644 --- a/packages/dd-trace/test/plugins/util/ci-env/usersupplied.json +++ b/packages/dd-trace/test/plugins/util/ci-env/usersupplied.json @@ -155,6 +155,56 @@ "git.tag": "0.0.2" } ], + [ + { + "DD_GIT_COMMIT_AUTHOR_DATE": "usersupplied-authordate", + "DD_GIT_COMMIT_AUTHOR_EMAIL": "usersupplied-authoremail", + "DD_GIT_COMMIT_AUTHOR_NAME": "usersupplied-authorname", + "DD_GIT_COMMIT_COMMITTER_DATE": "usersupplied-comitterdate", + "DD_GIT_COMMIT_COMMITTER_EMAIL": "usersupplied-comitteremail", + "DD_GIT_COMMIT_COMMITTER_NAME": "usersupplied-comittername", + "DD_GIT_COMMIT_MESSAGE": "usersupplied-message", + "DD_GIT_COMMIT_SHA": "b9f0fb3fdbb94c9d24b2c75b49663122a529e123", + "DD_GIT_REPOSITORY_URL": "https://github.com/DataDog/dogweb", + "DD_TEST_CASE_NAME": "http-repository-url-no-git-suffix" + }, + { + "git.commit.author.date": "usersupplied-authordate", + "git.commit.author.email": "usersupplied-authoremail", + "git.commit.author.name": "usersupplied-authorname", + "git.commit.committer.date": "usersupplied-comitterdate", + "git.commit.committer.email": "usersupplied-comitteremail", + "git.commit.committer.name": "usersupplied-comittername", + "git.commit.message": "usersupplied-message", + "git.commit.sha": "b9f0fb3fdbb94c9d24b2c75b49663122a529e123", + "git.repository_url": "https://github.com/DataDog/dogweb" + } + ], + [ + { + "DD_GIT_COMMIT_AUTHOR_DATE": "usersupplied-authordate", + "DD_GIT_COMMIT_AUTHOR_EMAIL": "usersupplied-authoremail", + "DD_GIT_COMMIT_AUTHOR_NAME": "usersupplied-authorname", + "DD_GIT_COMMIT_COMMITTER_DATE": "usersupplied-comitterdate", + "DD_GIT_COMMIT_COMMITTER_EMAIL": "usersupplied-comitteremail", + "DD_GIT_COMMIT_COMMITTER_NAME": "usersupplied-comittername", + "DD_GIT_COMMIT_MESSAGE": "usersupplied-message", + "DD_GIT_COMMIT_SHA": "b9f0fb3fdbb94c9d24b2c75b49663122a529e123", + "DD_GIT_REPOSITORY_URL": "ssh://host.xz:port/path/to/repo/", + "DD_TEST_CASE_NAME": "ssh-repository-url-no-git-suffix" + }, + { + "git.commit.author.date": "usersupplied-authordate", + "git.commit.author.email": "usersupplied-authoremail", + "git.commit.author.name": "usersupplied-authorname", + "git.commit.committer.date": "usersupplied-comitterdate", + "git.commit.committer.email": "usersupplied-comitteremail", + "git.commit.committer.name": "usersupplied-comittername", + "git.commit.message": "usersupplied-message", + "git.commit.sha": "b9f0fb3fdbb94c9d24b2c75b49663122a529e123", + "git.repository_url": "ssh://host.xz:port/path/to/repo/" + } + ], [ { "DD_GIT_COMMIT_AUTHOR_DATE": "usersupplied-authordate", diff --git a/packages/dd-trace/test/plugins/util/env.spec.js b/packages/dd-trace/test/plugins/util/env.spec.js index d3cd7bf47e3..5a799897df4 100644 --- a/packages/dd-trace/test/plugins/util/env.spec.js +++ b/packages/dd-trace/test/plugins/util/env.spec.js @@ -9,7 +9,8 @@ const { OS_PLATFORM, OS_VERSION, RUNTIME_NAME, - RUNTIME_VERSION + RUNTIME_VERSION, + DD_HOST_CPU_COUNT } = require('../../../src/plugins/util/env') describe('env', () => { @@ -22,7 +23,8 @@ describe('env', () => { [OS_ARCHITECTURE]: process.arch, [OS_PLATFORM]: process.platform, [RUNTIME_NAME]: 'node', - [OS_VERSION]: os.release() + [OS_VERSION]: os.release(), + [DD_HOST_CPU_COUNT]: os.cpus().length } ) }) diff --git a/packages/dd-trace/test/plugins/util/fixtures/github_event_payload.json b/packages/dd-trace/test/plugins/util/fixtures/github_event_payload.json new file mode 100644 index 00000000000..64828fe2b7b --- /dev/null +++ b/packages/dd-trace/test/plugins/util/fixtures/github_event_payload.json @@ -0,0 +1,70 @@ +{ + "action": "synchronize", + "after": "df289512a51123083a8e6931dd6f57bb3883d4c4", + "before": "f659d2fdd7bedffb40d9ab223dbde6afa5eadc32", + "number": 1, + "pull_request": { + "_links": {}, + "active_lock_reason": null, + "additions": 2, + "assignee": null, + "assignees": [], + "author_association": "OWNER", + "auto_merge": null, + "base": { + "label": "datadog:main", + "ref": "main", + "repo": {}, + "sha": "52e0974c74d41160a03d59ddc73bb9f5adab054b", + "user": {} + }, + "body": "# What Does This Do\r\n\r\n# Motivation\r\n\r\n# Additional Notes\r\n", + "changed_files": 3, + "closed_at": null, + "comments": 0, + "comments_url": "", + "commits": 2, + "commits_url": "", + "created_at": "2024-09-11T15:08:02Z", + "deletions": 0, + "diff_url": "", + "draft": false, + "head": { + "label": "forked_org:test-branch", + "ref": "test-branch", + "repo": {}, + "sha": "df289512a51123083a8e6931dd6f57bb3883d4c4", + "user": {} + }, + "html_url": "", + "id": 2066570986, + "issue_url": "", + "labels": [], + "locked": false, + "maintainer_can_modify": false, + "merge_commit_sha": "d9a3212d0d5d1483426dbbdf0beea32ee50abcde", + "mergeable": null, + "mergeable_state": "unknown", + "merged": false, + "merged_at": null, + "merged_by": null, + "milestone": null, + "node_id": "PR_kwDOIvpGAs57LV7q", + "number": 1, + "patch_url": "", + "rebaseable": null, + "requested_reviewers": [], + "requested_teams": [], + "review_comment_url": "", + "review_comments": 0, + "review_comments_url": "", + "state": "open", + "statuses_url": "", + "title": "Test commit", + "updated_at": "2024-09-11T15:12:26Z", + "url": "", + "user": {} + }, + "repository": {}, + "sender": {} +} diff --git a/packages/dd-trace/test/plugins/util/fixtures/github_event_payload_malformed.json b/packages/dd-trace/test/plugins/util/fixtures/github_event_payload_malformed.json new file mode 100644 index 00000000000..0967ef424bc --- /dev/null +++ b/packages/dd-trace/test/plugins/util/fixtures/github_event_payload_malformed.json @@ -0,0 +1 @@ +{} diff --git a/packages/dd-trace/test/plugins/util/git.spec.js b/packages/dd-trace/test/plugins/util/git.spec.js index 90553564f98..9c971701f89 100644 --- a/packages/dd-trace/test/plugins/util/git.spec.js +++ b/packages/dd-trace/test/plugins/util/git.spec.js @@ -7,9 +7,8 @@ const os = require('os') const fs = require('fs') const path = require('path') -const { GIT_REV_LIST_MAX_BUFFER } = require('../../../src/plugins/util/git') +const { GIT_REV_LIST_MAX_BUFFER, isGitAvailable } = require('../../../src/plugins/util/git') const proxyquire = require('proxyquire') -const sanitizedExecStub = sinon.stub().returns('') const execFileSyncStub = sinon.stub().returns('') const { @@ -29,10 +28,7 @@ const { const { getGitMetadata, unshallowRepository } = proxyquire('../../../src/plugins/util/git', { - './exec': { - sanitizedExec: sanitizedExecStub - }, - 'child_process': { + child_process: { execFileSync: execFileSyncStub } } @@ -47,7 +43,7 @@ function getFakeDirectory () { describe('git', () => { afterEach(() => { - sanitizedExecStub.reset() + execFileSyncStub.reset() delete process.env.DD_GIT_COMMIT_SHA delete process.env.DD_GIT_REPOSITORY_URL delete process.env.DD_GIT_BRANCH @@ -60,6 +56,7 @@ describe('git', () => { delete process.env.DD_GIT_COMMIT_COMMITTER_EMAIL delete process.env.DD_GIT_COMMIT_COMMITTER_DATE }) + it('returns ci metadata if it is present and does not call git for those parameters', () => { const ciMetadata = { commitSHA: 'ciSHA', @@ -80,15 +77,16 @@ describe('git', () => { } ) expect(metadata[GIT_REPOSITORY_URL]).not.to.equal('ciRepositoryUrl') - expect(sanitizedExecStub).to.have.been.calledWith('git', ['ls-remote', '--get-url']) - expect(sanitizedExecStub).to.have.been.calledWith('git', ['show', '-s', '--format=%an,%ae,%aI,%cn,%ce,%cI']) - expect(sanitizedExecStub).not.to.have.been.calledWith('git', ['show', '-s', '--format=%s']) - expect(sanitizedExecStub).not.to.have.been.calledWith('git', ['rev-parse', 'HEAD']) - expect(sanitizedExecStub).not.to.have.been.calledWith('git', ['rev-parse', '--abbrev-ref', 'HEAD']) - expect(sanitizedExecStub).not.to.have.been.calledWith('git', ['rev-parse', '--show-toplevel']) + expect(execFileSyncStub).to.have.been.calledWith('git', ['ls-remote', '--get-url']) + expect(execFileSyncStub).to.have.been.calledWith('git', ['show', '-s', '--format=%an,%ae,%aI,%cn,%ce,%cI']) + expect(execFileSyncStub).not.to.have.been.calledWith('git', ['show', '-s', '--format=%s']) + expect(execFileSyncStub).not.to.have.been.calledWith('git', ['rev-parse', 'HEAD']) + expect(execFileSyncStub).not.to.have.been.calledWith('git', ['rev-parse', '--abbrev-ref', 'HEAD']) + expect(execFileSyncStub).not.to.have.been.calledWith('git', ['rev-parse', '--show-toplevel']) }) + it('does not crash if git is not available', () => { - sanitizedExecStub.returns('') + execFileSyncStub.returns('') const ciMetadata = { repositoryUrl: 'https://github.com/datadog/safe-repository.git' } const metadata = getGitMetadata(ciMetadata) expect(metadata).to.eql({ @@ -106,8 +104,9 @@ describe('git', () => { [CI_WORKSPACE_PATH]: '' }) }) + it('returns all git metadata is git is available', () => { - sanitizedExecStub + execFileSyncStub .onCall(0).returns( 'git author,git.author@email.com,2022-02-14T16:22:03-05:00,' + 'git committer,git.committer@email.com,2022-02-14T16:23:03-05:00' @@ -133,23 +132,23 @@ describe('git', () => { [GIT_COMMIT_COMMITTER_NAME]: 'git committer', [CI_WORKSPACE_PATH]: 'ciWorkspacePath' }) - expect(sanitizedExecStub).to.have.been.calledWith('git', ['ls-remote', '--get-url']) - expect(sanitizedExecStub).to.have.been.calledWith('git', ['show', '-s', '--format=%s']) - expect(sanitizedExecStub).to.have.been.calledWith('git', ['show', '-s', '--format=%an,%ae,%aI,%cn,%ce,%cI']) - expect(sanitizedExecStub).to.have.been.calledWith('git', ['rev-parse', 'HEAD']) - expect(sanitizedExecStub).to.have.been.calledWith('git', ['rev-parse', '--abbrev-ref', 'HEAD']) - expect(sanitizedExecStub).to.have.been.calledWith('git', ['rev-parse', '--show-toplevel']) + expect(execFileSyncStub).to.have.been.calledWith('git', ['ls-remote', '--get-url']) + expect(execFileSyncStub).to.have.been.calledWith('git', ['show', '-s', '--format=%s']) + expect(execFileSyncStub).to.have.been.calledWith('git', ['show', '-s', '--format=%an,%ae,%aI,%cn,%ce,%cI']) + expect(execFileSyncStub).to.have.been.calledWith('git', ['rev-parse', 'HEAD']) + expect(execFileSyncStub).to.have.been.calledWith('git', ['rev-parse', '--abbrev-ref', 'HEAD']) + expect(execFileSyncStub).to.have.been.calledWith('git', ['rev-parse', '--show-toplevel']) }) }) -describe('getCommitsToUpload', () => { +describe('getCommitsRevList', () => { it('gets the commits to upload if the repository is smaller than the limit', () => { const logErrorSpy = sinon.spy() - const { getCommitsToUpload } = proxyquire('../../../src/plugins/util/git', + const { getCommitsRevList } = proxyquire('../../../src/plugins/util/git', { - 'child_process': { - 'execFileSync': (command, flags, options) => + child_process: { + execFileSync: (command, flags, options) => execSync(`head -c ${Math.floor(GIT_REV_LIST_MAX_BUFFER * 0.9)} /dev/zero`, options) }, '../../log': { @@ -157,17 +156,17 @@ describe('getCommitsToUpload', () => { } } ) - getCommitsToUpload([], []) + getCommitsRevList([], []) expect(logErrorSpy).not.to.have.been.called }) it('does not crash and logs the error if the repository is bigger than the limit', () => { const logErrorSpy = sinon.spy() - const { getCommitsToUpload } = proxyquire('../../../src/plugins/util/git', + const { getCommitsRevList } = proxyquire('../../../src/plugins/util/git', { - 'child_process': { - 'execFileSync': (command, flags, options) => + child_process: { + execFileSync: (command, flags, options) => execSync(`head -c ${GIT_REV_LIST_MAX_BUFFER * 2} /dev/zero`, options) }, '../../log': { @@ -175,31 +174,58 @@ describe('getCommitsToUpload', () => { } } ) - const commitsToUpload = getCommitsToUpload([], []) + getCommitsRevList([], []) expect(logErrorSpy).to.have.been.called - expect(commitsToUpload.length).to.equal(0) + }) + + it('returns null if the repository is bigger than the limit', () => { + const { getCommitsRevList } = proxyquire('../../../src/plugins/util/git', + { + child_process: { + execFileSync: (command, flags, options) => + execSync(`head -c ${GIT_REV_LIST_MAX_BUFFER * 2} /dev/zero`, options) + } + } + ) + const commitsToUpload = getCommitsRevList([], []) + expect(commitsToUpload).to.be.null + }) + + it('returns null if execFileSync fails for whatever reason', () => { + const { getCommitsRevList } = proxyquire('../../../src/plugins/util/git', + { + child_process: { + execFileSync: () => { throw new Error('error!') } + } + } + ) + const commitsToUpload = getCommitsRevList([], []) + expect(commitsToUpload).to.be.null }) }) describe('generatePackFilesForCommits', () => { let tmpdirStub, statSyncStub const fakeDirectory = getFakeDirectory() + beforeEach(() => { sinon.stub(Math, 'random').returns('0.1234') tmpdirStub = sinon.stub(os, 'tmpdir').returns(fakeDirectory) sinon.stub(process, 'cwd').returns('cwd') statSyncStub = sinon.stub(fs, 'statSync').returns({ isDirectory: () => true }) }) + afterEach(() => { sinon.restore() }) + it('creates pack files in temporary path', () => { const execFileSyncSpy = sinon.stub().returns(['commitSHA']) const { generatePackFilesForCommits } = proxyquire('../../../src/plugins/util/git', { - 'child_process': { - 'execFileSync': execFileSyncSpy + child_process: { + execFileSync: execFileSyncSpy } } ) @@ -216,8 +242,8 @@ describe('generatePackFilesForCommits', () => { const { generatePackFilesForCommits } = proxyquire('../../../src/plugins/util/git', { - 'child_process': { - 'execFileSync': execFileSyncSpy + child_process: { + execFileSync: execFileSyncSpy } } ) @@ -234,8 +260,8 @@ describe('generatePackFilesForCommits', () => { const { generatePackFilesForCommits } = proxyquire('../../../src/plugins/util/git', { - 'child_process': { - 'execFileSync': execFileSyncSpy + child_process: { + execFileSync: execFileSyncSpy } } ) @@ -246,11 +272,11 @@ describe('generatePackFilesForCommits', () => { describe('unshallowRepository', () => { afterEach(() => { - sanitizedExecStub.reset() execFileSyncStub.reset() }) + it('works for the usual case', () => { - sanitizedExecStub + execFileSyncStub .onCall(0).returns( 'git version 2.39.0' ) @@ -270,17 +296,16 @@ describe('unshallowRepository', () => { unshallowRepository() expect(execFileSyncStub).to.have.been.calledWith('git', options) }) + it('works if the local HEAD is a commit that has not been pushed to the remote', () => { - sanitizedExecStub + execFileSyncStub .onCall(0).returns( 'git version 2.39.0' ) .onCall(1).returns('origin') .onCall(2).returns('daede5785233abb1a3cb76b9453d4eb5b98290b3') - .onCall(3).returns('origin/master') - - execFileSyncStub - .onCall(0).throws() + .onCall(3).throws() + .onCall(4).returns('origin/master') const options = [ 'fetch', @@ -295,18 +320,17 @@ describe('unshallowRepository', () => { unshallowRepository() expect(execFileSyncStub).to.have.been.calledWith('git', options) }) + it('works if the CI is working on a detached HEAD or branch tracking hasn’t been set up', () => { - sanitizedExecStub + execFileSyncStub .onCall(0).returns( 'git version 2.39.0' ) .onCall(1).returns('origin') .onCall(2).returns('daede5785233abb1a3cb76b9453d4eb5b98290b3') - .onCall(3).returns('origin/master') - - execFileSyncStub - .onCall(0).throws() - .onCall(1).throws() + .onCall(3).throws() + .onCall(4).returns('origin/master') + .onCall(5).throws() const options = [ 'fetch', @@ -318,17 +342,18 @@ describe('unshallowRepository', () => { ] unshallowRepository() - expect(sanitizedExecStub).to.have.been.calledWith('git', options) + expect(execFileSyncStub).to.have.been.calledWith('git', options) }) }) describe('user credentials', () => { afterEach(() => { - sanitizedExecStub.reset() + execFileSyncStub.reset() execFileSyncStub.reset() }) + it('scrubs https user credentials', () => { - sanitizedExecStub + execFileSyncStub .onCall(0).returns( 'git author,git.author@email.com,2022-02-14T16:22:03-05:00,' + 'git committer,git.committer@email.com,2022-02-14T16:23:03-05:00' @@ -339,8 +364,9 @@ describe('user credentials', () => { expect(metadata[GIT_REPOSITORY_URL]) .to.equal('https://github.com/datadog/safe-repository.git') }) + it('scrubs ssh user credentials', () => { - sanitizedExecStub + execFileSyncStub .onCall(0).returns( 'git author,git.author@email.com,2022-02-14T16:22:03-05:00,' + 'git committer,git.committer@email.com,2022-02-14T16:23:03-05:00' @@ -352,3 +378,25 @@ describe('user credentials', () => { .to.equal('ssh://host.xz:port/path/to/repo.git/') }) }) + +describe('isGitAvailable', () => { + let originalPath + + beforeEach(() => { + originalPath = process.env.PATH + }) + + afterEach(() => { + process.env.PATH = originalPath + }) + + it('returns true if git is available', () => { + expect(isGitAvailable()).to.be.true + }) + + it('returns false if git is not available', () => { + process.env.PATH = '' + + expect(isGitAvailable()).to.be.false + }) +}) diff --git a/packages/dd-trace/test/plugins/util/ip_extractor.spec.js b/packages/dd-trace/test/plugins/util/ip_extractor.spec.js index d6ca2ba950d..21e48289eef 100644 --- a/packages/dd-trace/test/plugins/util/ip_extractor.spec.js +++ b/packages/dd-trace/test/plugins/util/ip_extractor.spec.js @@ -9,11 +9,13 @@ const axios = require('axios') describe('ip extractor', () => { let port, appListener, controller + before(() => { return getPort().then(newPort => { port = newPort }) }) + before(done => { const server = new http.Server(async (req, res) => { controller && await controller(req, res) @@ -23,6 +25,7 @@ describe('ip extractor', () => { appListener = server .listen(port, 'localhost', () => done()) }) + after(() => { appListener && appListener.close() }) diff --git a/packages/dd-trace/test/plugins/util/stacktrace.spec.js b/packages/dd-trace/test/plugins/util/stacktrace.spec.js new file mode 100644 index 00000000000..3fefc2b29ef --- /dev/null +++ b/packages/dd-trace/test/plugins/util/stacktrace.spec.js @@ -0,0 +1,68 @@ +'use strict' + +const { isAbsolute } = require('path') + +require('../../setup/tap') + +const { + getCallSites, + getUserLandFrames +} = require('../../../src/plugins/util/stacktrace') + +describe('stacktrace utils', () => { + it('should get callsites array from getCallsites', () => { + const callsites = getCallSites() + expect(callsites).to.be.an('array') + expect(callsites.length).to.be.gt(0) + callsites.forEach((callsite) => { + expect(callsite).to.be.an.instanceof(Object) + expect(callsite.constructor.name).to.equal('CallSite') + expect(callsite.getFileName).to.be.an.instanceof(Function) + }) + }) + + describe('getUserLandFrames', () => { + it('should return array of frame objects', function helloWorld () { + function someFunction () { + const frames = getUserLandFrames(someFunction) + + expect(frames).to.be.an('array') + expect(frames.length).to.be.gt(1) + frames.forEach((frame) => { + expect(frame).to.be.an.instanceof(Object) + expect(frame).to.have.all.keys('file', 'line', 'column', 'method', 'type') + expect(frame.file).to.be.a('string') + expect(frame.line).to.be.gt(0) + expect(frame.column).to.be.gt(0) + expect(typeof frame.method).to.be.oneOf(['string', 'undefined']) + expect(typeof frame.type).to.be.oneOf(['string', 'undefined']) + expect(isAbsolute(frame.file)).to.be.true + }) + + const frame = frames[0] + expect(frame.file).to.equal(__filename) + expect(frame.line).to.equal(lineNumber) + expect(frame.method).to.equal('helloWorld') + expect(frame.type).to.equal('Test') + } + + const lineNumber = getNextLineNumber() + someFunction() + }) + + it('should respect limit', function helloWorld () { + (function someFunction () { + const frames = getUserLandFrames(someFunction, 1) + expect(frames.length).to.equal(1) + const frame = frames[0] + expect(frame.file).to.equal(__filename) + expect(frame.method).to.equal('helloWorld') + expect(frame.type).to.equal('Test') + })() + }) + }) +}) + +function getNextLineNumber () { + return Number(new Error().stack.split('\n')[2].match(/:(\d+):/)[1]) + 1 +} diff --git a/packages/dd-trace/test/plugins/util/test-environment.spec.js b/packages/dd-trace/test/plugins/util/test-environment.spec.js index 4b7856be4bb..63726622fb5 100644 --- a/packages/dd-trace/test/plugins/util/test-environment.spec.js +++ b/packages/dd-trace/test/plugins/util/test-environment.spec.js @@ -6,19 +6,25 @@ const fs = require('fs') const path = require('path') const proxyquire = require('proxyquire') -const sanitizedExecStub = sinon.stub().returns('') +const execFileSyncStub = sinon.stub().returns('') const { getCIMetadata } = require('../../../src/plugins/util/ci') -const { CI_ENV_VARS, CI_NODE_LABELS } = require('../../../src/plugins/util/tags') +const { + CI_ENV_VARS, + CI_NODE_LABELS, + GIT_PULL_REQUEST_BASE_BRANCH, + GIT_PULL_REQUEST_BASE_BRANCH_SHA, + GIT_COMMIT_HEAD_SHA +} = require('../../../src/plugins/util/tags') const { getGitMetadata } = proxyquire('../../../src/plugins/util/git', { - './exec': { - 'sanitizedExec': sanitizedExecStub + child_process: { + execFileSync: execFileSyncStub } }) const { getTestEnvironmentMetadata } = proxyquire('../../../src/plugins/util/test', { './git': { - 'getGitMetadata': getGitMetadata + getGitMetadata } }) @@ -27,6 +33,7 @@ describe('test environment data', () => { const tags = getTestEnvironmentMetadata('jest', { service: 'service-name' }) expect(tags).to.contain({ 'service.name': 'service-name' }) }) + it('getCIMetadata returns an empty object if the CI is not supported', () => { process.env = {} expect(getCIMetadata()).to.eql({}) @@ -35,10 +42,49 @@ describe('test environment data', () => { const ciProviders = fs.readdirSync(path.join(__dirname, 'ci-env')) ciProviders.forEach(ciProvider => { const assertions = require(path.join(__dirname, 'ci-env', ciProvider)) + if (ciProvider === 'github.json') { + // We grab the first assertion because we only need to test one + const [env] = assertions[0] + it('can read pull request data from GitHub Actions', () => { + process.env = env + process.env.GITHUB_BASE_REF = 'datadog:main' + process.env.GITHUB_EVENT_PATH = path.join(__dirname, 'fixtures', 'github_event_payload.json') + const { + [GIT_PULL_REQUEST_BASE_BRANCH]: pullRequestBaseBranch, + [GIT_PULL_REQUEST_BASE_BRANCH_SHA]: pullRequestBaseBranchSha, + [GIT_COMMIT_HEAD_SHA]: headCommitSha + } = getTestEnvironmentMetadata() + + expect({ + pullRequestBaseBranch, + pullRequestBaseBranchSha, + headCommitSha + }).to.eql({ + pullRequestBaseBranch: 'datadog:main', + pullRequestBaseBranchSha: '52e0974c74d41160a03d59ddc73bb9f5adab054b', + headCommitSha: 'df289512a51123083a8e6931dd6f57bb3883d4c4' + }) + }) + it('does not crash if GITHUB_EVENT_PATH is not a valid JSON file', () => { + process.env = env + process.env.GITHUB_BASE_REF = 'datadog:main' + process.env.GITHUB_EVENT_PATH = path.join(__dirname, 'fixtures', 'github_event_payload_malformed.json') + const { + [GIT_PULL_REQUEST_BASE_BRANCH]: pullRequestBaseBranch, + [GIT_PULL_REQUEST_BASE_BRANCH_SHA]: pullRequestBaseBranchSha, + [GIT_COMMIT_HEAD_SHA]: headCommitSha + } = getTestEnvironmentMetadata() + + expect(pullRequestBaseBranch).to.equal('datadog:main') + expect(pullRequestBaseBranchSha).to.be.undefined + expect(headCommitSha).to.be.undefined + }) + } assertions.forEach(([env, expectedSpanTags], index) => { it(`reads env info for spec ${index} from ${ciProvider}`, () => { process.env = env + const { DD_TEST_CASE_NAME: testCaseName } = env const { [CI_ENV_VARS]: envVars, [CI_NODE_LABELS]: nodeLabels, ...restOfTags } = getTestEnvironmentMetadata() const { [CI_ENV_VARS]: expectedEnvVars, @@ -46,7 +92,7 @@ describe('test environment data', () => { ...restOfExpectedTags } = expectedSpanTags - expect(restOfTags).to.contain(restOfExpectedTags) + expect(restOfTags, testCaseName ? `${testCaseName} has failed.` : undefined).to.contain(restOfExpectedTags) // `CI_ENV_VARS` key contains a dictionary, so we do a `eql` comparison if (envVars && expectedEnvVars) { expect(JSON.parse(envVars)).to.eql(JSON.parse(expectedEnvVars)) diff --git a/packages/dd-trace/test/plugins/util/test.spec.js b/packages/dd-trace/test/plugins/util/test.spec.js index 4a992955397..f79ab8fd34d 100644 --- a/packages/dd-trace/test/plugins/util/test.spec.js +++ b/packages/dd-trace/test/plugins/util/test.spec.js @@ -14,35 +14,39 @@ const { mergeCoverage, resetCoverage, removeInvalidMetadata, - parseAnnotations + parseAnnotations, + getIsFaultyEarlyFlakeDetection, + getNumFromKnownTests } = require('../../../src/plugins/util/test') const { GIT_REPOSITORY_URL, GIT_COMMIT_SHA, CI_PIPELINE_URL } = require('../../../src/plugins/util/tags') describe('getTestParametersString', () => { it('returns formatted test parameters and removes params from input', () => { - const input = { 'test_stuff': [['params'], [{ b: 'c' }]] } + const input = { test_stuff: [['params'], [{ b: 'c' }]] } expect(getTestParametersString(input, 'test_stuff')).to.equal( JSON.stringify({ arguments: ['params'], metadata: {} }) ) - expect(input).to.eql({ 'test_stuff': [[{ b: 'c' }]] }) + expect(input).to.eql({ test_stuff: [[{ b: 'c' }]] }) expect(getTestParametersString(input, 'test_stuff')).to.equal( JSON.stringify({ arguments: [{ b: 'c' }], metadata: {} }) ) - expect(input).to.eql({ 'test_stuff': [] }) + expect(input).to.eql({ test_stuff: [] }) }) + it('does not crash when test name is not found and does not modify input', () => { - const input = { 'test_stuff': [['params'], ['params2']] } + const input = { test_stuff: [['params'], ['params2']] } expect(getTestParametersString(input, 'test_not_present')).to.equal('') - expect(input).to.eql({ 'test_stuff': [['params'], ['params2']] }) + expect(input).to.eql({ test_stuff: [['params'], ['params2']] }) }) + it('does not crash when parameters can not be serialized and removes params from input', () => { const circular = { a: 'b' } circular.b = circular - const input = { 'test_stuff': [[circular], ['params2']] } + const input = { test_stuff: [[circular], ['params2']] } expect(getTestParametersString(input, 'test_stuff')).to.equal('') - expect(input).to.eql({ 'test_stuff': [['params2']] }) + expect(input).to.eql({ test_stuff: [['params2']] }) expect(getTestParametersString(input, 'test_stuff')).to.equal( JSON.stringify({ arguments: ['params2'], metadata: {} }) ) @@ -55,6 +59,7 @@ describe('getTestSuitePath', () => { const testSuitePath = getTestSuitePath(undefined, sourceRoot) expect(testSuitePath).to.equal(sourceRoot) }) + it('returns sourceRoot if the test path has the same value', () => { const sourceRoot = '/users/opt' const testSuiteAbsolutePath = sourceRoot @@ -77,11 +82,34 @@ describe('getCodeOwnersFileEntries', () => { owners: ['@datadog-dd-trace-js'] }) }) + it('returns null if CODEOWNERS can not be found', () => { const rootDir = path.join(__dirname, '__not_found__') + // We have to change the working directory, + // otherwise it will find the CODEOWNERS file in the root of dd-trace-js + const oldCwd = process.cwd() + process.chdir(path.join(__dirname)) const codeOwnersFileEntries = getCodeOwnersFileEntries(rootDir) - expect(codeOwnersFileEntries).to.equal(null) + process.chdir(oldCwd) + }) + + it('tries both input rootDir and process.cwd()', () => { + const rootDir = path.join(__dirname, '__not_found__') + const oldCwd = process.cwd() + + process.chdir(path.join(__dirname, '__test__')) + const codeOwnersFileEntries = getCodeOwnersFileEntries(rootDir) + + expect(codeOwnersFileEntries[0]).to.eql({ + pattern: 'packages/dd-trace/test/plugins/util/test.spec.js', + owners: ['@datadog-ci-app'] + }) + expect(codeOwnersFileEntries[1]).to.eql({ + pattern: 'packages/dd-trace/test/plugins/util/*', + owners: ['@datadog-dd-trace-js'] + }) + process.chdir(oldCwd) }) }) @@ -91,6 +119,7 @@ describe('getCodeOwnersForFilename', () => { expect(codeOwners).to.equal(null) }) + it('returns the code owners for a given file path', () => { const rootDir = path.join(__dirname, '__test__') const codeOwnersFileEntries = getCodeOwnersFileEntries(rootDir) @@ -113,15 +142,18 @@ describe('getCodeOwnersForFilename', () => { describe('coverage utils', () => { let coverage + beforeEach(() => { delete require.cache[require.resolve('./fixtures/coverage.json')] coverage = require('./fixtures/coverage.json') }) + describe('getCoveredFilenamesFromCoverage', () => { it('returns the list of files the code coverage includes', () => { const coverageFiles = getCoveredFilenamesFromCoverage(coverage) expect(coverageFiles).to.eql(['subtract.js', 'add.js']) }) + it('returns an empty list if coverage is empty', () => { const coverageFiles = getCoveredFilenamesFromCoverage({}) expect(coverageFiles).to.eql([]) @@ -144,6 +176,7 @@ describe('coverage utils', () => { mergeCoverage(coverage, newCoverageMap) expect(JSON.stringify(coverage)).to.equal(JSON.stringify(newCoverageMap.toJSON())) }) + it('returns a copy that is not affected by other copies being reset', () => { const newCoverageMap = istanbul.createCoverageMap() @@ -177,7 +210,7 @@ describe('metadata validation', () => { [GIT_COMMIT_SHA]: 'abc123' } const invalidMetadata2 = { - [GIT_REPOSITORY_URL]: 'https://datadog.com/repo', + [GIT_REPOSITORY_URL]: 'htps://datadog.com/repo', [CI_PIPELINE_URL]: 'datadog.com', [GIT_COMMIT_SHA]: 'abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123' } @@ -194,9 +227,12 @@ describe('metadata validation', () => { const invalidMetadata5 = { [GIT_REPOSITORY_URL]: '', [CI_PIPELINE_URL]: '', [GIT_COMMIT_SHA]: '' } const invalidMetadatas = [invalidMetadata1, invalidMetadata2, invalidMetadata3, invalidMetadata4, invalidMetadata5] invalidMetadatas.forEach((invalidMetadata) => { - expect(JSON.stringify(removeInvalidMetadata(invalidMetadata))).to.equal(JSON.stringify({})) + expect( + JSON.stringify(removeInvalidMetadata(invalidMetadata)), `${JSON.stringify(invalidMetadata)} is valid` + ).to.equal(JSON.stringify({})) }) }) + it('should keep valid metadata', () => { const validMetadata1 = { [GIT_REPOSITORY_URL]: 'https://datadoghq.com/myrepo/repo.git', @@ -237,6 +273,7 @@ describe('parseAnnotations', () => { 'test.responsible_team': 'sales' }) }) + it('does not crash with invalid arguments', () => { const tags = parseAnnotations([ {}, @@ -248,3 +285,83 @@ describe('parseAnnotations', () => { expect(tags).to.eql({}) }) }) + +describe('getIsFaultyEarlyFlakeDetection', () => { + it('returns false if the absolute number of new suites is smaller or equal than the threshold', () => { + const faultyThreshold = 30 + + // Session has 50 tests and 25 are marked as new (50%): not faulty. + const projectSuites = Array.from({ length: 50 }).map((_, i) => `test${i}.spec.js`) + const knownSuites = Array.from({ length: 25 }).reduce((acc, _, i) => { + acc[`test${i}.spec.js`] = ['test'] + return acc + }, {}) + + const isFaulty = getIsFaultyEarlyFlakeDetection( + projectSuites, + knownSuites, + faultyThreshold + ) + expect(isFaulty).to.be.false + + // Session has 60 tests and 30 are marked as new (50%): not faulty. + const projectSuites2 = Array.from({ length: 60 }).map((_, i) => `test${i}.spec.js`) + const knownSuites2 = Array.from({ length: 30 }).reduce((acc, _, i) => { + acc[`test${i}.spec.js`] = ['test'] + return acc + }, {}) + const isFaulty2 = getIsFaultyEarlyFlakeDetection( + projectSuites2, + knownSuites2, + faultyThreshold + ) + expect(isFaulty2).to.be.false + }) + + it('returns true if the percentage is above the threshold', () => { + const faultyThreshold = 30 + + // Session has 100 tests and 31 are marked as new (31%): faulty. + const projectSuites = Array.from({ length: 100 }).map((_, i) => `test${i}.spec.js`) + const knownSuites = Array.from({ length: 69 }).reduce((acc, _, i) => { + acc[`test${i}.spec.js`] = ['test'] + return acc + }, {}) + + const isFaulty = getIsFaultyEarlyFlakeDetection( + projectSuites, + knownSuites, + faultyThreshold + ) + expect(isFaulty).to.be.true + }) +}) + +describe('getNumFromKnownTests', () => { + it('calculates the number of tests from the known tests', () => { + const knownTests = { + testModule: { + 'test1.spec.js': ['test1', 'test2'], + 'test2.spec.js': ['test3'] + } + } + + const numTests = getNumFromKnownTests(knownTests) + expect(numTests).to.equal(3) + }) + + it('does not crash with empty dictionaries', () => { + const knownTests = {} + + const numTests = getNumFromKnownTests(knownTests) + expect(numTests).to.equal(0) + }) + + it('does not crash if known tests is undefined or null', () => { + const numTestsUndefined = getNumFromKnownTests(undefined) + expect(numTestsUndefined).to.equal(0) + + const numTestsNull = getNumFromKnownTests(null) + expect(numTestsNull).to.equal(0) + }) +}) diff --git a/packages/dd-trace/test/plugins/util/url.spec.js b/packages/dd-trace/test/plugins/util/url.spec.js index dc1d0c2d15a..5fdaefa7cb0 100644 --- a/packages/dd-trace/test/plugins/util/url.spec.js +++ b/packages/dd-trace/test/plugins/util/url.spec.js @@ -16,6 +16,7 @@ describe('filterSensitiveInfoFromRepository', () => { expect(filterSensitiveInfoFromRepository(url)).to.equal(url) }) }) + it('returns the scrubbed url if credentials are present', () => { const sensitiveUrls = [ 'https://username:password@datadog.com/repository.git', @@ -26,6 +27,7 @@ describe('filterSensitiveInfoFromRepository', () => { expect(filterSensitiveInfoFromRepository(sensitiveUrls[1])).to.equal('ssh://host.xz:port/path/to/repo.git/') expect(filterSensitiveInfoFromRepository(sensitiveUrls[2])).to.equal('https://datadog.com/repository.git') }) + it('does not crash for empty or invalid repository URLs', () => { const invalidUrls = [ null, diff --git a/packages/dd-trace/test/plugins/util/web.spec.js b/packages/dd-trace/test/plugins/util/web.spec.js index f3d61a743be..821d2d8d537 100644 --- a/packages/dd-trace/test/plugins/util/web.spec.js +++ b/packages/dd-trace/test/plugins/util/web.spec.js @@ -41,8 +41,8 @@ describe('plugins/util/web', () => { req = { method: 'GET', headers: { - 'host': 'localhost', - 'date': 'now' + host: 'localhost', + date: 'now' } } end = sinon.stub() @@ -250,8 +250,8 @@ describe('plugins/util/web', () => { }) it('should add configured headers to the span tags', () => { - req.headers['req'] = 'incoming' - req.headers['res'] = 'outgoing' + req.headers.req = 'incoming' + req.headers.res = 'outgoing' config.headers = ['host', 'req:http.req', 'server', 'res:http.res'] config = web.normalizeConfig(config) @@ -353,7 +353,7 @@ describe('plugins/util/web', () => { ].join(',') req.method = 'OPTIONS' - req.headers['origin'] = 'http://test.com' + req.headers.origin = 'http://test.com' req.headers['access-control-request-headers'] = headers res.getHeaders.returns({ @@ -374,7 +374,7 @@ describe('plugins/util/web', () => { ].join(',') req.method = 'OPTIONS' - req.headers['origin'] = 'http://test.com' + req.headers.origin = 'http://test.com' req.headers['access-control-request-headers'] = headers res.getHeaders.returns({ @@ -392,7 +392,7 @@ describe('plugins/util/web', () => { const headers = ['x-datadog-trace-id'] req.method = 'OPTIONS' - req.headers['origin'] = 'http://test.com' + req.headers.origin = 'http://test.com' req.headers['access-control-request-headers'] = headers web.instrument(tracer, config, req, res, 'test.request') @@ -404,7 +404,7 @@ describe('plugins/util/web', () => { it('should handle CORS preflight when no header was requested', () => { req.method = 'OPTIONS' - req.headers['origin'] = 'http://test.com' + req.headers.origin = 'http://test.com' res.getHeaders.returns({ 'access-control-allow-origin': 'http://test.com' @@ -902,7 +902,7 @@ describe('plugins/util/web', () => { beforeEach(() => { config = { - queryStringObfuscation: new RegExp('secret', 'gi') + queryStringObfuscation: /secret/gi } }) diff --git a/packages/dd-trace/test/priority_sampler.spec.js b/packages/dd-trace/test/priority_sampler.spec.js index 00753c79a44..5000d81ff09 100644 --- a/packages/dd-trace/test/priority_sampler.spec.js +++ b/packages/dd-trace/test/priority_sampler.spec.js @@ -9,6 +9,8 @@ const { SAMPLING_MECHANISM_AGENT, SAMPLING_MECHANISM_RULE, SAMPLING_MECHANISM_MANUAL, + SAMPLING_MECHANISM_REMOTE_USER, + SAMPLING_MECHANISM_REMOTE_DYNAMIC, DECISION_MAKER_KEY } = require('../src/constants') @@ -24,6 +26,7 @@ const USER_KEEP = ext.priority.USER_KEEP describe('PrioritySampler', () => { let PrioritySampler let prioritySampler + let SamplingRule let Sampler let sampler let context @@ -32,7 +35,8 @@ describe('PrioritySampler', () => { beforeEach(() => { context = { _tags: { - 'service.name': 'test' + 'service.name': 'test', + 'resource.name': 'resource' }, _sampling: {}, _trace: { @@ -65,10 +69,15 @@ describe('PrioritySampler', () => { }) Sampler.withArgs(0.5).returns(sampler) - PrioritySampler = proxyquire('../src/priority_sampler', { + SamplingRule = proxyquire('../src/sampling_rule', { './sampler': Sampler }) + PrioritySampler = proxyquire('../src/priority_sampler', { + './sampler': Sampler, + './sampling_rule': SamplingRule + }) + prioritySampler = new PrioritySampler('test') }) @@ -182,28 +191,11 @@ describe('PrioritySampler', () => { expect(context._sampling.mechanism).to.equal(SAMPLING_MECHANISM_RULE) }) - it('should support a sample rate from a rule on service as string', () => { - context._tags['service.name'] = 'test' - - prioritySampler = new PrioritySampler('test', { - rules: [ - { sampleRate: 0, service: 'foo' }, - { sampleRate: 1, service: 'test' } - ] - }) - prioritySampler.sample(context) - - expect(context._sampling).to.have.property('priority', USER_KEEP) - expect(context._sampling.mechanism).to.equal(SAMPLING_MECHANISM_RULE) - }) - - it('should support a sample rate from a rule on service as regex', () => { - context._tags['service.name'] = 'test' - + it('should support a rule-based sampling', () => { prioritySampler = new PrioritySampler('test', { rules: [ - { sampleRate: 0, service: /fo/ }, - { sampleRate: 1, service: /tes/ } + { sampleRate: 0, service: 'foo', resource: /res.*/ }, + { sampleRate: 1, service: 'test', resource: /res.*/ } ] }) prioritySampler.sample(context) @@ -212,36 +204,28 @@ describe('PrioritySampler', () => { expect(context._sampling.mechanism).to.equal(SAMPLING_MECHANISM_RULE) }) - it('should support a sample rate from a rule on name as string', () => { - context._name = 'foo' - context._tags['service.name'] = 'test' - + it('should support a customer-defined remote configuration sampling', () => { prioritySampler = new PrioritySampler('test', { rules: [ - { sampleRate: 0, name: 'bar' }, - { sampleRate: 1, name: 'foo' } + { sampleRate: 1, service: 'test', resource: /res.*/, provenance: 'customer' } ] }) prioritySampler.sample(context) expect(context._sampling).to.have.property('priority', USER_KEEP) - expect(context._sampling.mechanism).to.equal(SAMPLING_MECHANISM_RULE) + expect(context._sampling.mechanism).to.equal(SAMPLING_MECHANISM_REMOTE_USER) }) - it('should support a sample rate from a rule on name as regex', () => { - context._name = 'foo' - context._tags['service.name'] = 'test' - + it('should support a dynamic remote configuration sampling', () => { prioritySampler = new PrioritySampler('test', { rules: [ - { sampleRate: 0, name: /ba/ }, - { sampleRate: 1, name: /fo/ } + { sampleRate: 0, service: 'test', resource: /res.*/, provenance: 'dynamic' } ] }) prioritySampler.sample(context) - expect(context._sampling).to.have.property('priority', USER_KEEP) - expect(context._sampling.mechanism).to.equal(SAMPLING_MECHANISM_RULE) + expect(context._sampling).to.have.property('priority', USER_REJECT) + expect(context._sampling.mechanism).to.equal(SAMPLING_MECHANISM_REMOTE_DYNAMIC) }) it('should validate JSON rule into an array', () => { @@ -308,6 +292,29 @@ describe('PrioritySampler', () => { expect(context._sampling.mechanism).to.equal(SAMPLING_MECHANISM_RULE) }) + it('should support a global rate limit', () => { + prioritySampler = new PrioritySampler('test', { + sampleRate: 1, + rateLimit: 1, + rules: [{ + service: 'test', + sampleRate: 1, + rateLimit: 1000 + }] + }) + prioritySampler.sample(context) + + expect(context._sampling).to.have.property('priority', USER_KEEP) + expect(context._sampling.mechanism).to.equal(SAMPLING_MECHANISM_RULE) + + delete context._sampling.priority + + prioritySampler.sample(context) + + expect(context._sampling).to.have.property('priority', USER_REJECT) + expect(context._sampling.mechanism).to.equal(SAMPLING_MECHANISM_RULE) + }) + it('should support disabling the rate limit', () => { prioritySampler = new PrioritySampler('test', { sampleRate: 1, diff --git a/packages/dd-trace/test/profiling/config.spec.js b/packages/dd-trace/test/profiling/config.spec.js index 70df5cc2296..27023e57b08 100644 --- a/packages/dd-trace/test/profiling/config.spec.js +++ b/packages/dd-trace/test/profiling/config.spec.js @@ -9,8 +9,12 @@ const { AgentExporter } = require('../../src/profiling/exporters/agent') const { FileExporter } = require('../../src/profiling/exporters/file') const WallProfiler = require('../../src/profiling/profilers/wall') const SpaceProfiler = require('../../src/profiling/profilers/space') +const EventsProfiler = require('../../src/profiling/profilers/events') const { ConsoleLogger } = require('../../src/profiling/loggers/console') +const samplingContextsAvailable = process.platform !== 'win32' +const oomMonitoringSupported = process.platform !== 'win32' + describe('config', () => { let Config let env @@ -35,7 +39,6 @@ describe('config', () => { const config = new Config() expect(config).to.deep.include({ - enabled: true, service: 'node', flushInterval: 65 * 1000 }) @@ -48,26 +51,25 @@ describe('config', () => { expect(config.logger).to.be.an.instanceof(ConsoleLogger) expect(config.exporters[0]).to.be.an.instanceof(AgentExporter) expect(config.profilers[0]).to.be.an.instanceof(WallProfiler) - expect(config.profilers[0].codeHotspotsEnabled()).false + expect(config.profilers[0].codeHotspotsEnabled()).to.equal(samplingContextsAvailable) expect(config.profilers[1]).to.be.an.instanceof(SpaceProfiler) expect(config.v8ProfilerBugWorkaroundEnabled).true + expect(config.cpuProfilingEnabled).to.equal(samplingContextsAvailable) }) it('should support configuration options', () => { const options = { - enabled: false, service: 'test', version: '1.2.3-test.0', logger: nullLogger, exporters: 'agent,file', profilers: 'space,wall', url: 'http://localhost:1234/', - codeHotspotsEnabled: true + codeHotspotsEnabled: false } const config = new Config(options) - expect(config.enabled).to.equal(options.enabled) expect(config.service).to.equal(options.service) expect(config.host).to.be.a('string') expect(config.version).to.equal(options.version) @@ -82,10 +84,13 @@ describe('config', () => { expect(config.exporters[0]._url.toString()).to.equal(options.url) expect(config.exporters[1]).to.be.an.instanceof(FileExporter) expect(config.profilers).to.be.an('array') - expect(config.profilers.length).to.equal(2) + expect(config.profilers.length).to.equal(2 + samplingContextsAvailable) expect(config.profilers[0]).to.be.an.instanceOf(SpaceProfiler) expect(config.profilers[1]).to.be.an.instanceOf(WallProfiler) - expect(config.profilers[1].codeHotspotsEnabled()).true + expect(config.profilers[1].codeHotspotsEnabled()).false + if (samplingContextsAvailable) { + expect(config.profilers[2]).to.be.an.instanceOf(EventsProfiler) + } }) it('should filter out invalid profilers', () => { @@ -129,9 +134,11 @@ describe('config', () => { it('should support profiler config with DD_PROFILING_PROFILERS', () => { process.env = { DD_PROFILING_PROFILERS: 'wall', - DD_PROFILING_CODEHOTSPOTS_ENABLED: '1', DD_PROFILING_V8_PROFILER_BUG_WORKAROUND: '0' } + if (samplingContextsAvailable) { + process.env.DD_PROFILING_EXPERIMENTAL_CPU_ENABLED = '1' + } const options = { logger: nullLogger } @@ -139,10 +146,14 @@ describe('config', () => { const config = new Config(options) expect(config.profilers).to.be.an('array') - expect(config.profilers.length).to.equal(1) + expect(config.profilers.length).to.equal(1 + samplingContextsAvailable) expect(config.profilers[0]).to.be.an.instanceOf(WallProfiler) - expect(config.profilers[0].codeHotspotsEnabled()).true + expect(config.profilers[0].codeHotspotsEnabled()).to.equal(samplingContextsAvailable) + if (samplingContextsAvailable) { + expect(config.profilers[1]).to.be.an.instanceOf(EventsProfiler) + } expect(config.v8ProfilerBugWorkaroundEnabled).false + expect(config.cpuProfilingEnabled).to.equal(samplingContextsAvailable) }) it('should support profiler config with DD_PROFILING_XXX_ENABLED', () => { @@ -174,14 +185,20 @@ describe('config', () => { const config = new Config(options) expect(config.profilers).to.be.an('array') - expect(config.profilers.length).to.equal(1) + expect(config.profilers.length).to.equal(1 + samplingContextsAvailable) expect(config.profilers[0]).to.be.an.instanceOf(WallProfiler) + if (samplingContextsAvailable) { + expect(config.profilers[1]).to.be.an.instanceOf(EventsProfiler) + } }) it('should prioritize options over env variables', () => { + if (!samplingContextsAvailable) { + return + } + process.env = { DD_PROFILING_PROFILERS: 'space', - DD_PROFILING_CODEHOTSPOTS_ENABLED: '1', DD_PROFILING_ENDPOINT_COLLECTION_ENABLED: '1' } const options = { @@ -194,13 +211,18 @@ describe('config', () => { const config = new Config(options) expect(config.profilers).to.be.an('array') - expect(config.profilers.length).to.equal(1) + expect(config.profilers.length).to.equal(2) expect(config.profilers[0]).to.be.an.instanceOf(WallProfiler) expect(config.profilers[0].codeHotspotsEnabled()).false expect(config.profilers[0].endpointCollectionEnabled()).false + expect(config.profilers[1]).to.be.an.instanceOf(EventsProfiler) }) it('should prioritize non-experimental env variables and warn about experimental ones', () => { + if (!samplingContextsAvailable) { + return + } + process.env = { DD_PROFILING_PROFILERS: 'wall', DD_PROFILING_CODEHOTSPOTS_ENABLED: '0', @@ -231,10 +253,51 @@ describe('config', () => { 'Use DD_PROFILING_CODEHOTSPOTS_ENABLED instead.') expect(config.profilers).to.be.an('array') - expect(config.profilers.length).to.equal(1) + expect(config.profilers.length).to.equal(2) expect(config.profilers[0]).to.be.an.instanceOf(WallProfiler) expect(config.profilers[0].codeHotspotsEnabled()).false expect(config.profilers[0].endpointCollectionEnabled()).false + expect(config.profilers[1]).to.be.an.instanceOf(EventsProfiler) + }) + + function optionOnlyWorksWithGivenCondition (property, name, condition) { + const options = { + [property]: true + } + + if (condition) { + // should silently succeed + // eslint-disable-next-line no-new + new Config(options) + } else { + // should throw + // eslint-disable-next-line no-new + expect(() => { new Config(options) }).to.throw(`${name} not supported on `) + } + } + + function optionOnlyWorksWithSamplingContexts (property, name) { + optionOnlyWorksWithGivenCondition(property, name, samplingContextsAvailable) + } + + it('should only allow code hotspots on supported platforms', () => { + optionOnlyWorksWithSamplingContexts('codeHotspotsEnabled', 'Code hotspots') + }) + + it('should only allow endpoint collection on supported platforms', () => { + optionOnlyWorksWithSamplingContexts('endpointCollection', 'Endpoint collection') + }) + + it('should only allow CPU profiling on supported platforms', () => { + optionOnlyWorksWithSamplingContexts('cpuProfilingEnabled', 'CPU profiling') + }) + + it('should only allow timeline view on supported platforms', () => { + optionOnlyWorksWithSamplingContexts('timelineEnabled', 'Timeline view') + }) + + it('should only allow OOM monitoring on supported platforms', () => { + optionOnlyWorksWithGivenCondition('oomMonitoring', 'OOM monitoring', oomMonitoringSupported) }) it('should support tags', () => { @@ -304,43 +367,49 @@ describe('config', () => { it('should enable OOM heap profiler by default and use process as default strategy', () => { const config = new Config() - expect(config.oomMonitoring).to.deep.equal({ - enabled: true, - heapLimitExtensionSize: 0, - maxHeapExtensionCount: 0, - exportStrategies: ['process'], - exportCommand: [ - process.execPath, - path.normalize(path.join(__dirname, '../../src/profiling', 'exporter_cli.js')), - 'http://localhost:8126/', - `host:${config.host},service:node,snapshot:on_oom`, - 'space' - ] - }) - }) - - it('should support OOM heap profiler configuration', () => { - process.env = { - DD_PROFILING_EXPERIMENTAL_OOM_MONITORING_ENABLED: '1', - DD_PROFILING_EXPERIMENTAL_OOM_HEAP_LIMIT_EXTENSION_SIZE: '1000000', - DD_PROFILING_EXPERIMENTAL_OOM_MAX_HEAP_EXTENSION_COUNT: '2', - DD_PROFILING_EXPERIMENTAL_OOM_EXPORT_STRATEGIES: 'process,async,process' + if (oomMonitoringSupported) { + expect(config.oomMonitoring).to.deep.equal({ + enabled: oomMonitoringSupported, + heapLimitExtensionSize: 0, + maxHeapExtensionCount: 0, + exportStrategies: ['process'], + exportCommand: [ + process.execPath, + path.normalize(path.join(__dirname, '../../src/profiling', 'exporter_cli.js')), + 'http://localhost:8126/', + `host:${config.host},service:node,snapshot:on_oom`, + 'space' + ] + }) + } else { + expect(config.oomMonitoring.enabled).to.be.false } + }) - const config = new Config({}) + if (oomMonitoringSupported) { + it('should support OOM heap profiler configuration', function () { + process.env = { + DD_PROFILING_EXPERIMENTAL_OOM_MONITORING_ENABLED: '1', + DD_PROFILING_EXPERIMENTAL_OOM_HEAP_LIMIT_EXTENSION_SIZE: '1000000', + DD_PROFILING_EXPERIMENTAL_OOM_MAX_HEAP_EXTENSION_COUNT: '2', + DD_PROFILING_EXPERIMENTAL_OOM_EXPORT_STRATEGIES: 'process,async,process' + } - expect(config.oomMonitoring).to.deep.equal({ - enabled: true, - heapLimitExtensionSize: 1000000, - maxHeapExtensionCount: 2, - exportStrategies: ['process', 'async'], - exportCommand: [ - process.execPath, - path.normalize(path.join(__dirname, '../../src/profiling', 'exporter_cli.js')), - 'http://localhost:8126/', - `host:${config.host},service:node,snapshot:on_oom`, - 'space' - ] + const config = new Config({}) + + expect(config.oomMonitoring).to.deep.equal({ + enabled: true, + heapLimitExtensionSize: 1000000, + maxHeapExtensionCount: 2, + exportStrategies: ['process', 'async'], + exportCommand: [ + process.execPath, + path.normalize(path.join(__dirname, '../../src/profiling', 'exporter_cli.js')), + 'http://localhost:8126/', + `host:${config.host},service:node,snapshot:on_oom`, + 'space' + ] + }) }) - }) + } }) diff --git a/packages/dd-trace/test/profiling/exporters/agent.spec.js b/packages/dd-trace/test/profiling/exporters/agent.spec.js index 92e7f097539..b318456eebd 100644 --- a/packages/dd-trace/test/profiling/exporters/agent.spec.js +++ b/packages/dd-trace/test/profiling/exporters/agent.spec.js @@ -20,6 +20,12 @@ const { Profile } = require('pprof-format') const semver = require('semver') const version = require('../../../../../package.json').version +const RUNTIME_ID = 'a1b2c3d4-a1b2-a1b2-a1b2-a1b2c3d4e5f6' +const ENV = 'test-env' +const HOST = 'test-host' +const SERVICE = 'test-service' +const APP_VERSION = '1.2.3' + if (!semver.satisfies(process.version, '>=10.12')) { describe = describe.skip // eslint-disable-line no-global-assign } @@ -31,7 +37,7 @@ function wait (ms) { } async function createProfile (periodType) { - const [ type ] = periodType + const [type] = periodType const profiler = type === 'wall' ? new WallProfiler() : new SpaceProfiler() profiler.start({ // Throw errors in test rather than logging them @@ -39,6 +45,7 @@ async function createProfile (periodType) { error (err) { throw err }, + // eslint-disable-next-line n/handle-callback-err warn (err) { } } @@ -46,8 +53,7 @@ async function createProfile (periodType) { await wait(50) - const profile = profiler.profile() - profiler.stop() + const profile = profiler.profile(false) return profiler.encode(profile) } @@ -66,35 +72,70 @@ describe('exporters/agent', function () { function verifyRequest (req, profiles, start, end) { expect(req.headers).to.have.property('datadog-container-id', docker.id()) - expect(req.body).to.have.property('language', 'javascript') - expect(req.body).to.have.property('runtime', 'nodejs') - expect(req.body).to.have.property('runtime_version', process.version) - expect(req.body).to.have.property('profiler_version', version) - expect(req.body).to.have.property('format', 'pprof') - expect(req.body).to.have.deep.property('tags', [ + expect(req.headers).to.have.property('dd-evp-origin', 'dd-trace-js') + expect(req.headers).to.have.property('dd-evp-origin-version', version) + + expect(req.files[0]).to.have.property('fieldname', 'event') + expect(req.files[0]).to.have.property('originalname', 'event.json') + expect(req.files[0]).to.have.property('mimetype', 'application/json') + expect(req.files[0]).to.have.property('size', req.files[0].buffer.length) + + const event = JSON.parse(req.files[0].buffer.toString()) + expect(event).to.have.property('attachments') + expect(event.attachments).to.have.lengthOf(2) + expect(event.attachments[0]).to.equal('wall.pprof') + expect(event.attachments[1]).to.equal('space.pprof') + expect(event).to.have.property('start', start.toISOString()) + expect(event).to.have.property('end', end.toISOString()) + expect(event).to.have.property('family', 'node') + expect(event).to.have.property('version', '4') + expect(event).to.have.property('tags_profiler', [ 'language:javascript', 'runtime:nodejs', + `runtime_arch:${process.arch}`, + `runtime_os:${process.platform}`, `runtime_version:${process.version}`, + `process_id:${process.pid}`, `profiler_version:${version}`, 'format:pprof', - 'runtime-id:a1b2c3d4-a1b2-a1b2-a1b2-a1b2c3d4e5f6' - ]) - expect(req.body).to.have.deep.property('types', ['wall', 'space']) - expect(req.body).to.have.property('recording-start', start.toISOString()) - expect(req.body).to.have.property('recording-end', end.toISOString()) - - expect(req.files[0]).to.have.property('fieldname', 'data[0]') - expect(req.files[0]).to.have.property('originalname', 'wall.pb.gz') - expect(req.files[0]).to.have.property('mimetype', 'application/octet-stream') - expect(req.files[0]).to.have.property('size', req.files[0].buffer.length) - - expect(req.files[1]).to.have.property('fieldname', 'data[1]') - expect(req.files[1]).to.have.property('originalname', 'space.pb.gz') + `runtime-id:${RUNTIME_ID}` + ].join(',')) + expect(event).to.have.property('info') + expect(event.info).to.have.property('application') + expect(Object.keys(event.info.application)).to.have.length(4) + expect(event.info.application).to.have.property('env', ENV) + expect(event.info.application).to.have.property('service', SERVICE) + expect(event.info.application).to.have.property('start_time') + expect(event.info.application).to.have.property('version', '1.2.3') + expect(event.info).to.have.property('platform') + expect(Object.keys(event.info.platform)).to.have.length(4) + expect(event.info.platform).to.have.property('hostname', HOST) + expect(event.info.platform).to.have.property('kernel_name', os.type()) + expect(event.info.platform).to.have.property('kernel_release', os.release()) + expect(event.info.platform).to.have.property('kernel_version', os.version()) + expect(event.info).to.have.property('profiler') + expect(Object.keys(event.info.profiler)).to.have.length(3) + expect(event.info.profiler).to.have.property('activation', 'unknown') + expect(event.info.profiler).to.have.property('ssi') + expect(event.info.profiler.ssi).to.have.property('mechanism', 'none') + expect(event.info.profiler).to.have.property('version', version) + expect(event.info).to.have.property('runtime') + expect(Object.keys(event.info.runtime)).to.have.length(2) + expect(event.info.runtime).to.have.property('engine', 'nodejs') + expect(event.info.runtime).to.have.property('version', process.version.substring(1)) + + expect(req.files[1]).to.have.property('fieldname', 'wall.pprof') + expect(req.files[1]).to.have.property('originalname', 'wall.pprof') expect(req.files[1]).to.have.property('mimetype', 'application/octet-stream') expect(req.files[1]).to.have.property('size', req.files[1].buffer.length) - const wallProfile = Profile.decode(gunzipSync(req.files[0].buffer)) - const spaceProfile = Profile.decode(gunzipSync(req.files[1].buffer)) + expect(req.files[2]).to.have.property('fieldname', 'space.pprof') + expect(req.files[2]).to.have.property('originalname', 'space.pprof') + expect(req.files[2]).to.have.property('mimetype', 'application/octet-stream') + expect(req.files[2]).to.have.property('size', req.files[2].buffer.length) + + const wallProfile = Profile.decode(gunzipSync(req.files[1].buffer)) + const spaceProfile = Profile.decode(gunzipSync(req.files[2].buffer)) expect(wallProfile).to.be.a.profile expect(spaceProfile).to.be.a.profile @@ -114,7 +155,7 @@ describe('exporters/agent', function () { } const agent = proxyquire('../../../src/profiling/exporters/agent', { '../../exporters/common/docker': docker, - 'http': http + http }) AgentExporter = agent.AgentExporter computeRetries = agent.computeRetries @@ -122,6 +163,18 @@ describe('exporters/agent', function () { app = express() }) + function newAgentExporter ({ url, logger, uploadTimeout = 100 }) { + return new AgentExporter({ + url, + logger, + uploadTimeout, + env: ENV, + service: SERVICE, + version: APP_VERSION, + host: HOST + }) + } + describe('using HTTP', () => { beforeEach(done => { getPort().then(port => { @@ -140,14 +193,14 @@ describe('exporters/agent', function () { }) it('should send profiles as pprof to the intake', async () => { - const exporter = new AgentExporter({ url, logger, uploadTimeout: 100 }) + const exporter = newAgentExporter({ url, logger }) const start = new Date() const end = new Date() const tags = { - 'runtime-id': 'a1b2c3d4-a1b2-a1b2-a1b2-a1b2c3d4e5f6' + 'runtime-id': RUNTIME_ID } - const [ wall, space ] = await Promise.all([ + const [wall, space] = await Promise.all([ createProfile(['wall', 'microseconds']), createProfile(['space', 'bytes']) ]) @@ -182,19 +235,15 @@ describe('exporters/agent', function () { it('should backoff up to the uploadTimeout', async () => { const uploadTimeout = 100 - const exporter = new AgentExporter({ - url, - logger, - uploadTimeout - }) + const exporter = newAgentExporter({ url, logger, uploadTimeout }) const start = new Date() const end = new Date() const tags = { - 'runtime-id': 'a1b2c3d4-a1b2-a1b2-a1b2-a1b2c3d4e5f6' + 'runtime-id': RUNTIME_ID } - const [ wall, space ] = await Promise.all([ + const [wall, space] = await Promise.all([ createProfile(['wall', 'microseconds']), createProfile(['space', 'bytes']) ]) @@ -220,7 +269,7 @@ describe('exporters/agent', function () { try { await exporter.export({ profiles, start, end, tags }) } catch (err) { - expect(err.message).to.match(/^Profiler agent export back-off period expired$/) + expect(err.message).to.match(/^HTTP Error 500$/) failed = true } expect(failed).to.be.true @@ -250,7 +299,7 @@ describe('exporters/agent', function () { it('should log exports and handle http errors gracefully', async function () { const expectedLogs = [ - /^Building agent export report: (\n {2}[a-z-_]+(\[\])?: [a-z0-9-TZ:.]+)+$/m, + /^Building agent export report:\n\{.+\}$/, /^Adding wall profile to agent export:( [0-9a-f]{2})+$/, /^Adding space profile to agent export:( [0-9a-f]{2})+$/, /^Submitting profiler agent report attempt #1 to:/i, @@ -272,19 +321,12 @@ describe('exporters/agent', function () { } let index = 0 - const exporter = new AgentExporter({ - url, - uploadTimeout: 100, - logger: { - debug: onMessage, - error: onMessage - } - }) + const exporter = newAgentExporter({ url, logger: { debug: onMessage, error: onMessage } }) const start = new Date() const end = new Date() const tags = { foo: 'bar' } - const [ wall, space ] = await Promise.all([ + const [wall, space] = await Promise.all([ createProfile(['wall', 'microseconds']), createProfile(['space', 'bytes']) ]) @@ -316,6 +358,58 @@ describe('exporters/agent', function () { }) }) + describe('using ipv6', () => { + beforeEach(done => { + getPort().then(port => { + url = new URL(`http://[0:0:0:0:0:0:0:1]:${port}`) + + listener = app.listen(port, '0:0:0:0:0:0:0:1', done) + listener.on('connection', socket => sockets.push(socket)) + startSpan = sinon.spy(tracer._tracer, 'startSpan') + }) + }) + + afterEach(done => { + listener.close(done) + sockets.forEach(socket => socket.end()) + tracer._tracer.startSpan.restore() + }) + + it('should support ipv6 urls', async () => { + const exporter = newAgentExporter({ url, logger }) + const start = new Date() + const end = new Date() + const tags = { + 'runtime-id': RUNTIME_ID + } + + const [wall, space] = await Promise.all([ + createProfile(['wall', 'microseconds']), + createProfile(['space', 'bytes']) + ]) + + const profiles = { + wall, + space + } + + await new Promise((resolve, reject) => { + app.post('/profiling/v1/input', upload.any(), (req, res) => { + try { + verifyRequest(req, profiles, start, end) + resolve() + } catch (e) { + reject(e) + } + + res.send() + }) + + exporter.export({ profiles, start, end, tags }).catch(reject) + }) + }) + }) + describeOnUnix('using UDS', () => { let listener @@ -332,12 +426,14 @@ describe('exporters/agent', function () { }) it('should support Unix domain sockets', async () => { - const exporter = new AgentExporter({ url: new URL(`unix://${url}`), logger, uploadTimeout: 100 }) + const exporter = newAgentExporter({ url: new URL(`unix://${url}`), logger }) const start = new Date() const end = new Date() - const tags = { foo: 'bar' } + const tags = { + 'runtime-id': RUNTIME_ID + } - const [ wall, space ] = await Promise.all([ + const [wall, space] = await Promise.all([ createProfile(['wall', 'microseconds']), createProfile(['space', 'bytes']) ]) @@ -350,42 +446,7 @@ describe('exporters/agent', function () { await new Promise((resolve, reject) => { app.post('/profiling/v1/input', upload.any(), (req, res) => { try { - expect(req.body).to.have.property('language', 'javascript') - expect(req.body).to.have.property('runtime', 'nodejs') - expect(req.body).to.have.property('runtime_version', process.version) - expect(req.body).to.have.property('profiler_version', version) - expect(req.body).to.have.property('format', 'pprof') - expect(req.body).to.have.deep.property('tags', [ - 'language:javascript', - 'runtime:nodejs', - `runtime_version:${process.version}`, - `profiler_version:${version}`, - 'format:pprof', - 'foo:bar' - ]) - expect(req.body).to.have.deep.property('types', ['wall', 'space']) - expect(req.body).to.have.property('recording-start', start.toISOString()) - expect(req.body).to.have.property('recording-end', end.toISOString()) - - expect(req.files[0]).to.have.property('fieldname', 'data[0]') - expect(req.files[0]).to.have.property('originalname', 'wall.pb.gz') - expect(req.files[0]).to.have.property('mimetype', 'application/octet-stream') - expect(req.files[0]).to.have.property('size', req.files[0].buffer.length) - - expect(req.files[1]).to.have.property('fieldname', 'data[1]') - expect(req.files[1]).to.have.property('originalname', 'space.pb.gz') - expect(req.files[1]).to.have.property('mimetype', 'application/octet-stream') - expect(req.files[1]).to.have.property('size', req.files[1].buffer.length) - - const wallProfile = Profile.decode(gunzipSync(req.files[0].buffer)) - const spaceProfile = Profile.decode(gunzipSync(req.files[1].buffer)) - - expect(wallProfile).to.be.a.profile - expect(spaceProfile).to.be.a.profile - - expect(wallProfile).to.deep.equal(Profile.decode(gunzipSync(profiles.wall))) - expect(spaceProfile).to.deep.equal(Profile.decode(gunzipSync(profiles.space))) - + verifyRequest(req, profiles, start, end) resolve() } catch (e) { reject(e) diff --git a/packages/dd-trace/test/profiling/exporters/file.spec.js b/packages/dd-trace/test/profiling/exporters/file.spec.js index dc6f7b086fe..bca561dce8b 100644 --- a/packages/dd-trace/test/profiling/exporters/file.spec.js +++ b/packages/dd-trace/test/profiling/exporters/file.spec.js @@ -28,7 +28,7 @@ describe('exporters/file', () => { await exporter.export({ profiles, end: new Date('2023-02-10T21:03:05Z') }) sinon.assert.calledOnce(fs.writeFile) - sinon.assert.calledWith(fs.writeFile, 'test_20230210T210305Z.pprof', buffer) + sinon.assert.calledWith(fs.writeFile, 'test_worker_0_20230210T210305Z.pprof', buffer) }) it('should export to a file per profile type with given prefix', async () => { @@ -40,6 +40,6 @@ describe('exporters/file', () => { await exporter.export({ profiles, end: new Date('2023-02-10T21:03:05Z') }) sinon.assert.calledOnce(fs.writeFile) - sinon.assert.calledWith(fs.writeFile, 'myprefix_test_20230210T210305Z.pprof', buffer) + sinon.assert.calledWith(fs.writeFile, 'myprefix_test_worker_0_20230210T210305Z.pprof', buffer) }) }) diff --git a/packages/dd-trace/test/profiling/profiler.spec.js b/packages/dd-trace/test/profiling/profiler.spec.js index f1d4f86e63e..d99eb6135ea 100644 --- a/packages/dd-trace/test/profiling/profiler.spec.js +++ b/packages/dd-trace/test/profiling/profiler.spec.js @@ -7,6 +7,9 @@ const sinon = require('sinon') const SpaceProfiler = require('../../src/profiling/profilers/space') const WallProfiler = require('../../src/profiling/profilers/wall') +const EventsProfiler = require('../../src/profiling/profilers/events') + +const samplingContextsAvailable = process.platform !== 'win32' describe('profiler', function () { let Profiler @@ -137,12 +140,12 @@ describe('profiler', function () { it('should allow configuring profilers by string or string arrays', async () => { const checks = [ ['space', SpaceProfiler], - ['wall', WallProfiler], - ['space,wall', SpaceProfiler, WallProfiler], - ['wall,space', WallProfiler, SpaceProfiler], - [['space', 'wall'], SpaceProfiler, WallProfiler], - [['wall', 'space'], WallProfiler, SpaceProfiler] - ] + ['wall', WallProfiler, EventsProfiler], + ['space,wall', SpaceProfiler, WallProfiler, EventsProfiler], + ['wall,space', WallProfiler, SpaceProfiler, EventsProfiler], + [['space', 'wall'], SpaceProfiler, WallProfiler, EventsProfiler], + [['wall', 'space'], WallProfiler, SpaceProfiler, EventsProfiler] + ].map(profilers => profilers.filter(profiler => samplingContextsAvailable || profiler !== EventsProfiler)) for (const [profilers, ...expected] of checks) { await profiler._start({ @@ -178,6 +181,21 @@ describe('profiler', function () { }) it('should stop when capturing failed', async () => { + wallProfiler.profile.throws(new Error('boom')) + + await profiler._start({ profilers, exporters, logger }) + + clock.tick(interval) + + sinon.assert.calledOnce(wallProfiler.stop) + sinon.assert.calledOnce(spaceProfiler.stop) + sinon.assert.calledOnce(consoleLogger.error) + sinon.assert.notCalled(wallProfiler.encode) + sinon.assert.notCalled(spaceProfiler.encode) + sinon.assert.notCalled(exporter.export) + }) + + it('should not stop when encoding failed', async () => { const rejected = Promise.reject(new Error('boom')) wallProfiler.encode.returns(rejected) @@ -187,9 +205,25 @@ describe('profiler', function () { await rejected.catch(() => {}) - sinon.assert.calledOnce(wallProfiler.stop) - sinon.assert.calledOnce(spaceProfiler.stop) + sinon.assert.notCalled(wallProfiler.stop) + sinon.assert.notCalled(spaceProfiler.stop) sinon.assert.calledOnce(consoleLogger.error) + sinon.assert.calledOnce(exporter.export) + }) + + it('should not stop when exporting failed', async () => { + const rejected = Promise.reject(new Error('boom')) + exporter.export.returns(rejected) + + await profiler._start({ profilers, exporters, logger }) + + clock.tick(interval) + + await rejected.catch(() => {}) + + sinon.assert.notCalled(wallProfiler.stop) + sinon.assert.notCalled(spaceProfiler.stop) + sinon.assert.calledOnce(exporter.export) }) it('should flush when the interval is reached', async () => { @@ -229,13 +263,6 @@ describe('profiler', function () { expect(tags).to.have.property('foo', 'foo') }) - it('should not start when disabled', async () => { - await profiler._start({ profilers, exporters, enabled: false }) - - sinon.assert.notCalled(wallProfiler.start) - sinon.assert.notCalled(spaceProfiler.start) - }) - it('should log exporter errors', async () => { exporter.export.rejects(new Error('boom')) @@ -274,17 +301,6 @@ describe('profiler', function () { sinon.assert.calledWithMatch(submit, 'Submitted profiles') }) - it('should skip submit with no profiles', async () => { - const start = new Date() - const end = new Date() - try { - await profiler._submit({}, start, end) - throw new Error('should have got exception from _submit') - } catch (err) { - expect(err.message).to.equal('No profiles to submit') - } - }) - it('should have a new start time for each capture', async () => { await profiler._start({ profilers, exporters }) diff --git a/packages/dd-trace/test/profiling/profilers/space.spec.js b/packages/dd-trace/test/profiling/profilers/space.spec.js index feab8b3ab1b..898fb212e2f 100644 --- a/packages/dd-trace/test/profiling/profilers/space.spec.js +++ b/packages/dd-trace/test/profiling/profilers/space.spec.js @@ -47,8 +47,11 @@ describe('profilers/native/space', () => { it('should stop the internal space profiler', () => { const profiler = new NativeSpaceProfiler() + expect(profiler.isStarted()).to.be.false profiler.start() + expect(profiler.isStarted()).to.be.true profiler.stop() + expect(profiler.isStarted()).to.be.false sinon.assert.calledOnce(pprof.heap.stop) }) @@ -60,7 +63,21 @@ describe('profilers/native/space', () => { pprof.heap.profile.returns('profile') - const profile = profiler.profile() + const profile = profiler.profile(true) + expect(profiler.isStarted()).to.be.true + + expect(profile).to.equal('profile') + }) + + it('should collect profiles from the pprof space profiler and stop profiler if not restarted', () => { + const profiler = new NativeSpaceProfiler() + + profiler.start() + + pprof.heap.profile.returns('profile') + + const profile = profiler.profile(false) + expect(profiler.isStarted()).to.be.false expect(profile).to.equal('profile') }) @@ -69,7 +86,7 @@ describe('profilers/native/space', () => { const profiler = new NativeSpaceProfiler() profiler.start() - const profile = profiler.profile() + const profile = profiler.profile(true) profiler.encode(profile) sinon.assert.calledOnce(pprof.encode) @@ -81,7 +98,7 @@ describe('profilers/native/space', () => { const mapper = {} profiler.start({ mapper }) - profiler.profile() + profiler.profile(true) sinon.assert.calledWith(pprof.heap.profile, undefined, mapper) }) diff --git a/packages/dd-trace/test/profiling/profilers/wall.spec.js b/packages/dd-trace/test/profiling/profilers/wall.spec.js index be39e892828..b64900c474c 100644 --- a/packages/dd-trace/test/profiling/profilers/wall.spec.js +++ b/packages/dd-trace/test/profiling/profilers/wall.spec.js @@ -55,7 +55,8 @@ describe('profilers/native/wall', () => { sourceMapper: undefined, withContexts: false, lineNumbers: false, - workaroundV8Bug: false + workaroundV8Bug: false, + collectCpuTime: false }) }) @@ -73,7 +74,8 @@ describe('profilers/native/wall', () => { sourceMapper: undefined, withContexts: false, lineNumbers: false, - workaroundV8Bug: false + workaroundV8Bug: false, + collectCpuTime: false }) }) @@ -107,15 +109,36 @@ describe('profilers/native/wall', () => { it('should collect profiles from the internal time profiler', () => { const profiler = new NativeWallProfiler() + expect(profiler.isStarted()).to.be.false profiler.start() + expect(profiler.isStarted()).to.be.true - const profile = profiler.profile() + const profile = profiler.profile(true) expect(profile).to.equal('profile') sinon.assert.calledOnce(pprof.time.stop) sinon.assert.calledOnce(pprof.time.start) + expect(profiler.isStarted()).to.be.true profiler.stop() + expect(profiler.isStarted()).to.be.false + sinon.assert.calledTwice(pprof.time.stop) + }) + + it('should collect profiles from the internal time profiler and stop profiler if not restarted', () => { + const profiler = new NativeWallProfiler() + + profiler.start() + + const profile = profiler.profile(false) + + expect(profile).to.equal('profile') + + sinon.assert.calledOnce(pprof.time.stop) + sinon.assert.calledOnce(pprof.time.start) + expect(profiler.isStarted()).to.be.false + profiler.stop() + sinon.assert.calledOnce(pprof.time.stop) }) it('should encode profiles from the pprof time profiler', () => { @@ -123,7 +146,7 @@ describe('profilers/native/wall', () => { profiler.start() - const profile = profiler.profile() + const profile = profiler.profile(true) profiler.encode(profile) @@ -147,7 +170,8 @@ describe('profilers/native/wall', () => { sourceMapper: mapper, withContexts: false, lineNumbers: false, - workaroundV8Bug: false + workaroundV8Bug: false, + collectCpuTime: false }) }) }) diff --git a/packages/dd-trace/test/profiling/ssi-heuristics.spec.js b/packages/dd-trace/test/profiling/ssi-heuristics.spec.js new file mode 100644 index 00000000000..81e91d3fe51 --- /dev/null +++ b/packages/dd-trace/test/profiling/ssi-heuristics.spec.js @@ -0,0 +1,261 @@ +'use strict' + +require('../setup/tap') + +const expect = require('chai').expect +const sinon = require('sinon') + +const telemetryManagerNamespace = sinon.stub() +telemetryManagerNamespace.returns() + +const dc = require('dc-polyfill') +const Config = require('../../src/config') + +describe('Profiling for SSI', () => { + describe('When injection is not present', () => { + describe('Neither telemetry nor heuristics should work when', () => { + it('profiler enablement is unspecified', () => { + delete process.env.DD_INJECTION_ENABLED + testInactive('') + }) + + it('the profiler is explicitly enabled', () => { + delete process.env.DD_INJECTION_ENABLED + testInactive('true') + }) + + it('the profiler is explicitly disabled', () => { + delete process.env.DD_INJECTION_ENABLED + testInactive('false') + }) + }) + + describe('Only telemetry should work when', () => { + it('the profiler is explicitly auto-enabled', () => { + delete process.env.DD_INJECTION_ENABLED + process.env.DD_PROFILING_ENABLED = 'auto' + return testEnabledHeuristics(undefined, true) + }) + }) + }) + + describe('When injection is present', () => { + describe('Neither telemetry nor heuristics should work when', () => { + it('the profiler is explicitly disabled', () => { + process.env.DD_INJECTION_ENABLED = 'tracer' + return testInactive('false') + }) + }) + + describe('Only telemetry should work when', () => { + it('profiler enablement is unspecified', () => { + process.env.DD_INJECTION_ENABLED = 'tracer' + delete process.env.DD_PROFILING_ENABLED + return testEnabledHeuristics('ssi_not_enabled', false) + }) + + it('the profiler is explicitly enabled', () => { + process.env.DD_INJECTION_ENABLED = 'tracer' + process.env.DD_PROFILING_ENABLED = 'true' + return testEnabledHeuristics('manually_enabled', false) + }) + }) + + describe('Both telemetry and heuristics should work when', () => { + it('the profiler is explicitly auto-enabled', () => { + process.env.DD_INJECTION_ENABLED = 'tracer' + process.env.DD_PROFILING_ENABLED = 'auto' + return testEnabledHeuristics('auto_enabled', true) + }) + + it('\'profiler\' value is in DD_INJECTION_ENABLED', () => { + process.env.DD_INJECTION_ENABLED = 'tracer,service_name,profiler' + return testEnabledHeuristics('ssi_enabled', true) + }) + }) + }) +}) + +function setupHarness () { + const profileCountCount = { + inc: sinon.stub() + } + const runtimeIdCount = { + inc: sinon.stub() + } + const ssiMetricsNamespace = { + count: sinon.stub().callsFake((name, tags) => { + if (name === 'ssi_heuristic.number_of_profiles') { + return profileCountCount + } else if (name === 'ssi_heuristic.number_of_runtime_id') { + return runtimeIdCount + } + }) + } + + const namespaceFn = sinon.stub().returns(ssiMetricsNamespace) + const { SSIHeuristics } = proxyquire('../src/profiling/ssi-heuristics', { + '../telemetry/metrics': { + manager: { + namespace: namespaceFn + } + } + }) + expect(namespaceFn.calledOnceWithExactly('profilers')).to.equal(true) + const stubs = { + profileCountCountInc: profileCountCount.inc, + runtimeIdCountInc: runtimeIdCount.inc, + count: ssiMetricsNamespace.count + } + return { stubs, SSIHeuristics } +} + +function testInactive (profilingEnabledValue) { + process.env.DD_PROFILING_ENABLED = profilingEnabledValue + + const { stubs, SSIHeuristics } = setupHarness() + const heuristics = new SSIHeuristics(new Config()) + heuristics.start() + expect(heuristics.emitsTelemetry).to.equal(false) + expect(heuristics.heuristicsActive).to.equal(false) + + dc.channel('dd-trace:span:start').publish() + dc.channel('datadog:profiling:profile-submitted').publish() + dc.channel('datadog:profiling:mock-profile-submitted').publish() + dc.channel('datadog:telemetry:app-closing').publish() + expect(heuristics.enablementChoice).to.equal(undefined) + // When it is disabled, the telemetry should not subscribe to any channel + // so the preceding publishes should not have any effect. + expect(heuristics._profileCount).to.equal(undefined) + expect(heuristics.hasSentProfiles).to.equal(false) + expect(heuristics.noSpan).to.equal(true) + expect(stubs.count.notCalled).to.equal(true) +} + +function executeTelemetryEnabledScenario ( + scenario, + profileCount, + sentProfiles, + enablementChoice, + heuristicsActive, + heuristicDecision, + longLived = false +) { + const { stubs, SSIHeuristics } = setupHarness() + const config = new Config() + if (longLived) { + config.profiling.longLivedThreshold = 2 + } + const heuristics = new SSIHeuristics(config) + heuristics.start() + expect(heuristics.heuristicsActive).to.equal(heuristicsActive) + + function runScenarioAndCheck () { + scenario(heuristics) + if (enablementChoice) { + createAndCheckMetrics(stubs, profileCount, sentProfiles, enablementChoice, heuristicDecision) + } else { + // enablementChose being undefined means telemetry should not be active so + // no metrics APIs must've been called + expect(stubs.count.args.length).to.equal(0) + expect(stubs.profileCountCountInc.args.length).to.equal(0) + expect(stubs.count.args.length).to.equal(0) + expect(stubs.runtimeIdCountInc.args.length).to.equal(0) + } + dc.channel('datadog:telemetry:app-closing').publish() + } + + if (longLived) { + return new Promise(resolve => setTimeout(resolve, 3)).then(runScenarioAndCheck) + } else { + runScenarioAndCheck() + } +} + +function createAndCheckMetrics ( + stubs, + profileCount, + sentProfiles, + enablementChoice, + heuristicDecision +) { + // Trigger metrics creation + dc.channel('datadog:telemetry:app-closing').publish() + + const tags = [ + 'installation:ssi', + `enablement_choice:${enablementChoice}`, + `has_sent_profiles:${sentProfiles}`, + `heuristic_hypothetical_decision:${heuristicDecision}` + ] + expect(stubs.count.calledWith('ssi_heuristic.number_of_profiles', tags)).to.equal(true) + expect(stubs.profileCountCountInc.args.length).to.equal(profileCount + 1) // once at the end with 0 + expect(stubs.count.calledWith('ssi_heuristic.number_of_runtime_id', tags)).to.equal(true) + expect(stubs.runtimeIdCountInc.args.length).to.equal(1) +} + +function testEnabledHeuristics (enablementChoice, heuristicsEnabled) { + testNoOp(enablementChoice, heuristicsEnabled) + testProfilesSent(enablementChoice, heuristicsEnabled) + testMockProfilesSent(enablementChoice, heuristicsEnabled) + testSpan(enablementChoice, heuristicsEnabled) + return testLongLived(enablementChoice, heuristicsEnabled).then( + () => testTriggered(enablementChoice, heuristicsEnabled) + ) +} + +function testNoOp (enablementChoice, heuristicsActive) { + executeTelemetryEnabledScenario(_ => {}, 0, false, enablementChoice, heuristicsActive, 'no_span_short_lived') +} + +function testProfilesSent (enablementChoice, heuristicsActive) { + executeTelemetryEnabledScenario(_ => { + dc.channel('datadog:profiling:profile-submitted').publish() + dc.channel('datadog:profiling:profile-submitted').publish() + }, 2, true, enablementChoice, heuristicsActive, 'no_span_short_lived') +} + +function testMockProfilesSent (enablementChoice, heuristicsActive) { + executeTelemetryEnabledScenario(_ => { + dc.channel('datadog:profiling:mock-profile-submitted').publish() + dc.channel('datadog:profiling:mock-profile-submitted').publish() + }, 2, false, enablementChoice, heuristicsActive, 'no_span_short_lived') +} + +function testSpan (enablementChoice, heuristicsActive) { + executeTelemetryEnabledScenario(heuristics => { + const ch = dc.channel('dd-trace:span:start') + expect(ch.hasSubscribers).to.equal(true) + ch.publish() + expect(heuristics.noSpan).to.equal(false) + dc.channel('datadog:profiling:profile-submitted').publish() + }, 1, true, enablementChoice, heuristicsActive, 'short_lived') +} + +function testLongLived (enablementChoice, heuristicsActive) { + let callbackInvoked = false + return executeTelemetryEnabledScenario(heuristics => { + heuristics.onTriggered(() => { + callbackInvoked = true + heuristics.onTriggered() + }) + dc.channel('datadog:profiling:profile-submitted').publish() + }, 1, true, enablementChoice, heuristicsActive, 'no_span', true).then(() => { + expect(callbackInvoked).to.equal(false) + }) +} + +function testTriggered (enablementChoice, heuristicsActive) { + let callbackInvoked = false + return executeTelemetryEnabledScenario(heuristics => { + heuristics.onTriggered(() => { + callbackInvoked = true + heuristics.onTriggered() + }) + dc.channel('dd-trace:span:start').publish() + expect(heuristics.noSpan).to.equal(false) + dc.channel('datadog:profiling:profile-submitted').publish() + }, 1, true, enablementChoice, heuristicsActive, 'triggered', true).then(() => { + expect(callbackInvoked).to.equal(true) + }) +} diff --git a/packages/dd-trace/test/profiling/tagger.spec.js b/packages/dd-trace/test/profiling/tagger.spec.js index 2aaec15abf2..db01befd9d6 100644 --- a/packages/dd-trace/test/profiling/tagger.spec.js +++ b/packages/dd-trace/test/profiling/tagger.spec.js @@ -27,7 +27,7 @@ describe('tagger', () => { const tags = { foo: 'bar', baz: 'qux', - undefined: undefined, + undefined, null: null } diff --git a/packages/dd-trace/test/proxy.spec.js b/packages/dd-trace/test/proxy.spec.js index 21a5443826e..a21e2f4226a 100644 --- a/packages/dd-trace/test/proxy.spec.js +++ b/packages/dd-trace/test/proxy.spec.js @@ -1,7 +1,5 @@ 'use strict' -const EventEmitter = require('events') - require('./setup/tap') describe('TracerProxy', () => { @@ -26,9 +24,13 @@ describe('TracerProxy', () => { let iast let PluginManager let pluginManager + let flare let remoteConfig + let handlers let rc - let noopDogStatsD + let dogStatsD + let noopDogStatsDClient + let NoopDogStatsDClient beforeEach(() => { process.env.DD_TRACE_MOCHA_ENABLED = false @@ -71,6 +73,14 @@ describe('TracerProxy', () => { trackCustomEvent: sinon.stub() } + noopDogStatsDClient = { + increment: sinon.spy(), + gauge: sinon.spy(), + distribution: sinon.spy(), + histogram: sinon.spy(), + flush: sinon.spy() + } + { const dogstatsdIncrements = [] let dogstatsdConfig @@ -80,15 +90,17 @@ describe('TracerProxy', () => { constructor (cfg) { dogstatsdConfig = cfg } + increment () { dogstatsdIncrements.push(arguments) } + flush () { dogstatsdFlushes++ } } - noopDogStatsD = { + dogStatsD = { CustomMetrics: FauxDogStatsDClient, _increments: () => dogstatsdIncrements, _config: () => dogstatsdConfig, @@ -105,10 +117,12 @@ describe('TracerProxy', () => { AppsecSdk = sinon.stub().returns(appsecSdk) NoopAppsecSdk = sinon.stub().returns(noopAppsecSdk) PluginManager = sinon.stub().returns(pluginManager) + NoopDogStatsDClient = sinon.stub().returns(noopDogStatsDClient) config = { tracing: true, experimental: {}, + injectionEnabled: [], logger: 'logger', debug: true, profiling: {}, @@ -130,7 +144,8 @@ describe('TracerProxy', () => { } appsec = { - enable: sinon.spy() + enable: sinon.spy(), + disable: sinon.spy() } telemetry = { @@ -138,21 +153,34 @@ describe('TracerProxy', () => { } iast = { - enable: sinon.spy() + enable: sinon.spy(), + disable: sinon.spy() + } + + flare = { + enable: sinon.spy(), + disable: sinon.spy(), + prepare: sinon.spy(), + send: sinon.spy(), + cleanup: sinon.spy() } remoteConfig = { enable: sinon.stub() } - rc = new EventEmitter() + handlers = new Map() + rc = { + setProductHandler (product, handler) { handlers.set(product, handler) }, + removeProductHandler (product) { handlers.delete(product) } + } remoteConfig.enable.returns(rc) NoopProxy = proxyquire('../src/noop/proxy', { './tracer': NoopTracer, '../appsec/sdk/noop': NoopAppsecSdk, - './dogstatsd': noopDogStatsD + './dogstatsd': NoopDogStatsDClient }) Proxy = proxyquire('../src/proxy', { @@ -168,7 +196,9 @@ describe('TracerProxy', () => { './telemetry': telemetry, './appsec/remote_config': remoteConfig, './appsec/sdk': AppsecSdk, - './dogstatsd': noopDogStatsD + './dogstatsd': dogStatsD, + './noop/dogstatsd': NoopDogStatsDClient, + './flare': flare }) proxy = new Proxy() @@ -226,13 +256,129 @@ describe('TracerProxy', () => { proxy.init() - rc.emit('APM_TRACING', 'apply', { lib_config: conf }) + handlers.get('APM_TRACING')('apply', { lib_config: conf }) expect(config.configure).to.have.been.calledWith(conf) expect(tracer.configure).to.have.been.calledWith(config) expect(pluginManager.configure).to.have.been.calledWith(config) }) + it('should support enabling debug logs for tracer flares', () => { + const logLevel = 'debug' + + proxy.init() + + handlers.get('AGENT_CONFIG')('apply', { + config: { + log_level: logLevel + }, + name: 'flare-log-level.debug' + }) + + expect(flare.enable).to.have.been.calledWith(config) + expect(flare.prepare).to.have.been.calledWith(logLevel) + }) + + it('should support sending tracer flares', () => { + const task = { + case_id: '111', + hostname: 'myhostname', + user_handle: 'user.name@datadoghq.com' + } + + proxy.init() + + handlers.get('AGENT_TASK')('apply', { + args: task, + task_type: 'tracer_flare', + uuid: 'd53fc8a4-8820-47a2-aa7d-d565582feb81' + }) + + expect(flare.enable).to.have.been.calledWith(config) + expect(flare.send).to.have.been.calledWith(task) + }) + + it('should cleanup flares when the config is removed', () => { + const conf = { + config: { + log_level: 'debug' + }, + name: 'flare-log-level.debug' + } + + proxy.init() + + handlers.get('AGENT_CONFIG')('apply', conf) + handlers.get('AGENT_CONFIG')('unapply', conf) + + expect(flare.disable).to.have.been.called + }) + + it('should support applying remote config', () => { + const RemoteConfigProxy = proxyquire('../src/proxy', { + './tracer': DatadogTracer, + './appsec': appsec, + './appsec/iast': iast, + './appsec/remote_config': remoteConfig, + './appsec/sdk': AppsecSdk + }) + + const remoteConfigProxy = new RemoteConfigProxy() + remoteConfigProxy.init() + expect(DatadogTracer).to.have.been.calledOnce + expect(AppsecSdk).to.have.been.calledOnce + expect(appsec.enable).to.not.have.been.called + expect(iast.enable).to.not.have.been.called + + let conf = { tracing_enabled: false } + handlers.get('APM_TRACING')('apply', { lib_config: conf }) + expect(appsec.disable).to.not.have.been.called + expect(iast.disable).to.not.have.been.called + + conf = { tracing_enabled: true } + handlers.get('APM_TRACING')('apply', { lib_config: conf }) + expect(DatadogTracer).to.have.been.calledOnce + expect(AppsecSdk).to.have.been.calledOnce + expect(appsec.enable).to.not.have.been.called + expect(iast.enable).to.not.have.been.called + }) + + it('should support applying remote config (only call disable if enabled before)', () => { + const RemoteConfigProxy = proxyquire('../src/proxy', { + './tracer': DatadogTracer, + './config': Config, + './appsec': appsec, + './appsec/iast': iast, + './appsec/remote_config': remoteConfig, + './appsec/sdk': AppsecSdk + }) + + config.telemetry = {} + config.appsec.enabled = true + config.iast.enabled = true + config.configure = conf => { + config.tracing = conf.tracing_enabled + } + + const remoteConfigProxy = new RemoteConfigProxy() + remoteConfigProxy.init() + + expect(appsec.enable).to.have.been.calledOnceWithExactly(config) + expect(iast.enable).to.have.been.calledOnceWithExactly(config, tracer) + + let conf = { tracing_enabled: false } + handlers.get('APM_TRACING')('apply', { lib_config: conf }) + expect(appsec.disable).to.have.been.called + expect(iast.disable).to.have.been.called + + conf = { tracing_enabled: true } + handlers.get('APM_TRACING')('apply', { lib_config: conf }) + expect(appsec.enable).to.have.been.calledTwice + expect(appsec.enable.secondCall).to.have.been.calledWithExactly(config) + expect(iast.enable).to.have.been.calledTwice + expect(iast.enable.secondCall).to.have.been.calledWithExactly(config, tracer) + }) + it('should start capturing runtimeMetrics when configured', () => { config.runtimeMetrics = true @@ -268,11 +414,11 @@ describe('TracerProxy', () => { proxy.init() - expect(noopDogStatsD._flushes()).to.equal(0) + expect(dogStatsD._flushes()).to.equal(0) clock.tick(10000) - expect(noopDogStatsD._flushes()).to.equal(1) + expect(dogStatsD._flushes()).to.equal(1) }) it('should expose real metrics methods after init when configured', () => { @@ -288,10 +434,10 @@ describe('TracerProxy', () => { proxy.init() - expect(noopDogStatsD._config().dogstatsd.hostname).to.equal('localhost') + expect(dogStatsD._config().dogstatsd.hostname).to.equal('localhost') proxy.dogstatsd.increment('foo', 10, { alpha: 'bravo' }) - const incs = noopDogStatsD._increments() + const incs = dogStatsD._increments() expect(incs.length).to.equal(1) expect(incs[0][0]).to.equal('foo') expect(incs[0][1]).to.equal(10) @@ -347,7 +493,7 @@ describe('TracerProxy', () => { }) it('should load profiler when configured', () => { - config.profiling = { enabled: true } + config.profiling = { enabled: 'true' } proxy.init() @@ -355,7 +501,7 @@ describe('TracerProxy', () => { }) it('should throw an error since profiler fails to be imported', () => { - config.profiling = { enabled: true } + config.profiling = { enabled: 'true' } const ProfilerImportFailureProxy = proxyquire('../src/proxy', { './tracer': DatadogTracer, @@ -365,6 +511,7 @@ describe('TracerProxy', () => { './log': log, './profiler': null, // this will cause the import failure error './appsec': appsec, + './telemetry': telemetry, './appsec/remote_config': remoteConfig }) @@ -380,6 +527,30 @@ describe('TracerProxy', () => { expect(telemetry.start).to.have.been.called }) + + it('should configure appsec standalone', () => { + const standalone = { + configure: sinon.stub() + } + + const options = {} + const DatadogProxy = proxyquire('../src/proxy', { + './tracer': DatadogTracer, + './config': Config, + './appsec': appsec, + './appsec/iast': iast, + './appsec/remote_config': remoteConfig, + './appsec/sdk': AppsecSdk, + './appsec/standalone': standalone, + './telemetry': telemetry + }) + + const proxy = new DatadogProxy() + proxy.init(options) + + const config = AppsecSdk.firstCall.args[1] + expect(standalone.configure).to.have.been.calledOnceWithExactly(config) + }) }) describe('trace', () => { @@ -496,6 +667,19 @@ describe('TracerProxy', () => { }) }) }) + + describe('dogstatsd', () => { + it('should not throw when calling noop methods', () => { + proxy.dogstatsd.increment('inc') + expect(noopDogStatsDClient.increment).to.have.been.calledWith('inc') + proxy.dogstatsd.distribution('dist') + expect(noopDogStatsDClient.distribution).to.have.been.calledWith('dist') + proxy.dogstatsd.histogram('hist') + expect(noopDogStatsDClient.histogram).to.have.been.calledWith('hist') + proxy.dogstatsd.flush() + expect(noopDogStatsDClient.flush).to.have.been.called + }) + }) }) describe('initialized', () => { diff --git a/packages/dd-trace/test/ritm.spec.js b/packages/dd-trace/test/ritm.spec.js index e05eeb32a50..df2a4e8b1a4 100644 --- a/packages/dd-trace/test/ritm.spec.js +++ b/packages/dd-trace/test/ritm.spec.js @@ -4,37 +4,79 @@ require('./setup/tap') const dc = require('dc-polyfill') const { assert } = require('chai') +const Module = require('module') const Hook = require('../src/ritm') -const moduleLoadStartChannel = dc.channel('dd-trace:moduleLoadStart') -const moduleLoadEndChannel = dc.channel('dd-trace:moduleLoadEnd') - describe('Ritm', () => { - it('should shim util', () => { - const startListener = sinon.fake() - const endListener = sinon.fake() + let moduleLoadStartChannel, moduleLoadEndChannel, startListener, endListener + let utilHook, aHook, bHook, httpHook + + before(() => { + moduleLoadStartChannel = dc.channel('dd-trace:moduleLoadStart') + moduleLoadEndChannel = dc.channel('dd-trace:moduleLoadEnd') + }) + + beforeEach(() => { + startListener = sinon.fake() + endListener = sinon.fake() moduleLoadStartChannel.subscribe(startListener) moduleLoadEndChannel.subscribe(endListener) - Hook('util') - require('util') + Module.prototype.require = new Proxy(Module.prototype.require, { + apply (target, thisArg, argArray) { + if (argArray[0] === '@azure/functions-core') { + return { + version: '1.0.0', + registerHook: () => { } + } + } else { + return Reflect.apply(target, thisArg, argArray) + } + } + }) + + utilHook = Hook('util') + aHook = Hook('module-a') + bHook = Hook('module-b') + httpHook = new Hook(['http'], function onRequire (exports, name, basedir) { + exports.foo = 1 + return exports + }) + }) + + afterEach(() => { + utilHook.unhook() + aHook.unhook() + bHook.unhook() + httpHook.unhook() + }) + + it('should shim util', () => { + require('util') assert.equal(startListener.callCount, 1) assert.equal(endListener.callCount, 1) }) it('should handle module load cycles', () => { - const startListener = sinon.fake() - const endListener = sinon.fake() - - moduleLoadStartChannel.subscribe(startListener) - moduleLoadEndChannel.subscribe(endListener) - Hook('module-a') - Hook('module-b') const { a } = require('./ritm-tests/module-a') - assert.equal(startListener.callCount, 2) assert.equal(endListener.callCount, 2) assert.equal(a(), 'Called by AJ') }) + + it('should fall back to monkey patched module', () => { + assert.equal(require('http').foo, 1, 'normal hooking still works') + + const fnCore = require('@azure/functions-core') + assert.ok(fnCore, 'requiring monkey patched in module works') + assert.equal(fnCore.version, '1.0.0') + assert.equal(typeof fnCore.registerHook, 'function') + + assert.throws( + () => require('package-does-not-exist'), + 'Cannot find module \'package-does-not-exist\'', + 'a failing `require(...)` can still throw as expected' + ) + }) }) diff --git a/packages/dd-trace/test/sampling_rule.spec.js b/packages/dd-trace/test/sampling_rule.spec.js new file mode 100644 index 00000000000..49ce1153d2e --- /dev/null +++ b/packages/dd-trace/test/sampling_rule.spec.js @@ -0,0 +1,409 @@ +'use strict' + +require('./setup/tap') + +const { expect } = require('chai') +const id = require('../src/id') + +function createDummySpans () { + const operations = [ + 'operation', + 'sub_operation', + 'second_operation', + 'sub_second_operation_1', + 'sub_second_operation_2', + 'sub_sub_second_operation_2', + 'custom_service_span_1', + 'custom_service_span_2', + 'renamed_operation', + 'tagged_operation', + 'resource_named_operation' + ] + + const ids = [ + id('0234567812345671'), + id('0234567812345672'), + id('0234567812345673'), + id('0234567812345674'), + id('0234567812345675'), + id('0234567812345676'), + id('0234567812345677'), + id('0234567812345678'), + id('0234567812345679'), + id('0234567812345680'), + id('0234567812345681') + ] + + const spans = [] + const spanContexts = [] + + for (let idx = 0; idx < operations.length; idx++) { + const operation = operations[idx] + const id = ids[idx] + const spanContext = { + _spanId: id, + _sampling: {}, + _trace: { + started: [] + }, + _name: operation, + _tags: {} + } + + // Give first span a custom service name + if ([6, 7].includes(idx)) { + spanContext._tags['service.name'] = 'span-service' + } + + if (idx === 8) { + spanContext._name = 'renamed' + } + + if (idx === 9) { + spanContext._tags.tagged = 'yup' + spanContext._tags.and = 'this' + spanContext._tags.not = 'this' + } + + if (idx === 10) { + spanContext._tags['resource.name'] = 'named' + } + + const span = { + context: sinon.stub().returns(spanContext), + tracer: sinon.stub().returns({ + _service: 'test' + }), + _name: operation + } + + spanContexts.push(spanContext) + spans.push(span) + } + + return { spans, spanContexts } +} + +describe('sampling rule', () => { + let spans + let spanContexts + let SamplingRule + let rule + + beforeEach(() => { + const info = createDummySpans() + spans = info.spans + spanContexts = info.spanContexts + + spanContexts[0]._trace.started.push(...spans) + + SamplingRule = require('../src/sampling_rule') + }) + + describe('match', () => { + it('should match with exact strings', () => { + rule = new SamplingRule({ + service: 'test', + name: 'operation' + }) + + expect(rule.match(spans[0])).to.equal(true) + expect(rule.match(spans[1])).to.equal(false) + expect(rule.match(spans[2])).to.equal(false) + expect(rule.match(spans[3])).to.equal(false) + expect(rule.match(spans[4])).to.equal(false) + expect(rule.match(spans[5])).to.equal(false) + expect(rule.match(spans[6])).to.equal(false) + expect(rule.match(spans[7])).to.equal(false) + expect(rule.match(spans[8])).to.equal(false) + expect(rule.match(spans[9])).to.equal(false) + expect(rule.match(spans[10])).to.equal(false) + }) + + it('should match with regexp', () => { + rule = new SamplingRule({ + service: /test/, + name: /op.*/ + }) + + expect(rule.match(spans[0])).to.equal(true) + expect(rule.match(spans[1])).to.equal(true) + expect(rule.match(spans[2])).to.equal(true) + expect(rule.match(spans[3])).to.equal(true) + expect(rule.match(spans[4])).to.equal(true) + expect(rule.match(spans[5])).to.equal(true) + expect(rule.match(spans[6])).to.equal(false) + expect(rule.match(spans[7])).to.equal(false) + expect(rule.match(spans[8])).to.equal(false) + expect(rule.match(spans[9])).to.equal(true) + expect(rule.match(spans[10])).to.equal(true) + }) + + it('should match with postfix glob', () => { + rule = new SamplingRule({ + service: 'test', + name: 'op*' + }) + + expect(rule.match(spans[0])).to.equal(true) + expect(rule.match(spans[1])).to.equal(false) + expect(rule.match(spans[2])).to.equal(false) + expect(rule.match(spans[3])).to.equal(false) + expect(rule.match(spans[4])).to.equal(false) + expect(rule.match(spans[5])).to.equal(false) + expect(rule.match(spans[6])).to.equal(false) + expect(rule.match(spans[7])).to.equal(false) + expect(rule.match(spans[8])).to.equal(false) + expect(rule.match(spans[9])).to.equal(false) + expect(rule.match(spans[10])).to.equal(false) + }) + + it('should match with prefix glob', () => { + rule = new SamplingRule({ + service: 'test', + name: '*operation' + }) + + expect(rule.match(spans[0])).to.equal(true) + expect(rule.match(spans[1])).to.equal(true) + expect(rule.match(spans[2])).to.equal(true) + expect(rule.match(spans[3])).to.equal(false) + expect(rule.match(spans[4])).to.equal(false) + expect(rule.match(spans[5])).to.equal(false) + expect(rule.match(spans[6])).to.equal(false) + expect(rule.match(spans[7])).to.equal(false) + expect(rule.match(spans[8])).to.equal(false) + expect(rule.match(spans[9])).to.equal(true) + expect(rule.match(spans[10])).to.equal(true) + }) + + it('should match with single character any matcher', () => { + rule = new SamplingRule({ + service: 'test', + name: 'o?eration' + }) + + expect(rule.match(spans[0])).to.equal(true) + expect(rule.match(spans[1])).to.equal(false) + expect(rule.match(spans[2])).to.equal(false) + expect(rule.match(spans[3])).to.equal(false) + expect(rule.match(spans[4])).to.equal(false) + expect(rule.match(spans[5])).to.equal(false) + expect(rule.match(spans[6])).to.equal(false) + expect(rule.match(spans[7])).to.equal(false) + expect(rule.match(spans[8])).to.equal(false) + expect(rule.match(spans[9])).to.equal(false) + expect(rule.match(spans[10])).to.equal(false) + }) + + it('should consider missing service as match-all for service name', () => { + rule = new SamplingRule({ + name: 'sub_second_operation_*' + }) + + expect(rule.match(spans[0])).to.equal(false) + expect(rule.match(spans[1])).to.equal(false) + expect(rule.match(spans[2])).to.equal(false) + // Only 3 and 4 should match because of the name pattern + expect(rule.match(spans[3])).to.equal(true) + expect(rule.match(spans[4])).to.equal(true) + expect(rule.match(spans[5])).to.equal(false) + expect(rule.match(spans[6])).to.equal(false) + expect(rule.match(spans[7])).to.equal(false) + expect(rule.match(spans[8])).to.equal(false) + expect(rule.match(spans[9])).to.equal(false) + expect(rule.match(spans[10])).to.equal(false) + }) + + it('should consider missing name as match-all for span name', () => { + rule = new SamplingRule({ + service: 'test' + }) + + expect(rule.match(spans[0])).to.equal(true) + expect(rule.match(spans[1])).to.equal(true) + expect(rule.match(spans[2])).to.equal(true) + expect(rule.match(spans[3])).to.equal(true) + expect(rule.match(spans[4])).to.equal(true) + expect(rule.match(spans[5])).to.equal(true) + expect(rule.match(spans[8])).to.equal(true) + expect(rule.match(spans[9])).to.equal(true) + expect(rule.match(spans[10])).to.equal(true) + // Should not match because of different service name + expect(rule.match(spans[6])).to.equal(false) + expect(rule.match(spans[7])).to.equal(false) + }) + + it('should use span service name tags where present', () => { + rule = new SamplingRule({ + service: 'span-service' + }) + + expect(rule.match(spans[0])).to.equal(false) + expect(rule.match(spans[1])).to.equal(false) + expect(rule.match(spans[2])).to.equal(false) + expect(rule.match(spans[3])).to.equal(false) + expect(rule.match(spans[4])).to.equal(false) + expect(rule.match(spans[5])).to.equal(false) + expect(rule.match(spans[6])).to.equal(true) + expect(rule.match(spans[7])).to.equal(true) + expect(rule.match(spans[8])).to.equal(false) + expect(rule.match(spans[9])).to.equal(false) + expect(rule.match(spans[10])).to.equal(false) + }) + + it('should match renamed spans', () => { + rule = new SamplingRule({ + service: 'test', + name: 'renamed' + }) + + expect(rule.match(spans[0])).to.equal(false) + expect(rule.match(spans[1])).to.equal(false) + expect(rule.match(spans[2])).to.equal(false) + expect(rule.match(spans[3])).to.equal(false) + expect(rule.match(spans[4])).to.equal(false) + expect(rule.match(spans[5])).to.equal(false) + expect(rule.match(spans[6])).to.equal(false) + expect(rule.match(spans[7])).to.equal(false) + expect(rule.match(spans[8])).to.equal(true) + expect(rule.match(spans[9])).to.equal(false) + expect(rule.match(spans[10])).to.equal(false) + }) + + it('should match tag sets', () => { + rule = new SamplingRule({ + tags: { + tagged: 'yup', + and: 'this' + } + }) + + expect(rule.match(spans[0])).to.equal(false) + expect(rule.match(spans[1])).to.equal(false) + expect(rule.match(spans[2])).to.equal(false) + expect(rule.match(spans[3])).to.equal(false) + expect(rule.match(spans[4])).to.equal(false) + expect(rule.match(spans[5])).to.equal(false) + expect(rule.match(spans[6])).to.equal(false) + expect(rule.match(spans[7])).to.equal(false) + expect(rule.match(spans[8])).to.equal(false) + expect(rule.match(spans[9])).to.equal(true) + expect(rule.match(spans[10])).to.equal(false) + }) + + it('should match resource name', () => { + rule = new SamplingRule({ + resource: 'named' + }) + + expect(rule.match(spans[0])).to.equal(false) + expect(rule.match(spans[1])).to.equal(false) + expect(rule.match(spans[2])).to.equal(false) + expect(rule.match(spans[3])).to.equal(false) + expect(rule.match(spans[4])).to.equal(false) + expect(rule.match(spans[5])).to.equal(false) + expect(rule.match(spans[6])).to.equal(false) + expect(rule.match(spans[7])).to.equal(false) + expect(rule.match(spans[8])).to.equal(false) + expect(rule.match(spans[9])).to.equal(false) + expect(rule.match(spans[10])).to.equal(true) + }) + }) + + describe('sampleRate', () => { + beforeEach(() => { + sinon.stub(Math, 'random').returns(0.5) + }) + + afterEach(() => { + Math.random.restore() + }) + + it('should sample on allowed sample rate', () => { + rule = new SamplingRule({ + service: 'test', + name: 'operation', + sampleRate: 1.0, + maxPerSecond: 1 + }) + + expect(rule.sample()).to.equal(true) + }) + + it('should not sample on non-allowed sample rate', () => { + rule = new SamplingRule({ + service: 'test', + name: 'operation', + sampleRate: 0.3, + maxPerSecond: 1 + }) + + expect(rule.sample()).to.equal(false) + }) + }) + + describe('maxPerSecond', () => { + it('should not create limiter without finite maxPerSecond', () => { + rule = new SamplingRule({ + service: 'test', + name: 'operation', + sampleRate: 1.0 + }) + + expect(rule._limiter).to.equal(undefined) + expect(rule.maxPerSecond).to.equal(undefined) + }) + + it('should create limiter with finite maxPerSecond', () => { + rule = new SamplingRule({ + service: 'test', + name: 'operation', + sampleRate: 1.0, + maxPerSecond: 123 + }) + + expect(rule._limiter).to.not.equal(undefined) + expect(rule).to.have.property('maxPerSecond', 123) + }) + + it('should not sample spans past the rate limit', () => { + rule = new SamplingRule({ + service: 'test', + name: 'operation', + sampleRate: 1.0, + maxPerSecond: 1 + }) + + expect(rule.sample()).to.equal(true) + expect(rule.sample()).to.equal(false) + }) + + it('should allow unlimited rate limits', () => { + rule = new SamplingRule({ + service: 'test', + name: 'operation', + sampleRate: 1.0 + }) + + for (let i = 0; i < 1e3; i++) { + expect(rule.sample()).to.equal(true) + } + }) + + it('should sample if enough time has elapsed', () => { + rule = new SamplingRule({ + service: 'test', + name: 'operation', + sampleRate: 1.0, + maxPerSecond: 1 + }) + + const clock = sinon.useFakeTimers(new Date()) + expect(rule.sample()).to.equal(true) + expect(rule.sample()).to.equal(false) + clock.tick(1000) + expect(rule.sample()).to.equal(true) + }) + }) +}) diff --git a/packages/dd-trace/test/serverless.spec.js b/packages/dd-trace/test/serverless.spec.js index cf4a75c4ac3..021f45e89e8 100644 --- a/packages/dd-trace/test/serverless.spec.js +++ b/packages/dd-trace/test/serverless.spec.js @@ -54,7 +54,7 @@ describe('Serverless', () => { expect(spawnStub).to.have.been.calledOnce }) - it('should spawn mini agent when FUNCTIONS_WORKER_RUNTIME, FUNCTIONS_EXTENSION_VERSION env vars are defined', () => { + it('should spawn mini agent when Azure env vars are defined', () => { process.env.FUNCTIONS_WORKER_RUNTIME = 'node' process.env.FUNCTIONS_EXTENSION_VERSION = '4' @@ -63,26 +63,6 @@ describe('Serverless', () => { expect(spawnStub).to.have.been.calledOnce }) - it('should spawn mini agent when Azure Function env vars are defined and SKU is dynamic', () => { - process.env.FUNCTIONS_WORKER_RUNTIME = 'node' - process.env.FUNCTIONS_EXTENSION_VERSION = '4' - process.env.WEBSITE_SKU = 'Dynamic' - - proxy.init() - - expect(spawnStub).to.have.been.calledOnce - }) - - it('should NOT spawn mini agent when Azure Function env vars are defined but SKU is NOT dynamic', () => { - process.env.FUNCTIONS_WORKER_RUNTIME = 'node' - process.env.FUNCTIONS_EXTENSION_VERSION = '4' - process.env.WEBSITE_SKU = 'Basic' - - proxy.init() - - expect(spawnStub).to.not.have.been.called - }) - it('should log error if mini agent binary path is invalid', () => { process.env.K_SERVICE = 'test_function' process.env.FUNCTION_TARGET = 'function_target' @@ -117,8 +97,8 @@ describe('Serverless', () => { }) expect(path).to.be.equal( - `/home/site/wwwroot/node_modules/@datadog/sma/\ -datadog-serverless-agent-linux-amd64/datadog-serverless-trace-mini-agent` + '/home/site/wwwroot/node_modules/@datadog/sma/' + + 'datadog-serverless-agent-linux-amd64/datadog-serverless-trace-mini-agent' ) }) @@ -130,8 +110,8 @@ datadog-serverless-agent-linux-amd64/datadog-serverless-trace-mini-agent` }) expect(path).to.be.equal( - `/home/site/wwwroot/node_modules/@datadog/sma/\ -datadog-serverless-agent-windows-amd64/datadog-serverless-trace-mini-agent.exe` + '/home/site/wwwroot/node_modules/@datadog/sma/' + + 'datadog-serverless-agent-windows-amd64/datadog-serverless-trace-mini-agent.exe' ) }) }) diff --git a/packages/dd-trace/test/service-naming/schema.spec.js b/packages/dd-trace/test/service-naming/schema.spec.js index ee85b089b57..ff1b183b6ea 100644 --- a/packages/dd-trace/test/service-naming/schema.spec.js +++ b/packages/dd-trace/test/service-naming/schema.spec.js @@ -75,9 +75,11 @@ describe('Service naming', () => { it('should answer undefined on inexistent plugin', () => { expect(resolver.getSchemaItem('messaging', 'inbound', 'foo')).to.be.equal(undefined) }) + it('should answer undefined on inexistent i/o dir', () => { expect(resolver.getSchemaItem('messaging', 'foo', 'kafka')).to.be.equal(undefined) }) + it('should answer undefined on inexistent type', () => { expect(resolver.getSchemaItem('foo', 'inbound', 'kafka')).to.be.equal(undefined) }) @@ -89,6 +91,7 @@ describe('Service naming', () => { expect(dummySchema.messaging.inbound.kafka.opName).to.be.calledWith(extra) }) }) + describe('Service name getter', () => { it('should add service name and passthrough service name arguments', () => { const opts = { tracerService: 'test-service', ...extra } diff --git a/packages/dd-trace/test/setup/core.js b/packages/dd-trace/test/setup/core.js index f7d32157a99..491a1a92122 100644 --- a/packages/dd-trace/test/setup/core.js +++ b/packages/dd-trace/test/setup/core.js @@ -5,6 +5,29 @@ const chai = require('chai') const sinonChai = require('sinon-chai') const proxyquire = require('../proxyquire') +{ + // get-port can often return a port that is already in use, thanks to a race + // condition. This patch adds a retry for 10 iterations, which should be + // enough to avoid flaky tests. The patch is added here in the require cache + // because it's used in all sorts of places. + const getPort = require('get-port') + require.cache[require.resolve('get-port')].exports = async function (...args) { + let tries = 0 + let err = null + while (tries++ < 10) { + try { + return await getPort(...args) + } catch (e) { + if (e.code !== 'EADDRINUSE') { + throw e + } + err = e + } + } + throw err + } +} + chai.use(sinonChai) chai.use(require('../asserts/profile')) @@ -19,3 +42,9 @@ if (global.describe && typeof global.describe.skip !== 'function') { } process.env.DD_INSTRUMENTATION_TELEMETRY_ENABLED = 'false' + +// If this is a release PR, set the SSI variables. +if (/^v\d+\.x$/.test(process.env.GITHUB_BASE_REF || '')) { + process.env.DD_INJECTION_ENABLED = 'true' + process.env.DD_INJECT_FORCE = 'true' +} diff --git a/packages/dd-trace/test/setup/mocha.js b/packages/dd-trace/test/setup/mocha.js index 7d75cd41170..d3520c3fe1c 100644 --- a/packages/dd-trace/test/setup/mocha.js +++ b/packages/dd-trace/test/setup/mocha.js @@ -6,7 +6,6 @@ const os = require('os') const path = require('path') const semver = require('semver') const externals = require('../plugins/externals.json') -const slackReport = require('./slack-report') const runtimeMetrics = require('../../src/runtime_metrics') const agent = require('../plugins/agent') const Nomenclature = require('../../src/service-naming') @@ -20,8 +19,6 @@ global.withPeerService = withPeerService const testedPlugins = agent.testedPlugins -const packageVersionFailures = Object.create({}) - function loadInst (plugin) { const instrumentations = [] @@ -29,7 +26,11 @@ function loadInst (plugin) { loadInstFile(`${plugin}/server.js`, instrumentations) loadInstFile(`${plugin}/client.js`, instrumentations) } catch (e) { - loadInstFile(`${plugin}.js`, instrumentations) + try { + loadInstFile(`${plugin}/main.js`, instrumentations) + } catch (e) { + loadInstFile(`${plugin}.js`, instrumentations) + } } return instrumentations @@ -84,23 +85,26 @@ function withNamingSchema ( const { opName, serviceName } = expected[versionName] - it(`should conform to the naming schema`, done => { - agent - .use(traces => { - const span = selectSpan(traces) - const expectedOpName = typeof opName === 'function' - ? opName() - : opName - const expectedServiceName = typeof serviceName === 'function' - ? serviceName() - : serviceName - - expect(span).to.have.property('name', expectedOpName) - expect(span).to.have.property('service', expectedServiceName) - }) - .then(done) - .catch(done) - spanProducerFn(done) + it('should conform to the naming schema', function () { + this.timeout(10000) + return new Promise((resolve, reject) => { + agent + .use(traces => { + const span = selectSpan(traces) + const expectedOpName = typeof opName === 'function' + ? opName() + : opName + const expectedServiceName = typeof serviceName === 'function' + ? serviceName() + : serviceName + + expect(span).to.have.property('name', expectedOpName) + expect(span).to.have.property('service', expectedServiceName) + }) + .then(resolve) + .catch(reject) + spanProducerFn(reject) + }) }) }) }) @@ -121,7 +125,7 @@ function withNamingSchema ( hooks('v0', true) - const { serviceName } = expected['v1'] + const { serviceName } = expected.v1 it('should pass service name through', done => { agent @@ -144,10 +148,12 @@ function withNamingSchema ( function withPeerService (tracer, pluginName, spanGenerationFn, service, serviceSource, opts = {}) { describe('peer service computation' + (opts.desc ? ` ${opts.desc}` : ''), () => { let computePeerServiceSpy + beforeEach(() => { const plugin = tracer()._pluginManager._pluginsByName[pluginName] computePeerServiceSpy = sinon.stub(plugin._tracerConfig, 'spanComputePeerService').value(true) }) + afterEach(() => { computePeerServiceSpy.restore() }) @@ -187,20 +193,31 @@ function withVersions (plugin, modules, range, cb) { } modules.forEach(moduleName => { + if (process.env.PACKAGE_NAMES) { + const packages = process.env.PACKAGE_NAMES.split(',') + + if (!packages.includes(moduleName)) return + } + const testVersions = new Map() instrumentations .filter(instrumentation => instrumentation.name === moduleName) .forEach(instrumentation => { - const versions = process.env.PACKAGE_VERSION_RANGE ? [process.env.PACKAGE_VERSION_RANGE] + const versions = process.env.PACKAGE_VERSION_RANGE + ? [process.env.PACKAGE_VERSION_RANGE] : instrumentation.versions versions .filter(version => !process.env.RANGE || semver.subset(version, process.env.RANGE)) .forEach(version => { - const min = semver.coerce(version).version + if (version !== '*') { + const min = semver.coerce(version).version + + testVersions.set(min, { range: version, test: min }) + } + const max = require(`../../../../versions/${moduleName}@${version}`).version() - testVersions.set(min, { range: version, test: min }) testVersions.set(max, { range: version, test: version }) }) }) @@ -210,11 +227,10 @@ function withVersions (plugin, modules, range, cb) { .sort(v => v[0].localeCompare(v[0])) .map(v => Object.assign({}, v[1], { version: v[0] })) .forEach(v => { - const versionPath = `${__dirname}/../../../../versions/${moduleName}@${v.test}/node_modules` - - // afterEach contains currentTest data - // after doesn't contain test data nor know if any tests passed/failed - let moduleVersionDidFail = false + const versionPath = path.resolve( + __dirname, '../../../../versions/', + `${moduleName}@${v.test}/node_modules` + ) describe(`with ${moduleName} ${v.range} (${v.version})`, () => { let nodePath @@ -235,23 +251,9 @@ function withVersions (plugin, modules, range, cb) { require('module').Module._initPaths() }) - cb(v.test, moduleName) - - afterEach(function () { - if (this.currentTest.state === 'failed') { - moduleVersionDidFail = true - } - }) + cb(v.test, moduleName, v.version) after(() => { - if (moduleVersionDidFail) { - if (!packageVersionFailures[moduleName]) { - packageVersionFailures[moduleName] = new Set() - } - - packageVersionFailures[moduleName].add(v.version) - } - process.env.NODE_PATH = nodePath require('module').Module._initPaths() }) @@ -278,11 +280,6 @@ function withExports (moduleName, version, exportNames, versionRange, fn) { } exports.mochaHooks = { - // TODO: Figure out how to do this with tap too. - async afterAll () { - await slackReport(packageVersionFailures) - }, - afterEach () { agent.reset() runtimeMetrics.stop() diff --git a/packages/dd-trace/test/setup/services/couchbase.js b/packages/dd-trace/test/setup/services/couchbase.js index 94bddbf7920..022f576be12 100644 --- a/packages/dd-trace/test/setup/services/couchbase.js +++ b/packages/dd-trace/test/setup/services/couchbase.js @@ -12,7 +12,7 @@ function waitForCouchbase () { axios({ method: 'POST', url: cbasEndpoint, - data: { 'statement': 'SELECT * FROM datatest', 'timeout': '75000000us' }, + data: { statement: 'SELECT * FROM datatest', timeout: '75000000us' }, auth: { username: 'Administrator', password: 'password' diff --git a/packages/dd-trace/test/setup/services/kafka.js b/packages/dd-trace/test/setup/services/kafka.js index db1c46b6b72..d114682ba7d 100644 --- a/packages/dd-trace/test/setup/services/kafka.js +++ b/packages/dd-trace/test/setup/services/kafka.js @@ -7,6 +7,7 @@ const kafka = new Kafka({ clientId: 'setup-client', brokers: ['127.0.0.1:9092'] }) +const admin = kafka.admin() const producer = kafka.producer() const consumer = kafka.consumer({ groupId: 'test-group' }) const topic = 'test-topic' @@ -17,6 +18,21 @@ function waitForKafka () { const operation = new RetryOperation('kafka') operation.attempt(async currentAttempt => { try { + await admin.listTopics() + try { + await admin.createTopics({ + topics: [{ + topic, + numPartitions: 1, + replicationFactor: 1 + }] + }) + } catch (e) { + // Ignore since this will fail when the topic already exists. + } finally { + await admin.disconnect() + } + await consumer.connect() await consumer.subscribe({ topic, fromBeginning: true }) await consumer.run({ diff --git a/packages/dd-trace/test/setup/services/mongo.js b/packages/dd-trace/test/setup/services/mongo.js index bafc11b7585..e140dd5078b 100644 --- a/packages/dd-trace/test/setup/services/mongo.js +++ b/packages/dd-trace/test/setup/services/mongo.js @@ -9,7 +9,7 @@ function waitForMongo () { operation.attempt(currentAttempt => { const server = new mongo.Server({ - host: 'localhost', + host: '127.0.0.1', port: 27017, reconnect: false }) diff --git a/packages/dd-trace/test/setup/slack-report.js b/packages/dd-trace/test/setup/slack-report.js deleted file mode 100644 index 5d39b4960f9..00000000000 --- a/packages/dd-trace/test/setup/slack-report.js +++ /dev/null @@ -1,136 +0,0 @@ -/** - * This allows the test suite to post a Slack channel message when test failures related to packages occur. - * The intent is to run nightly and proactively discover incompatibilities as new packages are released. - * The Slack message contains data about the failing package as well as release dates for previous versions. - * This should help an on-call engineer quickly diagnose when an incompatibility was introduced. - */ - -/* eslint-disable no-console */ - -const SLACK_WEBHOOK = process.env.SLACK_WEBHOOK -const SLACK_REPORT_ENABLE = process.env.SLACK_REPORT_ENABLE -const SLACK_MOREINFO = process.env.SLACK_MOREINFO - -const VERSION_EXTRACT = /^v?(\d+)\.(\d+)\.(\d+)$/ - -const FORMATTER = new Intl.RelativeTimeFormat('en-us', { numeric: 'auto' }) - -const TIME_THRESHOLDS = [ - { threshold: 60, unit: 'seconds' }, - { threshold: 60, unit: 'minutes' }, - { threshold: 24, unit: 'hours' }, - { threshold: 7, unit: 'days' }, - { threshold: 365 / 12 / 7, unit: 'weeks' }, - { threshold: 12, unit: 'months' }, - { threshold: Infinity, unit: 'years' } -] - -/** - * failures: { moduleName: Set(moduleVersion) } - */ -module.exports = async (failures) => { - if (!SLACK_REPORT_ENABLE) { - console.log('Slack Reporter is disabled') - return - } - - console.log('Slack Reporter is enabled') - - if (!SLACK_WEBHOOK) { - throw new Error('package reporting via slack webhook is enabled but misconfigured') - } - - const packageNames = Object.keys(failures) - - if (!packageNames.length) { - console.log('Slack Reporter has nothing to report') - return - } - - const descriptions = [] - - for (const packageName of packageNames) { - const versions = Array.from(failures[packageName]) - const description = await describe(packageName, versions) - descriptions.push(description) - } - - let message = descriptions.join('\n\n') - - if (SLACK_MOREINFO) { - // It's not easy to contextually link to individual job failures. - // @see https://github.com/community/community/discussions/8945 - // Instead we add a single link at the end to the overall run. - message += `\n<${SLACK_MOREINFO}|View the failing test(s) here>.` - } - - reportToSlack(message) -} - -async function describe (packageName, versions) { - const pairs = versions.map((v) => `\`${packageName}@${v}\``).join(' and ') - let output = `Nightly tests for ${pairs} are failing!\n` - - const suspects = getSuspectedVersions(versions) - const timestamps = await getVersionData(packageName) - - for (const version of suspects) { - output += `• version .\n` - } - - return output -} - -async function reportToSlack (message) { - await fetch(SLACK_WEBHOOK, { - method: 'POST', - headers: { - 'Content-Type': 'application/json' - }, - body: JSON.stringify({ - text: message, - unfurl_links: false, - unfurl_media: false - }) - }) -} - -async function getVersionData (packageName) { - const res = await fetch(`https://registry.npmjs.org/${packageName}`) - - const body = await res.json() - - const timestamps = body.time - - return timestamps -} - -// TODO: could just to 'semver' package and use .major(), etc, instead of regex -// returns the last three versions of a package that are the most likely to have caused a breaking change -// 1.2.3 -> "1.2.3", "1.2.0", "1.0.0" -// 3.0.0 -> "3.0.0" -function getSuspectedVersions (versions) { - const result = new Set() - - for (const version of versions) { - const [, major, minor, patch] = VERSION_EXTRACT.exec(version) - - result.add(`${major}.${minor}.${patch}`) - result.add(`${major}.${minor}.0`) - result.add(`${major}.0.0`) - } - - return Array.from(result) -} - -function formatTimeAgo (date) { - let duration = (date - new Date()) / 1000 - - for (const range of TIME_THRESHOLDS) { - if (Math.abs(duration) < range.threshold) { - return FORMATTER.format(Math.round(duration), range.unit) - } - duration /= range.threshold - } -} diff --git a/packages/dd-trace/test/span_processor.spec.js b/packages/dd-trace/test/span_processor.spec.js index 584fd680023..5198e1702bc 100644 --- a/packages/dd-trace/test/span_processor.spec.js +++ b/packages/dd-trace/test/span_processor.spec.js @@ -131,4 +131,24 @@ describe('SpanProcessor', () => { expect(SpanSampler).to.have.been.calledWith(config.sampler) }) + + it('should erase the trace and stop execution when tracing=false', () => { + const config = { + tracing: false, + stats: { + enabled: false + } + } + + const processor = new SpanProcessor(exporter, prioritySampler, config) + trace.started = [activeSpan] + trace.finished = [finishedSpan] + + processor.process(finishedSpan) + + expect(trace).to.have.deep.property('started', []) + expect(trace).to.have.deep.property('finished', []) + expect(finishedSpan.context()).to.have.deep.property('_tags', {}) + expect(exporter.export).not.to.have.been.called + }) }) diff --git a/packages/dd-trace/test/span_sampler.spec.js b/packages/dd-trace/test/span_sampler.spec.js index aa79047f7b5..cebc524478c 100644 --- a/packages/dd-trace/test/span_sampler.spec.js +++ b/packages/dd-trace/test/span_sampler.spec.js @@ -2,575 +2,279 @@ require('./setup/tap') -const { expect } = require('chai') const id = require('../src/id') -function createDummySpans () { - const operations = [ - 'operation', - 'sub_operation', - 'second_operation', - 'sub_second_operation_1', - 'sub_second_operation_2', - 'sub_sub_second_operation_2', - 'custom_service_span_1', - 'custom_service_span_2', - 'renamed_operation' - ] +describe('span sampler', () => { + const spies = {} + let SpanSampler + let SamplingRule - const ids = [ - id('0234567812345671'), - id('0234567812345672'), - id('0234567812345673'), - id('0234567812345674'), - id('0234567812345675'), - id('0234567812345676'), - id('0234567812345677') - ] + beforeEach(() => { + if (!SamplingRule) { + SamplingRule = require('../src/sampling_rule') + spies.match = sinon.spy(SamplingRule.prototype, 'match') + spies.sample = sinon.spy(SamplingRule.prototype, 'sample') + spies.sampleRate = sinon.spy(SamplingRule.prototype, 'sampleRate', ['get']) + spies.maxPerSecond = sinon.spy(SamplingRule.prototype, 'maxPerSecond', ['get']) + } + + SpanSampler = proxyquire('../src/span_sampler', { + './sampling_rule': SamplingRule + }) + }) - const spans = [] - const spanContexts = [] + it('should not sample anything when trace is kept', done => { + const sampler = new SpanSampler({}) - for (let idx = 0; idx < operations.length; idx++) { - const operation = operations[idx] - const id = ids[idx] const spanContext = { - _spanId: id, - _sampling: {}, + _spanId: id('1234567812345678'), + _sampling: { + priority: 2 + }, _trace: { started: [] }, - _name: operation, + _name: 'operation', _tags: {} } - - // Give first span a custom service name - if ([6, 7].includes(idx)) { - spanContext._tags['service.name'] = 'span-service' - } - - if (idx === 8) { - spanContext._name = 'renamed' - } - - const span = { + spanContext._trace.started.push({ context: sinon.stub().returns(spanContext), tracer: sinon.stub().returns({ _service: 'test' }), - _name: operation - } - - spanContexts.push(spanContext) - spans.push(span) - } - - return { spans, spanContexts } -} - -describe('span sampler', () => { - let spans - let spanContexts - let SpanSampler - let sampler - - beforeEach(() => { - const info = createDummySpans() - spans = info.spans - spanContexts = info.spanContexts - - spanContexts[0]._trace.started.push(...spans) - - SpanSampler = require('../src/span_sampler') - }) - - describe('without drop', () => { - beforeEach(() => { - spanContexts[0]._sampling.priority = 2 // user keep + _name: 'operation' }) - afterEach(() => { - delete spanContexts[0]._sampling.priority - }) - - it('should not sample anything when trace is kept', done => { - sampler = new SpanSampler({}) - try { - const ingested = sampler.sample(spanContexts[0]) - expect(ingested).to.be.undefined - done() - } catch (err) { done(err) } - }) + try { + const ingested = sampler.sample(spanContext) + expect(ingested).to.be.undefined + done() + } catch (err) { done(err) } }) - describe('rules match properly', () => { - it('should properly sample a single span', () => { - sampler = new SpanSampler({ - spanSamplingRules: [ - { - service: 'test', - name: 'operation', - sampleRate: 1.0, - maxPerSecond: 5 - } - ] - }) - - sampler.sample(spanContexts[0]) - expect(spans[0].context()._spanSampling).to.eql({ - sampleRate: 1.0, - maxPerSecond: 5 - }) - expect(spans[1].context()._spanSampling).to.be.undefined - expect(spans[2].context()._spanSampling).to.be.undefined - expect(spans[3].context()._spanSampling).to.be.undefined - expect(spans[4].context()._spanSampling).to.be.undefined - expect(spans[5].context()._spanSampling).to.be.undefined - expect(spans[6].context()._spanSampling).to.be.undefined - expect(spans[7].context()._spanSampling).to.be.undefined - expect(spans[8].context()._spanSampling).to.be.undefined + it('adds _spanSampling when sampled successfully', () => { + const sampler = new SpanSampler({ + spanSamplingRules: [ + { + service: 'test', + name: 'operation', + sampleRate: 1.0, + maxPerSecond: 5 + } + ] }) - it('should consider missing service as match-all for service name', () => { - sampler = new SpanSampler({ - spanSamplingRules: [ - { - name: 'sub_second_operation_*', - sampleRate: 1.0, - maxPerSecond: 5 - } - ] - }) - - const spanSampling = { - sampleRate: 1.0, - maxPerSecond: 5 - } - sampler.sample(spanContexts[0]) - expect(spans[0].context()._spanSampling).to.be.undefined - expect(spans[1].context()._spanSampling).to.be.undefined - expect(spans[2].context()._spanSampling).to.be.undefined - // Only 3 and 4 should match because of the name pattern - expect(spans[3].context()._spanSampling).to.eql(spanSampling) - expect(spans[4].context()._spanSampling).to.eql(spanSampling) - expect(spans[5].context()._spanSampling).to.be.undefined - expect(spans[6].context()._spanSampling).to.be.undefined - expect(spans[7].context()._spanSampling).to.be.undefined - expect(spans[8].context()._spanSampling).to.be.undefined + const spanContext = { + _spanId: id('1234567812345678'), + _sampling: {}, + _trace: { + started: [] + }, + _name: 'operation', + _tags: {} + } + spanContext._trace.started.push({ + context: sinon.stub().returns(spanContext), + tracer: sinon.stub().returns({ + _service: 'test' + }), + _name: 'operation' }) - it('should consider missing name as match-all for span name', () => { - sampler = new SpanSampler({ - spanSamplingRules: [ - { - service: 'test', - sampleRate: 1.0, - maxPerSecond: 10 - } - ] - }) - - const spanSampling = { - sampleRate: 1.0, - maxPerSecond: 10 - } - sampler.sample(spanContexts[0]) - expect(spans[0].context()._spanSampling).to.eql(spanSampling) - expect(spans[1].context()._spanSampling).to.eql(spanSampling) - expect(spans[2].context()._spanSampling).to.eql(spanSampling) - expect(spans[3].context()._spanSampling).to.eql(spanSampling) - expect(spans[4].context()._spanSampling).to.eql(spanSampling) - expect(spans[5].context()._spanSampling).to.eql(spanSampling) - expect(spans[8].context()._spanSampling).to.eql(spanSampling) - // Should not match because of different service name - expect(spans[6].context()._spanSampling).to.be.undefined - expect(spans[7].context()._spanSampling).to.be.undefined - }) + sampler.sample(spanContext) - it('should stop at first rule match', () => { - sampler = new SpanSampler({ - spanSamplingRules: [ - { - service: 'test', - name: 'operation', - sampleRate: 1.0, - maxPerSecond: 5 - }, - { - service: 'test', - name: 'operation', - sampleRate: 1.0, - maxPerSecond: 10 - } - ] - }) + expect(spies.match).to.be.called + expect(spies.sample).to.be.called + expect(spies.sampleRate.get).to.be.called + expect(spies.maxPerSecond.get).to.be.called - sampler.sample(spanContexts[0]) - expect(spans[0].context()._spanSampling).to.eql({ - sampleRate: 1.0, - maxPerSecond: 5 - }) - expect(spans[1].context()._spanSampling).to.be.undefined - expect(spans[2].context()._spanSampling).to.be.undefined - expect(spans[3].context()._spanSampling).to.be.undefined - expect(spans[4].context()._spanSampling).to.be.undefined - expect(spans[5].context()._spanSampling).to.be.undefined - expect(spans[6].context()._spanSampling).to.be.undefined - expect(spans[7].context()._spanSampling).to.be.undefined - expect(spans[8].context()._spanSampling).to.be.undefined + expect(spanContext._spanSampling).to.eql({ + sampleRate: 1.0, + maxPerSecond: 5 }) + }) - it('should use span service name tags where present', () => { - sampler = new SpanSampler({ - spanSamplingRules: [ - { - service: 'span-service', - sampleRate: 1.0, - maxPerSecond: 5 - } - ] - }) - - const spanSampling = { - sampleRate: 1.0, - maxPerSecond: 5 - } - sampler.sample(spanContexts[0]) - expect(spans[0].context()._spanSampling).to.be.undefined - expect(spans[1].context()._spanSampling).to.be.undefined - expect(spans[2].context()._spanSampling).to.be.undefined - expect(spans[3].context()._spanSampling).to.be.undefined - expect(spans[4].context()._spanSampling).to.be.undefined - expect(spans[5].context()._spanSampling).to.be.undefined - expect(spans[6].context()._spanSampling).to.eql(spanSampling) - expect(spans[7].context()._spanSampling).to.eql(spanSampling) - expect(spans[8].context()._spanSampling).to.be.undefined + it('should stop at first rule match', () => { + const sampler = new SpanSampler({ + spanSamplingRules: [ + { + service: 'does-not-match', + name: 'operation', + sampleRate: 1.0, + maxPerSecond: 3 + }, + { + service: 'test', + name: 'operation', + sampleRate: 1.0, + maxPerSecond: 5 + }, + { + service: 'test', + name: 'operation', + sampleRate: 1.0, + maxPerSecond: 10 + } + ] }) - it('should properly sample multiple single spans with one rule', () => { - sampler = new SpanSampler({ - spanSamplingRules: [ - { - service: 'test', - name: '*_2', - sampleRate: 1.0, - maxPerSecond: 5 - } - ] - }) - - const spanSampling = { - sampleRate: 1.0, - maxPerSecond: 5 - } - sampler.sample(spanContexts[0]) - expect(spans[1].context()._spanSampling).to.be.undefined - expect(spans[2].context()._spanSampling).to.be.undefined - expect(spans[3].context()._spanSampling).to.be.undefined - expect(spans[4].context()._spanSampling).to.eql(spanSampling) - expect(spans[5].context()._spanSampling).to.eql(spanSampling) - expect(spans[6].context()._spanSampling).to.be.undefined - expect(spans[7].context()._spanSampling).to.be.undefined - expect(spans[8].context()._spanSampling).to.be.undefined + const spanContext = { + _spanId: id('1234567812345678'), + _sampling: {}, + _trace: { + started: [] + }, + _name: 'operation', + _tags: {} + } + spanContext._trace.started.push({ + context: sinon.stub().returns(spanContext), + tracer: sinon.stub().returns({ + _service: 'test' + }), + _name: 'operation' }) - it('should properly sample mutiple single spans with multiple rules', () => { - sampler = new SpanSampler({ - spanSamplingRules: [ - { - service: 'test', - name: 'operation', - sampleRate: 1.0, - maxPerSecond: 5 - }, - { - service: 'test', - name: '*_second*', - sampleRate: 1.0, - maxPerSecond: 10 - } - ] - }) - - sampler.sample(spanContexts[0]) - expect(spans[0].context()._spanSampling, { - sampleRate: 1.0, - maxPerSecond: 5 - }) - expect(spans[3].context()._spanSampling, { - sampleRate: 1.0, - maxPerSecond: 10 - }) - expect(spans[4].context()._spanSampling, { - sampleRate: 1.0, - maxPerSecond: 10 - }) - expect(spans[5].context()._spanSampling, { - sampleRate: 1.0, - maxPerSecond: 10 - }) - }) + sampler.sample(spanContext) - it('should properly sample renamed spans', () => { - sampler = new SpanSampler({ - spanSamplingRules: [ - { - service: 'test', - name: 'renamed', - sampleRate: 1.0, - maxPerSecond: 1 - } - ] - }) + expect(spies.match).to.be.called + expect(spies.sample).to.be.called + expect(spies.sampleRate.get).to.be.called + expect(spies.maxPerSecond.get).to.be.called - sampler.sample(spanContexts[0]) - expect(spans[0].context()._spanSampling).to.be.undefined - expect(spans[1].context()._spanSampling).to.be.undefined - expect(spans[2].context()._spanSampling).to.be.undefined - expect(spans[3].context()._spanSampling).to.be.undefined - expect(spans[4].context()._spanSampling).to.be.undefined - expect(spans[5].context()._spanSampling).to.be.undefined - expect(spans[6].context()._spanSampling).to.be.undefined - expect(spans[7].context()._spanSampling).to.be.undefined - expect(spans[8].context()._spanSampling).to.eql({ - sampleRate: 1.0, - maxPerSecond: 1 - }) + expect(spanContext._spanSampling).to.eql({ + sampleRate: 1.0, + maxPerSecond: 5 }) }) - describe('sampleRate', () => { - beforeEach(() => { - sinon.stub(Math, 'random') + it('should sample multiple spans with one rule', () => { + const sampler = new SpanSampler({ + spanSamplingRules: [ + { + service: 'test', + name: '*operation', + sampleRate: 1.0, + maxPerSecond: 5 + } + ] }) - afterEach(() => { - Math.random.restore() - }) - - it('should sample a matched span on allowed sample rate', () => { - Math.random.returns(0.5) - sampler = new SpanSampler({ - spanSamplingRules: [ - { - service: 'test', - name: 'operation', - sampleRate: 1.0, - maxPerSecond: 1 - } - ] - }) + // Create two span contexts + const started = [] + const firstSpanContext = { + _spanId: id('1234567812345678'), + _sampling: {}, + _trace: { + started + }, + _name: 'operation', + _tags: {} + } + const secondSpanContext = { + ...firstSpanContext, + _spanId: id('1234567812345679'), + _name: 'second operation' + } - sampler.sample(spanContexts[0]) - expect(spans[0].context()).to.haveOwnProperty('_spanSampling') + // Add spans for both to the context + started.push({ + context: sinon.stub().returns(firstSpanContext), + tracer: sinon.stub().returns({ + _service: 'test' + }), + _name: 'operation' }) - - it('should not sample a matched span on non-allowed sample rate', () => { - Math.random.returns(0.5) - sampler = new SpanSampler({ - spanSamplingRules: [ - { - service: 'test', - name: 'operation', - sampleRate: 0.3, - maxPerSecond: 1 - } - ] - }) - - sampler.sample(spanContexts[0]) - for (const span of spans) { - expect(span.context()).to.not.haveOwnProperty('_spanSampling') - } + started.push({ + context: sinon.stub().returns(secondSpanContext), + tracer: sinon.stub().returns({ + _service: 'test' + }), + _name: 'operation' }) - it('should selectively sample based on sample rates', () => { - Math.random.returns(0.5) - sampler = new SpanSampler({ - spanSamplingRules: [ - { - service: 'test', - name: 'operation', - sampleRate: 0.3, - maxPerSecond: 1 - }, - { - service: 'test', - name: 'second_operation', - sampleRate: 1.0, - maxPerSecond: 1 - } - ] - }) - - sampler.sample(spanContexts[0]) - expect(spans[2].context()).to.haveOwnProperty('_spanSampling') - expect(spans[0].context()).to.not.haveOwnProperty('_spanSampling') - }) - }) + sampler.sample(firstSpanContext) - describe('maxPerSecond', () => { - it('should not create limiter without finite maxPerSecond', () => { - sampler = new SpanSampler({ - spanSamplingRules: [ - { - service: 'test', - name: 'operation', - sampleRate: 1.0 - } - ] - }) + expect(spies.match).to.be.called + expect(spies.sample).to.be.called + expect(spies.sampleRate.get).to.be.called + expect(spies.maxPerSecond.get).to.be.called - const rule = sampler._rules[0] - expect(rule._limiter).to.equal(undefined) - expect(rule.maxPerSecond).to.equal(undefined) + expect(firstSpanContext._spanSampling).to.eql({ + sampleRate: 1.0, + maxPerSecond: 5 }) - - it('should create limiter with finite maxPerSecond', () => { - sampler = new SpanSampler({ - spanSamplingRules: [ - { - service: 'test', - name: 'operation', - sampleRate: 1.0, - maxPerSecond: 123 - } - ] - }) - - const rule = sampler._rules[0] - expect(rule._limiter).to.not.equal(undefined) - expect(rule).to.have.property('maxPerSecond', 123) + expect(secondSpanContext._spanSampling).to.eql({ + sampleRate: 1.0, + maxPerSecond: 5 }) + }) - it('should not sample spans past the rate limit', () => { - sampler = new SpanSampler({ - spanSamplingRules: [ - { - service: 'test', - name: 'operation', - sampleRate: 1.0, - maxPerSecond: 1 - } - ] - }) - - sampler.sample(spanContexts[0]) - expect(spans[0].context()).to.haveOwnProperty('_spanSampling') - delete spans[0].context()._spanSampling - - // with how quickly these tests execute, the limiter should not allow the - // next call to sample any spans - sampler.sample(spanContexts[0]) - expect(spans[0].context()).to.not.haveOwnProperty('_spanSampling') + it('should sample mutiple spans with multiple rules', () => { + const sampler = new SpanSampler({ + spanSamplingRules: [ + { + service: 'test', + name: 'operation', + sampleRate: 1.0, + maxPerSecond: 5 + }, + { + service: 'test', + name: 'second*', + sampleRate: 1.0, + maxPerSecond: 3 + } + ] }) - it('should map different rules to different rate limiters', () => { - sampler = new SpanSampler({ - spanSamplingRules: [ - { - service: 'test', - name: 'operation', - sampleRate: 1.0, - maxPerSecond: 1 - }, - { - service: 'test', - name: 'sub_operation', - sampleRate: 1.0, - maxPerSecond: 2 - } - ] - }) - - sampler.sample(spanContexts[0]) - expect(spans[0].context()).to.haveOwnProperty('_spanSampling') - expect(spans[1].context()).to.haveOwnProperty('_spanSampling') - delete spans[0].context()._spanSampling - delete spans[1].context()._spanSampling + // Create two span contexts + const started = [] + const firstSpanContext = { + _spanId: id('1234567812345678'), + _sampling: {}, + _trace: { + started + }, + _name: 'operation', + _tags: {} + } + const secondSpanContext = { + ...firstSpanContext, + _spanId: id('1234567812345679'), + _name: 'second operation' + } - // with how quickly these tests execute, the limiter should not allow the - // next call to sample any spans - sampler.sample(spanContexts[0]) - expect(spans[0].context()).to.not.haveOwnProperty('_spanSampling') - expect(spans[1].context()).to.haveOwnProperty('_spanSampling') + // Add spans for both to the context + started.push({ + context: sinon.stub().returns(firstSpanContext), + tracer: sinon.stub().returns({ + _service: 'test' + }), + _name: 'operation' }) - - it('should map limit by all spans matching pattern', () => { - sampler = new SpanSampler({ - spanSamplingRules: [ - { - service: 'test', - name: 'sub_second_operation_*', - sampleRate: 1.0, - maxPerSecond: 3 - } - ] - }) - - // First time around both should have spanSampling to prove match - sampler.sample(spanContexts[0]) - expect(spans[3].context()).to.haveOwnProperty('_spanSampling') - expect(spans[4].context()).to.haveOwnProperty('_spanSampling') - delete spans[3].context()._spanSampling - delete spans[4].context()._spanSampling - - // Second time around only first should have spanSampling to prove limits - sampler.sample(spanContexts[0]) - expect(spans[3].context()).to.haveOwnProperty('_spanSampling') - expect(spans[4].context()).to.not.haveOwnProperty('_spanSampling') + started.push({ + context: sinon.stub().returns(secondSpanContext), + tracer: sinon.stub().returns({ + _service: 'test' + }), + _name: 'operation' }) - it('should allow unlimited rate limits', async () => { - sampler = new SpanSampler({ - spanSamplingRules: [ - { - service: 'test', - name: 'operation', - sampleRate: 1.0 - } - ] - }) - - const interval = setInterval(() => { - sampler.sample(spanContexts[0]) - expect(spans[0].context()).to.haveOwnProperty('_spanSampling') - delete spans[0].context()._spanSampling - }, 1) + sampler.sample(firstSpanContext) - await new Promise(resolve => { - setTimeout(resolve, 1000) - }) + expect(spies.match).to.be.called + expect(spies.sample).to.be.called + expect(spies.sampleRate.get).to.be.called + expect(spies.maxPerSecond.get).to.be.called - clearInterval(interval) + expect(firstSpanContext._spanSampling).to.eql({ + sampleRate: 1.0, + maxPerSecond: 5 }) - - it('should sample if enough time has elapsed', async () => { - sampler = new SpanSampler({ - spanSamplingRules: [ - { - service: 'test', - name: 'operation', - sampleRate: 1.0, - maxPerSecond: 1 - } - ] - }) - - await new Promise(resolve => { - sampler.sample(spanContexts[0]) - const before = spans[0].context()._spanSampling - delete spans[0].context()._spanSampling - - setTimeout(() => { - sampler.sample(spanContexts[0]) - const after = spans[0].context()._spanSampling - delete spans[0].context()._spanSampling - - expect(before).to.eql(after) - resolve() - }, 1000) - }) + expect(secondSpanContext._spanSampling).to.eql({ + sampleRate: 1.0, + maxPerSecond: 3 }) }) }) diff --git a/packages/dd-trace/test/span_stats.spec.js b/packages/dd-trace/test/span_stats.spec.js index 77f59f68641..94aa0e4573b 100644 --- a/packages/dd-trace/test/span_stats.spec.js +++ b/packages/dd-trace/test/span_stats.spec.js @@ -234,7 +234,8 @@ describe('SpanStatsProcessor', () => { port: 8126, url: new URL('http://127.0.0.1:8126'), env: 'test', - tags: { tag: 'some tag' } + tags: { tag: 'some tag' }, + version: '1.0.0' } it('should construct', () => { @@ -253,6 +254,15 @@ describe('SpanStatsProcessor', () => { expect(processor.enabled).to.equal(config.stats.enabled) expect(processor.env).to.equal(config.env) expect(processor.tags).to.deep.equal(config.tags) + expect(processor.version).to.equal(config.version) + }) + + it('should construct a disabled instance if appsec standalone is enabled', () => { + const standaloneConfig = { appsec: { standalone: { enabled: true } }, ...config } + const processor = new SpanStatsProcessor(standaloneConfig) + + expect(processor.enabled).to.be.false + expect(processor.timer).to.be.undefined }) it('should track span stats', () => { @@ -298,7 +308,7 @@ describe('SpanStatsProcessor', () => { expect(exporter.export).to.be.calledWith({ Hostname: hostname(), Env: config.env, - Version: version, + Version: config.version, Stats: [{ Start: 12340000000000, Duration: 10000000000, @@ -323,4 +333,22 @@ describe('SpanStatsProcessor', () => { Sequence: processor.sequence }) }) + + it('should export on interval with default version', () => { + const versionlessConfig = { ...config } + delete versionlessConfig.version + const processor = new SpanStatsProcessor(versionlessConfig) + processor.onInterval() + + expect(exporter.export).to.be.calledWith({ + Hostname: hostname(), + Env: config.env, + Version: version, + Stats: [], + Lang: 'javascript', + TracerVersion: pkg.version, + RuntimeID: processor.tags['runtime-id'], + Sequence: processor.sequence + }) + }) }) diff --git a/packages/dd-trace/test/startup-log.spec.js b/packages/dd-trace/test/startup-log.spec.js index 3b9c2a233f8..4f5cf46b525 100644 --- a/packages/dd-trace/test/startup-log.spec.js +++ b/packages/dd-trace/test/startup-log.spec.js @@ -8,6 +8,7 @@ const tracerVersion = require('../../../package.json').version describe('startup logging', () => { let firstStderrCall let secondStderrCall + before(() => { sinon.stub(console, 'info') sinon.stub(console, 'warn') diff --git a/packages/dd-trace/test/tagger.spec.js b/packages/dd-trace/test/tagger.spec.js index 4f0426ea179..161136819d2 100644 --- a/packages/dd-trace/test/tagger.spec.js +++ b/packages/dd-trace/test/tagger.spec.js @@ -1,6 +1,10 @@ 'use strict' +const constants = require('../src/constants') require('./setup/tap') +const ERROR_MESSAGE = constants.ERROR_MESSAGE +const ERROR_STACK = constants.ERROR_STACK +const ERROR_TYPE = constants.ERROR_TYPE describe('tagger', () => { let carrier @@ -45,4 +49,29 @@ describe('tagger', () => { it('should handle missing carrier', () => { expect(() => tagger.add()).not.to.throw() }) + + it('should set trace error', () => { + tagger.add(carrier, { + [ERROR_TYPE]: 'foo', + [ERROR_MESSAGE]: 'foo', + [ERROR_STACK]: 'foo', + doNotSetTraceError: true + }) + + expect(carrier).to.have.property(ERROR_TYPE, 'foo') + expect(carrier).to.have.property(ERROR_MESSAGE, 'foo') + expect(carrier).to.have.property(ERROR_STACK, 'foo') + expect(carrier).to.not.have.property('setTraceError') + + tagger.add(carrier, { + [ERROR_TYPE]: 'foo', + [ERROR_MESSAGE]: 'foo', + [ERROR_STACK]: 'foo' + }) + + expect(carrier).to.have.property(ERROR_TYPE, 'foo') + expect(carrier).to.have.property(ERROR_MESSAGE, 'foo') + expect(carrier).to.have.property(ERROR_STACK, 'foo') + expect(carrier).to.have.property('setTraceError', true) + }) }) diff --git a/packages/dd-trace/test/telemetry/dependencies.spec.js b/packages/dd-trace/test/telemetry/dependencies.spec.js index 971d8066503..385415137d8 100644 --- a/packages/dd-trace/test/telemetry/dependencies.spec.js +++ b/packages/dd-trace/test/telemetry/dependencies.spec.js @@ -15,6 +15,7 @@ describe('dependencies', () => { const dependencies = proxyquire('../../src/telemetry/dependencies', { 'dc-polyfill': dc }) + dependencies.start() expect(subscribe).to.have.been.calledOnce }) @@ -29,17 +30,22 @@ describe('dependencies', () => { let dependencies let sendData let requirePackageJson + let getRetryData + let updateRetryData beforeEach(() => { requirePackageJson = sinon.stub() sendData = sinon.stub() + getRetryData = sinon.stub() + updateRetryData = sinon.stub() dependencies = proxyquire('../../src/telemetry/dependencies', { + './index': { getRetryData, updateRetryData }, './send-data': { sendData }, '../require-package-json': requirePackageJson }) global.setImmediate = function (callback) { callback() } - dependencies.start(config, application, host) + dependencies.start(config, application, host, getRetryData, updateRetryData) // force first publish to load cached requires moduleLoadStartChannel.publish({}) @@ -48,6 +54,8 @@ describe('dependencies', () => { afterEach(() => { dependencies.stop() sendData.reset() + getRetryData.reset() + updateRetryData.reset() global.setImmediate = originalSetImmediate }) @@ -265,7 +273,7 @@ describe('dependencies', () => { expect(sendData).to.have.been.calledOnce }) - it('should call sendData twice with more than 1000 dependencies', (done) => { + it('should call sendData twice with more than 2000 dependencies', (done) => { const requestPrefix = 'custom-module' requirePackageJson.returns({ version: '1.0.0' }) const timeouts = [] @@ -280,7 +288,7 @@ describe('dependencies', () => { timeouts.push(timeout) return timeout } - for (let i = 0; i < 1200; i++) { + for (let i = 0; i < 2200; i++) { const request = requestPrefix + i const filename = path.join(basepathWithoutNodeModules, 'node_modules', request, 'index.js') moduleLoadStartChannel.publish({ request, filename }) @@ -294,4 +302,150 @@ describe('dependencies', () => { }) }) }) + + describe('with configuration', () => { + const config = { + telemetry: { + dependencyCollection: false + } + } + const application = 'test' + const host = 'host' + const basepathWithoutNodeModules = process.cwd().replace(/node_modules/g, 'nop') + + let dependencies + let sendData + let requirePackageJson + let getRetryData + let updateRetryData + + beforeEach(() => { + requirePackageJson = sinon.stub() + sendData = sinon.stub() + getRetryData = sinon.stub() + updateRetryData = sinon.stub() + dependencies = proxyquire('../../src/telemetry/dependencies', { + './index': { getRetryData, updateRetryData }, + './send-data': { sendData }, + '../require-package-json': requirePackageJson + }) + global.setImmediate = function (callback) { callback() } + + dependencies.start(config, application, host, getRetryData, updateRetryData) + + // force first publish to load cached requires + moduleLoadStartChannel.publish({}) // called once here + const request = 'custom-module' + requirePackageJson.returns({ version: '1.0.0' }) + const filename = path.join(basepathWithoutNodeModules, 'node_modules', request, 'index.js') + moduleLoadStartChannel.publish({ request, filename }) // called again here + }) + + afterEach(() => { + dependencies.stop() + sendData.reset() + getRetryData.reset() + updateRetryData.reset() + global.setImmediate = originalSetImmediate + }) + + it('should not call sendData for modules not captured in the initial load', done => { + setTimeout(() => { + // using sendData.callCount wasn't working properly + const timesCalledBeforeLazyLoad = sendData.getCalls().length + + const request = 'custom-module2' + const filename = path.join(basepathWithoutNodeModules, 'node_modules', request, 'index.js') + moduleLoadStartChannel.publish({ request, filename }) // should not be called here + + expect(sendData.getCalls().length).to.equal(timesCalledBeforeLazyLoad) + done() + }, 5) // simulate lazy-loaded dependency, small ms delay to be safe + }) + }) + + describe('on failed request', () => { + const config = {} + const application = 'test' + const host = 'host' + const basepathWithoutNodeModules = process.cwd().replace(/node_modules/g, 'nop') + let dependencies + let sendData + let requirePackageJson + let capturedRequestType + let getRetryData + let updateRetryData + + beforeEach(() => { + requirePackageJson = sinon.stub() + sendData = (config, application, host, reqType, payload, cb = () => {}) => { + capturedRequestType = reqType + // Simulate an HTTP error by calling the callback with an error + cb(new Error('HTTP request error'), { + payload, + reqType: 'app-integrations-change' + }) + } + getRetryData = sinon.stub() + updateRetryData = sinon.stub() + dependencies = proxyquire('../../src/telemetry/dependencies', { + './send-data': { sendData }, + '../require-package-json': requirePackageJson + }) + global.setImmediate = function (callback) { callback() } + + dependencies.start(config, application, host, getRetryData, updateRetryData) + + // force first publish to load cached requires + moduleLoadStartChannel.publish({}) + }) + + afterEach(() => { + dependencies.stop() + getRetryData.reset() + updateRetryData.reset() + global.setImmediate = originalSetImmediate + }) + + it('should update retry data', () => { + const request = 'custom-module' + requirePackageJson.returns({ version: '1.0.0' }) + const filename = path.join(basepathWithoutNodeModules, 'node_modules', request, 'index.js') + moduleLoadStartChannel.publish({ request, filename }) + // expect(getRetryData).to.have.been.calledOnce + expect(capturedRequestType).to.equals('app-dependencies-loaded') + // expect(sendData).to.have.been.calledOnce + // expect(updateRetryData).to.have.been.calledOnce + }) + + it('should create batch request', () => { + let request = 'custom-module' + requirePackageJson.returns({ version: '1.0.0' }) + let filename = path.join(basepathWithoutNodeModules, 'node_modules', request, 'index.js') + moduleLoadStartChannel.publish({ request, filename }) + expect(getRetryData).to.have.been.calledOnce + expect(capturedRequestType).to.equals('app-dependencies-loaded') + expect(updateRetryData).to.have.been.calledOnce + + getRetryData.returns({ + request_type: 'app-integrations-change', + payload: { + integrations: [{ + name: 'zoo1', + enabled: true, + auto_enabled: true + }] + } + + }) + + request = 'even-more-custom-module' + requirePackageJson.returns({ version: '1.0.0' }) + filename = path.join(basepathWithoutNodeModules, 'node_modules', request, 'index.js') + moduleLoadStartChannel.publish({ request, filename }) + expect(getRetryData).to.have.been.calledTwice + expect(capturedRequestType).to.equals('message-batch') + expect(updateRetryData).to.have.been.calledTwice + }) + }) }) diff --git a/packages/dd-trace/test/telemetry/index.spec.js b/packages/dd-trace/test/telemetry/index.spec.js index 0a089b0c642..306d7a16c30 100644 --- a/packages/dd-trace/test/telemetry/index.spec.js +++ b/packages/dd-trace/test/telemetry/index.spec.js @@ -3,33 +3,27 @@ require('../setup/tap') const tracerVersion = require('../../../../package.json').version -const proxyquire = require('proxyquire') +const proxyquire = require('proxyquire').noPreserveCache() const http = require('http') const { once } = require('events') const { storage } = require('../../../datadog-core') const os = require('os') +const sinon = require('sinon') + +const DEFAULT_HEARTBEAT_INTERVAL = 60000 let traceAgent describe('telemetry', () => { - const HEARTBEAT_INTERVAL = 60000 - let origSetInterval let telemetry let pluginsByName before(done => { - origSetInterval = setInterval - - global.setInterval = (fn, interval) => { - expect(interval).to.equal(HEARTBEAT_INTERVAL) - // we only want one of these - return setTimeout(fn, 100) - } - // I'm not sure how, but some other test in some other file keeps context // alive after it's done, meaning this test here runs in its async context. // If we don't no-op the server inside it, it will trace it, which will // screw up this test file entirely. -- bengl + storage.run({ noop: true }, () => { traceAgent = http.createServer(async (req, res) => { const chunks = [] @@ -65,7 +59,7 @@ describe('telemetry', () => { circularObject.child.parent = circularObject telemetry.start({ - telemetry: { enabled: true, heartbeatInterval: HEARTBEAT_INTERVAL }, + telemetry: { enabled: true, heartbeatInterval: DEFAULT_HEARTBEAT_INTERVAL }, hostname: 'localhost', port: traceAgent.address().port, service: 'test service', @@ -75,9 +69,16 @@ describe('telemetry', () => { 'runtime-id': '1a2b3c' }, circularObject, + appsec: { enabled: true }, + profiling: { enabled: 'true' }, peerServiceMapping: { - 'service_1': 'remapped_service_1', - 'service_2': 'remapped_service_2' + service_1: 'remapped_service_1', + service_2: 'remapped_service_2' + }, + installSignature: { + id: '68e75c48-57ca-4a12-adfc-575c4b05fcbe', + type: 'k8s_single_step', + time: '1703188212' } }, { _pluginsByName: pluginsByName @@ -87,29 +88,30 @@ describe('telemetry', () => { after(() => { telemetry.stop() traceAgent.close() - global.setInterval = origSetInterval }) it('should send app-started', () => { return testSeq(1, 'app-started', payload => { - expect(payload).to.deep.include({ + expect(payload).to.have.property('products').that.deep.equal({ + appsec: { enabled: true }, + profiler: { version: tracerVersion, enabled: true } + }) + expect(payload).to.have.property('install_signature').that.deep.equal({ + install_id: '68e75c48-57ca-4a12-adfc-575c4b05fcbe', + install_type: 'k8s_single_step', + install_time: '1703188212' + }) + }) + }) + + it('should send app-integrations', () => { + return testSeq(2, 'app-integrations-change', payload => { + expect(payload).to.deep.equal({ integrations: [ { name: 'foo2', enabled: true, auto_enabled: true }, { name: 'bar2', enabled: false, auto_enabled: true } - ], - dependencies: [] - }).and.to.have.property('configuration').that.include.members([ - { name: 'telemetry.enabled', value: true }, - { name: 'hostname', value: 'localhost' }, - { name: 'port', value: traceAgent.address().port }, - { name: 'service', value: 'test service' }, - { name: 'version', value: '1.2.3-beta4' }, - { name: 'env', value: 'preprod' }, - { name: 'tags.runtime-id', value: '1a2b3c' }, - { name: 'circularObject.field', value: 'parent_value' }, - { name: 'circularObject.child.field', value: 'child_value' }, - { name: 'peerServiceMapping', value: 'service_1:remapped_service_1,service_2:remapped_service_2' } - ]) + ] + }) }) }) @@ -117,7 +119,7 @@ describe('telemetry', () => { pluginsByName.baz2 = { _enabled: true } telemetry.updateIntegrations() - return testSeq(2, 'app-integrations-change', payload => { + return testSeq(3, 'app-integrations-change', payload => { expect(payload).to.deep.equal({ integrations: [ { name: 'baz2', enabled: true, auto_enabled: true } @@ -130,7 +132,7 @@ describe('telemetry', () => { pluginsByName.boo2 = { _enabled: true } telemetry.updateIntegrations() - return testSeq(3, 'app-integrations-change', payload => { + return testSeq(4, 'app-integrations-change', payload => { expect(payload).to.deep.equal({ integrations: [ { name: 'boo2', enabled: true, auto_enabled: true } @@ -139,9 +141,9 @@ describe('telemetry', () => { }) }) - // TODO: make this work regardless of the test runner - it.skip('should send app-closing', () => { - process.emit('beforeExit') + // TODO: test it's called on beforeExit instead of calling directly + it('should send app-closing', () => { + telemetry.appClosing() return testSeq(5, 'app-closing', payload => { expect(payload).to.deep.equal({}) }) @@ -163,140 +165,769 @@ describe('telemetry', () => { server.close() done() }, 10) + clearTimeout() + }) + }) + + it('should not send app-closing if telemetry is not enabled', () => { + const sendDataStub = sinon.stub() + const notEnabledTelemetry = proxyquire('../../src/telemetry', { + './send-data': { + sendData: sendDataStub + } + }) + notEnabledTelemetry.start({ + telemetry: { enabled: false, heartbeatInterval: DEFAULT_HEARTBEAT_INTERVAL }, + appsec: { enabled: false }, + profiling: { enabled: false } + }, { + _pluginsByName: pluginsByName }) + notEnabledTelemetry.appClosing() + expect(sendDataStub.called).to.be.false }) }) describe('telemetry app-heartbeat', () => { - const HEARTBEAT_INTERVAL = 60 - let origSetInterval + const HEARTBEAT_INTERVAL = 60000 let telemetry let pluginsByName + let clock - before(done => { - origSetInterval = setInterval + before(() => { + clock = sinon.useFakeTimers() + }) - global.setInterval = (fn, interval) => { - expect(interval).to.equal(HEARTBEAT_INTERVAL) - // we only want one of these - return setTimeout(fn, 100) - } + after(() => { + clock.restore() + telemetry.stop() + traceAgent.close() + }) - storage.run({ noop: true }, () => { - traceAgent = http.createServer(async (req, res) => { - const chunks = [] - for await (const chunk of req) { - chunks.push(chunk) + it('should send heartbeat in uniform intervals', (done) => { + let beats = 0 // to keep track of the amont of times extendedHeartbeat is called + const sendDataRequest = { + sendData: (config, application, host, reqType, payload, cb = () => {}) => { + if (reqType === 'app-heartbeat') { + beats++ } - req.body = JSON.parse(Buffer.concat(chunks).toString('utf8')) - traceAgent.reqs.push(req) - traceAgent.emit('handled-req') - res.end() - }).listen(0, done) + } + } + telemetry = proxyquire('../../src/telemetry', { + '../exporters/common/docker': { + id () { + return 'test docker id' + } + }, + './send-data': sendDataRequest }) - traceAgent.reqs = [] + telemetry.start({ + telemetry: { enabled: true, heartbeatInterval: HEARTBEAT_INTERVAL }, + hostname: 'localhost', + port: 0, + service: 'test service', + version: '1.2.3-beta4', + appsec: { enabled: true }, + profiling: { enabled: true }, + env: 'preprod', + tags: { + 'runtime-id': '1a2b3c' + } + }, { + _pluginsByName: pluginsByName + }) + clock.tick(HEARTBEAT_INTERVAL) + expect(beats).to.equal(1) + clock.tick(HEARTBEAT_INTERVAL) + expect(beats).to.equal(2) + done() + }) +}) + +describe('Telemetry extended heartbeat', () => { + const HEARTBEAT_INTERVAL = 43200000 + let telemetry + let pluginsByName + let clock + beforeEach(() => { + clock = sinon.useFakeTimers() + }) + + afterEach(() => { + clock.restore() + telemetry.stop() + traceAgent.close() + }) + + it('should be sent every 24 hours', (done) => { + let extendedHeartbeatRequest + let beats = 0 // to keep track of the amont of times extendedHeartbeat is called + const sendDataRequest = { + sendData: (config, application, host, reqType, payload, cb = () => {}) => { + if (reqType === 'app-started') { + cb() + return + } + + if (reqType === 'app-extended-heartbeat') { + beats++ + extendedHeartbeatRequest = reqType + } + } + + } telemetry = proxyquire('../../src/telemetry', { '../exporters/common/docker': { id () { return 'test docker id' } + }, + './send-data': sendDataRequest + }) + + telemetry.start({ + telemetry: { enabled: true, heartbeatInterval: HEARTBEAT_INTERVAL }, + hostname: 'localhost', + port: 0, + service: 'test service', + version: '1.2.3-beta4', + appsec: { enabled: true }, + profiling: { enabled: true }, + env: 'preprod', + tags: { + 'runtime-id': '1a2b3c' + } + }, { + _pluginsByName: pluginsByName + }) + clock.tick(86400000) + expect(extendedHeartbeatRequest).to.equal('app-extended-heartbeat') + expect(beats).to.equal(1) + clock.tick(86400000) + expect(beats).to.equal(2) + done() + }) + + it('be sent with up-to-date configuration values', (done) => { + let configuration + const sendDataRequest = { + sendData: (config, application, host, reqType, payload, cb = () => {}) => { + if (reqType === 'app-extended-heartbeat') { + configuration = payload.configuration + } } + + } + telemetry = proxyquire('../../src/telemetry', { + '../exporters/common/docker': { + id () { + return 'test docker id' + } + }, + './send-data': sendDataRequest }) + const config = { + telemetry: { enabled: true, heartbeatInterval: HEARTBEAT_INTERVAL }, + hostname: 'localhost', + port: 0, + service: 'test service', + version: '1.2.3-beta4', + appsec: { enabled: true }, + profiling: { enabled: true }, + env: 'preprod', + tags: { + 'runtime-id': '1a2b3c' + } + } + + telemetry.start(config, { _pluginsByName: pluginsByName }) + + clock.tick(86400000) + expect(configuration).to.deep.equal([]) + + const changes = [ + { + name: 'test', + value: true, + origin: 'code' + } + ] + telemetry.updateConfig(changes, config) + clock.tick(86400000) + expect(configuration).to.deep.equal(changes) + + const updatedChanges = [ + { + name: 'test', + value: false, + origin: 'code' + } + ] + telemetry.updateConfig(updatedChanges, config) + clock.tick(86400000) + expect(configuration).to.deep.equal(updatedChanges) + + const changeNeedingNameRemapping = [ + { + name: 'sampleRate', // one of the config names that require a remapping + value: 0, + origin: 'code' + } + ] + const expectedConfigList = [ + updatedChanges[0], + { + ...changeNeedingNameRemapping[0], + name: 'DD_TRACE_SAMPLE_RATE' // remapped name + } + ] + telemetry.updateConfig(changeNeedingNameRemapping, config) + clock.tick(86400000) + expect(configuration).to.deep.equal(expectedConfigList) + + const samplingRule = [ + { + name: 'sampler.rules', // one of the config names that require a remapping + value: [ + { service: '*', sampling_rate: 1 }, + { + service: 'svc*', + resource: '*abc', + name: 'op-??', + tags: { 'tag-a': 'ta-v*', 'tag-b': 'tb-v?', 'tag-c': 'tc-v' }, + sample_rate: 0.5 + } + ], + origin: 'code' + } + ] + const expectedConfigListWithSamplingRules = + expectedConfigList.concat([ + { + name: 'DD_TRACE_SAMPLING_RULES', + value: + // eslint-disable-next-line max-len + '[{"service":"*","sampling_rate":1},{"service":"svc*","resource":"*abc","name":"op-??","tags":{"tag-a":"ta-v*","tag-b":"tb-v?","tag-c":"tc-v"},"sample_rate":0.5}]', + origin: 'code' + } + ]) + telemetry.updateConfig(samplingRule, config) + clock.tick(86400000) + expect(configuration).to.deep.equal(expectedConfigListWithSamplingRules) + done() + }) +}) + +// deleted this test for now since the global interval is now used for app-extended heartbeat +// which is not supposed to be configurable +// will ask Bryan why being able to change the interval is important after he is back from parental leave +describe('Telemetry retry', () => { + let telemetry + let capturedRequestType + let capturedPayload + let count = 0 + let pluginsByName + let clock + const HEARTBEAT_INTERVAL = 60000 + + beforeEach(() => { + clock = sinon.useFakeTimers() pluginsByName = { foo2: { _enabled: true }, bar2: { _enabled: false } } + }) + + afterEach(() => { + clock.restore() + }) + + it('should retry data on next app change', () => { + const sendDataError = { + sendData: (config, application, host, reqType, payload, cb = () => {}) => { + capturedRequestType = reqType + capturedPayload = payload + + if (count < 2) { + count += 1 + return + } + // Simulate an HTTP error by calling the callback with an error + cb(new Error('HTTP request error'), { + payload, + reqType: 'app-integrations-change' + }) + } - const circularObject = { - child: { parent: null, field: 'child_value' }, - field: 'parent_value' } - circularObject.child.parent = circularObject + telemetry = proxyquire('../../src/telemetry', { + '../exporters/common/docker': { + id () { + return 'test docker id' + } + }, + './send-data': sendDataError + }) telemetry.start({ telemetry: { enabled: true, heartbeatInterval: HEARTBEAT_INTERVAL }, hostname: 'localhost', - port: traceAgent.address().port, + port: 0, service: 'test service', version: '1.2.3-beta4', + appsec: { enabled: true }, + profiling: { enabled: true }, env: 'preprod', tags: { 'runtime-id': '1a2b3c' - }, - circularObject + } }, { _pluginsByName: pluginsByName }) - }) - after(() => { - setTimeout(() => { - telemetry.stop() - traceAgent.close() - global.setInterval = origSetInterval - }, HEARTBEAT_INTERVAL * 3) + pluginsByName.boo3 = { _enabled: true } + telemetry.updateIntegrations() + expect(capturedRequestType).to.equal('app-integrations-change') + expect(capturedPayload).to.deep.equal({ + integrations: [{ + name: 'boo3', + enabled: true, + auto_enabled: true + }] + }) + + pluginsByName.boo5 = { _enabled: true } + telemetry.updateIntegrations() + expect(capturedRequestType).to.equal('message-batch') + expect(capturedPayload).to.deep.equal([{ + request_type: 'app-integrations-change', + payload: { + integrations: [{ + name: 'boo5', + enabled: true, + auto_enabled: true + }] + } + + }, { + request_type: 'app-integrations-change', + payload: { + integrations: [{ + name: 'boo3', + enabled: true, + auto_enabled: true + }] + } + + }] + ) }) - it('should send app-heartbeat at uniform intervals', () => { - // TODO: switch to clock.tick - setTimeout(() => { - const heartbeats = [] - const reqCount = traceAgent.reqs.length - for (let i = 0; i < reqCount; i++) { - const req = traceAgent.reqs[i] - if (req.headers && req.headers['dd-telemetry-request-type'] === 'app-heartbeat') { - heartbeats.push(req.body.tracer_time) + it('should retry data on next heartbeat', () => { + const sendDataError = { + sendData: (config, application, host, reqType, payload, cb = () => {}) => { + // skipping startup command + if (reqType === 'app-started') { + cb() + return + } + // skipping startup command + if (reqType === 'message-batch') { + capturedRequestType = reqType + capturedPayload = payload + cb() + return + } + // Simulate an HTTP error by calling the callback with an error + cb(new Error('HTTP request error'), { + payload, + reqType + }) + } + + } + telemetry = proxyquire('../../src/telemetry', { + '../exporters/common/docker': { + id () { + return 'test docker id' } + }, + './send-data': sendDataError + }) + + telemetry.start({ + telemetry: { enabled: true, heartbeatInterval: HEARTBEAT_INTERVAL }, + hostname: 'localhost', + port: 0, + service: 'test service', + version: '1.2.3-beta4', + appsec: { enabled: true }, + profiling: { enabled: true }, + env: 'preprod', + tags: { + 'runtime-id': '1a2b3c' } - expect(heartbeats.length).to.be.greaterThanOrEqual(2) - for (let k = 0; k++; k < heartbeats.length - 1) { - expect(heartbeats[k + 1] - heartbeats[k]).to.be.equal(1) + }, { + _pluginsByName: pluginsByName + }) + // jump to next heartbeat request + clock.tick(HEARTBEAT_INTERVAL) + expect(capturedRequestType).to.equal('message-batch') + expect(capturedPayload).to.deep.equal([{ + request_type: 'app-heartbeat', + payload: {} + }, { + request_type: 'app-integrations-change', + payload: { + integrations: [{ + name: 'foo2', + enabled: true, + auto_enabled: true + }, + { + name: 'bar2', + enabled: false, + auto_enabled: true + }] } - }, HEARTBEAT_INTERVAL * 3) + + }] + ) }) -}) -describe('telemetry with interval change', () => { - it('should set the interval correctly', (done) => { - const telemetry = proxyquire('../../src/telemetry', { + it('should send regular request after completed batch request ', () => { + const sendDataError = { + sendData: (config, application, host, reqType, payload, cb = () => {}) => { + capturedRequestType = reqType + capturedPayload = payload + + // skipping startup command + if (reqType === 'app-started' || reqType === 'message-batch') { + cb() + return + } + + // Simulate an HTTP error by calling the callback with an error + cb(new Error('HTTP request error'), { + payload, + reqType: 'app-integrations-change' + }) + } + + } + telemetry = proxyquire('../../src/telemetry', { '../exporters/common/docker': { id () { return 'test docker id' } }, - './send-data': { - sendData: () => {} + './send-data': sendDataError + }) + + telemetry.start({ + telemetry: { enabled: true, heartbeatInterval: HEARTBEAT_INTERVAL }, + hostname: 'localhost', + port: 0, + service: 'test service', + version: '1.2.3-beta4', + appsec: { enabled: true }, + profiling: { enabled: true }, + env: 'preprod', + tags: { + 'runtime-id': '1a2b3c' } + }, { + _pluginsByName: pluginsByName }) + pluginsByName.foo1 = { _enabled: true } + telemetry.updateIntegrations() // This sends an batch message and succeeds + + pluginsByName.zoo1 = { _enabled: true } + telemetry.updateIntegrations() + expect(capturedRequestType).to.equal('app-integrations-change') + + expect(capturedPayload).to.deep.equal({ + integrations: [{ + name: 'zoo1', + enabled: true, + auto_enabled: true + }] + }) + }) + + it('should updated batch request after previous fail', () => { + const sendDataError = { + sendData: (config, application, host, reqType, payload, cb = () => {}) => { + capturedRequestType = reqType + capturedPayload = payload + + // skipping startup command + if (reqType === 'app-started') { + cb() + return + } + + // Simulate an HTTP error by calling the callback with an error + cb(new Error('HTTP request error'), { + payload, + reqType + }) + } - let intervalSetCorrectly - global.setInterval = (fn, interval) => { - expect(interval).to.equal(12345000) - intervalSetCorrectly = true - return setTimeout(fn, 1) } + telemetry = proxyquire('../../src/telemetry', { + '../exporters/common/docker': { + id () { + return 'test docker id' + } + }, + './send-data': sendDataError + }) + // Start function sends 2 messages app-started & app-integrations-change telemetry.start({ - telemetry: { enabled: true, heartbeatInterval: 12345000 }, + telemetry: { enabled: true, heartbeatInterval: HEARTBEAT_INTERVAL }, hostname: 'localhost', - port: 8126, + port: 0, service: 'test service', version: '1.2.3-beta4', + appsec: { enabled: true }, + profiling: { enabled: true }, env: 'preprod', tags: { 'runtime-id': '1a2b3c' } }, { - _pluginsByName: {} + _pluginsByName: pluginsByName + }) + + pluginsByName.foo1 = { _enabled: true } + telemetry.updateIntegrations() // This sends an batch message and fails + + pluginsByName.zoo1 = { _enabled: true } + telemetry.updateIntegrations() + + expect(capturedRequestType).to.equal('message-batch') + expect(capturedPayload).to.deep.equal([{ + request_type: 'app-integrations-change', + payload: { + integrations: [{ + name: 'zoo1', + enabled: true, + auto_enabled: true + }] + } + + }, { + request_type: 'app-integrations-change', + payload: { + integrations: [{ + name: 'foo1', + enabled: true, + auto_enabled: true + }] + } + + }] + ) + }) + + it('should set extended heartbeat payload', async () => { + let extendedHeartbeatRequest + let extendedHeartbeatPayload + const sendDataError = { + sendData: (config, application, host, reqType, payload, cb = () => {}) => { + // skipping startup command + if (reqType === 'app-started') { + cb() + return + } + + if (reqType === 'app-extended-heartbeat') { + extendedHeartbeatRequest = reqType + extendedHeartbeatPayload = payload + return + } + + // Simulate an HTTP error by calling the callback with an error + cb(new Error('HTTP request error'), { + payload, + reqType + }) + } + + } + telemetry = proxyquire('../../src/telemetry', { + '../exporters/common/docker': { + id () { + return 'test docker id' + } + }, + './send-data': sendDataError }) - process.nextTick(() => { - expect(intervalSetCorrectly).to.be.true + // Start function sends 2 messages app-started & app-integrations-change + telemetry.start({ + telemetry: { enabled: true, heartbeatInterval: HEARTBEAT_INTERVAL }, + hostname: 'localhost', + port: 0, + service: 'test service', + version: '1.2.3-beta4', + appsec: { enabled: true }, + profiling: { enabled: true }, + env: 'preprod', + tags: { + 'runtime-id': '1a2b3c' + } + }, + { + _pluginsByName: pluginsByName + }) + pluginsByName.foo1 = { _enabled: true } + telemetry.updateIntegrations() // This sends an batch message and fails + // Skip forward a day + clock.tick(86400000) + expect(extendedHeartbeatRequest).to.equal('app-extended-heartbeat') + expect(extendedHeartbeatPayload).to.haveOwnProperty('integrations') + expect(extendedHeartbeatPayload.integrations).to.deep.include({ + integrations: [ + { name: 'foo2', enabled: true, auto_enabled: true }, + { name: 'bar2', enabled: false, auto_enabled: true } + ] + }) + }) +}) + +describe('AVM OSS', () => { + describe('SCA configuration in telemetry messages', () => { + let telemetry + let telemetryConfig + let clock + + const HEARTBEAT_INTERVAL = 86410000 + + const suite = [ + { + scaValue: true, + scaValueOrigin: 'env_var', + testDescription: 'should send when env var is true' + }, + { + scaValue: false, + scaValueOrigin: 'env_var', + testDescription: 'should send when env var is false' + }, + { + scaValue: null, + scaValueOrigin: 'default', + testDescription: 'should send null (default) when no env var is set' + } + ] + + suite.forEach(({ scaValue, scaValueOrigin, testDescription }) => { + describe(testDescription, () => { + before((done) => { + clock = sinon.useFakeTimers() + + storage.run({ noop: true }, () => { + traceAgent = http.createServer(async (req, res) => { + const chunks = [] + for await (const chunk of req) { + chunks.push(chunk) + } + req.body = JSON.parse(Buffer.concat(chunks).toString('utf8')) + traceAgent.reqs.push(req) + traceAgent.emit('handled-req') + res.end() + }).listen(0, done) + }) + + traceAgent.reqs = [] + + delete require.cache[require.resolve('../../src/telemetry/send-data')] + delete require.cache[require.resolve('../../src/telemetry')] + telemetry = require('../../src/telemetry') + + telemetryConfig = { + telemetry: { enabled: true, heartbeatInterval: HEARTBEAT_INTERVAL }, + hostname: 'localhost', + port: traceAgent.address().port, + service: 'test service', + version: '1.2.3-beta4', + env: 'preprod', + tags: { + 'runtime-id': '1a2b3c' + }, + appsec: { enabled: false }, + profiling: { enabled: false } + } + }) + + before(() => { + telemetry.updateConfig( + [{ name: 'appsec.sca.enabled', value: scaValue, origin: scaValueOrigin }], + telemetryConfig + ) + telemetry.start(telemetryConfig, { _pluginsByName: {} }) + }) + + after((done) => { + clock.restore() + telemetry.stop() + traceAgent.close(done) + }) + + it('in app-started message', () => { + return testSeq(1, 'app-started', payload => { + expect(payload).to.have.property('configuration').that.deep.equal([ + { name: 'appsec.sca.enabled', value: scaValue, origin: scaValueOrigin } + ]) + }, true) + }) + + it('in app-extended-heartbeat message', () => { + // Skip a full day + clock.tick(86400000) + return testSeq(2, 'app-extended-heartbeat', payload => { + expect(payload).to.have.property('configuration').that.deep.equal([ + { name: 'appsec.sca.enabled', value: scaValue, origin: scaValueOrigin } + ]) + }, true) + }) + }) + }) + }) + + describe('Telemetry and SCA misconfiguration', () => { + let telemetry + + const logSpy = { + warn: sinon.spy() + } + + before(() => { + telemetry = proxyquire('../../src/telemetry', { + '../log': logSpy + }) + }) + + after(() => { telemetry.stop() - done() + sinon.restore() + }) + + it('should log a warning when sca is enabled and telemetry no', () => { + telemetry.start( + { + telemetry: { enabled: false }, + sca: { enabled: true } + } + ) + + expect(logSpy.warn).to.have.been.calledOnceWith('DD_APPSEC_SCA_ENABLED requires enabling telemetry to work.') }) }) }) @@ -307,10 +938,10 @@ async function testSeq (seqId, reqType, validatePayload) { } const req = traceAgent.reqs[seqId - 1] expect(req.method).to.equal('POST') - expect(req.url).to.equal(`/telemetry/proxy/api/v2/apmtelemetry`) + expect(req.url).to.equal('/telemetry/proxy/api/v2/apmtelemetry') expect(req.headers).to.include({ 'content-type': 'application/json', - 'dd-telemetry-api-version': 'v1', + 'dd-telemetry-api-version': 'v2', 'dd-telemetry-request-type': reqType }) const osName = os.type() @@ -336,7 +967,8 @@ async function testSeq (seqId, reqType, validatePayload) { } } expect(req.body).to.deep.include({ - api_version: 'v1', + api_version: 'v2', + naming_schema_version: '', request_type: reqType, runtime_id: '1a2b3c', seq_id: seqId, diff --git a/packages/dd-trace/test/telemetry/logs/index.spec.js b/packages/dd-trace/test/telemetry/logs/index.spec.js index 82a2d380122..f00c8f17655 100644 --- a/packages/dd-trace/test/telemetry/logs/index.spec.js +++ b/packages/dd-trace/test/telemetry/logs/index.spec.js @@ -40,7 +40,7 @@ describe('telemetry logs', () => { logs.start(defaultConfig) - expect(telemetryLog.subscribe).to.have.been.calledOnce + expect(telemetryLog.subscribe).to.have.been.calledTwice }) it('should be subscribe only once', () => { @@ -52,7 +52,7 @@ describe('telemetry logs', () => { logs.start(defaultConfig) logs.start(defaultConfig) - expect(telemetryLog.subscribe).to.have.been.calledOnce + expect(telemetryLog.subscribe).to.have.been.calledTwice }) it('should be disabled and not subscribe if DD_TELEMETRY_LOG_COLLECTION_ENABLED = false', () => { @@ -76,7 +76,7 @@ describe('telemetry logs', () => { logs.stop() - expect(telemetryLog.unsubscribe).to.have.been.calledOnce + expect(telemetryLog.unsubscribe).to.have.been.calledTwice }) }) @@ -84,9 +84,11 @@ describe('telemetry logs', () => { const dc = require('dc-polyfill') let logCollectorAdd let telemetryLog + let errorLog beforeEach(() => { telemetryLog = dc.channel('datadog:telemetry:log') + errorLog = dc.channel('datadog:log:error') logCollectorAdd = sinon.stub() const logs = proxyquire('../../../src/telemetry/logs', { @@ -134,6 +136,32 @@ describe('telemetry logs', () => { expect(logCollectorAdd).to.not.be.called }) + + describe('datadog:log:error', () => { + it('should be called when an Error object is published to datadog:log:error', () => { + const error = new Error('message') + const stack = error.stack + errorLog.publish(error) + + expect(logCollectorAdd).to.be.calledOnceWith(match({ message: 'message', level: 'ERROR', stack_trace: stack })) + }) + + it('should be called when an error string is published to datadog:log:error', () => { + errorLog.publish('custom error message') + + expect(logCollectorAdd).to.be.calledOnceWith(match({ + message: 'custom error message', + level: 'ERROR', + stack_trace: undefined + })) + }) + + it('should not be called when an invalid object is published to datadog:log:error', () => { + errorLog.publish({ invalid: 'field' }) + + expect(logCollectorAdd).not.to.be.called + }) + }) }) describe('send', () => { @@ -165,7 +193,7 @@ describe('telemetry logs', () => { logs.send(defaultConfig, application, host) - expect(sendData).to.be.calledOnceWithExactly(defaultConfig, application, host, 'logs', collectedLogs) + expect(sendData).to.be.calledOnceWithExactly(defaultConfig, application, host, 'logs', { logs: collectedLogs }) }) it('should not drain logCollector and call sendData if not enabled', () => { diff --git a/packages/dd-trace/test/telemetry/logs/log-collector.spec.js b/packages/dd-trace/test/telemetry/logs/log-collector.spec.js index 57f634db2ef..168378a2251 100644 --- a/packages/dd-trace/test/telemetry/logs/log-collector.spec.js +++ b/packages/dd-trace/test/telemetry/logs/log-collector.spec.js @@ -5,6 +5,7 @@ require('../../setup/tap') const { calculateDDBasePath } = require('../../../src/util') const ddBasePath = calculateDDBasePath(__dirname) +const EOL = '\n' describe('telemetry log collector', () => { const logCollector = require('../../../src/telemetry/logs/log-collector') @@ -41,6 +42,46 @@ describe('telemetry log collector', () => { expect(logCollector.add({ message: 'Error 1', level: 'WARN', stack_trace: `stack 1\n${ddFrame}` })).to.be.true expect(logCollector.add({ message: 'Error 1', level: 'DEBUG', stack_trace: `stack 1\n${ddFrame}` })).to.be.true }) + + it('should include original message and dd frames', () => { + const ddFrame = `at T (${ddBasePath}packages/dd-trace/test/telemetry/logs/log_collector.spec.js:29:21)` + const stack = new Error('Error 1') + .stack.replace(`Error 1${EOL}`, `Error 1${EOL}${ddFrame}${EOL}`) + + const ddFrames = stack + .split(EOL) + .filter(line => line.includes(ddBasePath)) + .map(line => line.replace(ddBasePath, '')) + .join(EOL) + + expect(logCollector.add({ message: 'Error 1', level: 'ERROR', stack_trace: stack })).to.be.true + + expect(logCollector.hasEntry({ + message: 'Error 1', + level: 'ERROR', + stack_trace: `Error: Error 1${EOL}${ddFrames}` + })).to.be.true + }) + + it('should not include original message if first frame is not a dd frame', () => { + const thirdPartyFrame = `at callFn (/this/is/not/a/dd/frame/runnable.js:366:21) + at T (${ddBasePath}packages/dd-trace/test/telemetry/logs/log_collector.spec.js:29:21)` + const stack = new Error('Error 1') + .stack.replace(`Error 1${EOL}`, `Error 1${EOL}${thirdPartyFrame}${EOL}`) + + const ddFrames = stack + .split(EOL) + .filter(line => line.includes(ddBasePath)) + .map(line => line.replace(ddBasePath, '')) + .join(EOL) + + expect(logCollector.add({ message: 'Error 1', level: 'ERROR', stack_trace: stack })).to.be.true + expect(logCollector.hasEntry({ + message: 'omitted', + level: 'ERROR', + stack_trace: ddFrames + })).to.be.true + }) }) describe('drain', () => { diff --git a/packages/dd-trace/test/telemetry/metrics.spec.js b/packages/dd-trace/test/telemetry/metrics.spec.js index 1a1f48b1925..e4bac2c24df 100644 --- a/packages/dd-trace/test/telemetry/metrics.spec.js +++ b/packages/dd-trace/test/telemetry/metrics.spec.js @@ -345,6 +345,23 @@ describe('metrics', () => { ]) }) + it('should decrement with explicit arg', () => { + const ns = new metrics.Namespace('tracers') + const metric = ns.count('name') + + metric.inc(3) + + metric.track = sinon.spy(metric.track) + + metric.dec(2) + + expect(metric.track).to.be.calledWith(-2) + + expect(metric.points).to.deep.equal([ + [now / 1e3, 1] + ]) + }) + it('should retain timestamp of first change', () => { const ns = new metrics.Namespace('tracers') const metric = ns.count('name') diff --git a/packages/dd-trace/test/telemetry/send-data.spec.js b/packages/dd-trace/test/telemetry/send-data.spec.js index 0fce52bb3a2..1f9992e0402 100644 --- a/packages/dd-trace/test/telemetry/send-data.spec.js +++ b/packages/dd-trace/test/telemetry/send-data.spec.js @@ -2,6 +2,7 @@ require('../setup/tap') +const { expect } = require('chai') const proxyquire = require('proxyquire') describe('sendData', () => { const application = { @@ -11,6 +12,7 @@ describe('sendData', () => { let sendDataModule let request + beforeEach(() => { request = sinon.stub() sendDataModule = proxyquire('../../src/telemetry/send-data', { @@ -33,7 +35,7 @@ describe('sendData', () => { path: '/telemetry/proxy/api/v2/apmtelemetry', headers: { 'content-type': 'application/json', - 'dd-telemetry-api-version': 'v1', + 'dd-telemetry-api-version': 'v2', 'dd-telemetry-request-type': 'req-type', 'dd-client-library-language': application.language_name, 'dd-client-library-version': application.tracer_version @@ -58,7 +60,7 @@ describe('sendData', () => { path: '/telemetry/proxy/api/v2/apmtelemetry', headers: { 'content-type': 'application/json', - 'dd-telemetry-api-version': 'v1', + 'dd-telemetry-api-version': 'v2', 'dd-telemetry-request-type': 'req-type', 'dd-client-library-language': application.language_name, 'dd-client-library-version': application.tracer_version @@ -84,7 +86,7 @@ describe('sendData', () => { path: '/telemetry/proxy/api/v2/apmtelemetry', headers: { 'content-type': 'application/json', - 'dd-telemetry-api-version': 'v1', + 'dd-telemetry-api-version': 'v2', 'dd-telemetry-request-type': 'req-type', 'dd-telemetry-debug-enabled': 'true', 'dd-client-library-language': application.language_name, @@ -112,13 +114,61 @@ describe('sendData', () => { expect(data.payload).to.deep.equal(trimmedPayload) }) - it('should not destructure a payload with array type', () => { - const arrayPayload = [{ message: 'test' }, { message: 'test2' }] - sendDataModule.sendData({ tags: { 'runtime-id': '123' } }, 'test', 'test', 'req-type', arrayPayload) + it('should send batch request with retryPayload', () => { + const retryObjData = { payload: { foo: 'bar' }, request_type: 'req-type-1' } + const payload = [{ + request_type: 'req-type-2', + payload: { + integrations: [ + { name: 'foo2', enabled: true, auto_enabled: true }, + { name: 'bar2', enabled: false, auto_enabled: true } + ] + } + + }, retryObjData] + + sendDataModule.sendData({ tags: { 'runtime-id': '123' } }, + { language: 'js' }, 'test', 'message-batch', payload) / expect(request).to.have.been.calledOnce + const data = JSON.parse(request.getCall(0).args[0]) + const expectedPayload = [{ + request_type: 'req-type-2', + payload: { + integrations: [ + { name: 'foo2', enabled: true, auto_enabled: true }, + { name: 'bar2', enabled: false, auto_enabled: true } + ] + } + }, { + request_type: 'req-type-1', + payload: { foo: 'bar' } + }] + expect(data.request_type).to.equal('message-batch') + expect(data.payload).to.deep.equal(expectedPayload) + }) - expect(data.payload).to.deep.equal(arrayPayload) + it('should also work in CI Visibility agentless mode', () => { + process.env.DD_CIVISIBILITY_AGENTLESS_ENABLED = 1 + sendDataModule.sendData( + { + isCiVisibility: true, + tags: { 'runtime-id': '123' }, + site: 'datadoghq.eu' + }, + application, + 'test', 'req-type' + ) + + expect(request).to.have.been.calledOnce + const options = request.getCall(0).args[1] + expect(options).to.include({ + method: 'POST', + path: '/api/v2/apmtelemetry' + }) + const { url } = options + expect(url).to.eql(new URL('https://instrumentation-telemetry-intake.datadoghq.eu')) + delete process.env.DD_CIVISIBILITY_AGENTLESS_ENABLED }) }) diff --git a/register.js b/register.js new file mode 100644 index 00000000000..58adc77bd68 --- /dev/null +++ b/register.js @@ -0,0 +1,4 @@ +const { register } = require('node:module') +const { pathToFileURL } = require('node:url') + +register('./loader-hook.mjs', pathToFileURL(__filename)) diff --git a/requirements.json b/requirements.json new file mode 100644 index 00000000000..85fc7c33894 --- /dev/null +++ b/requirements.json @@ -0,0 +1,85 @@ +{ + "$schema": "https://raw.githubusercontent.com/DataDog/auto_inject/refs/heads/main/preload_go/cmd/library_requirements_tester/testdata/requirements_schema.json", + "version": 1, + "native_deps": { + "glibc": [{ + "arch": "arm", + "supported": true, + "description": "From ubuntu xenial (16.04)", + "min": "2.23" + },{ + "arch": "arm64", + "supported": true, + "description": "From centOS 7", + "min": "2.17" + },{ + "arch": "x64", + "supported": true, + "description": "From centOS 7", + "min": "2.17" + },{ + "arch": "x86", + "supported": true, + "description": "From debian jessie (8)", + "min": "2.19" + }], + "musl": [{ + "arch": "arm", + "supported": true, + "description": "From alpine 3.13" + },{ + "arch": "arm64", + "supported": true, + "description": "From alpine 3.13" + },{ + "arch": "x64", + "supported": true, + "description": "From alpine 3.13" + },{ + "arch": "x86", + "supported": true, + "description": "From alpine 3.13" + }] + }, + "deny": [ + { + "id": "npm", + "description": "Ignore the npm CLI", + "os": null, + "cmds": [ + "**/node", + "**/nodejs", + "**/ts-node", + "**/ts-node-*" + ], + "args": [{ "args": ["*/npm-cli.js"], "position": 1}], + "envars": null + }, + { + "id": "yarn", + "description": "Ignore the yarn CLI", + "os": null, + "cmds": [ + "**/node", + "**/nodejs", + "**/ts-node", + "**/ts-node-*" + ], + "args": [{ "args": ["*/yarn.js"], "position": 1}], + "envars": null + }, + { + "id": "pnpm", + "description": "Ignore the pnpm CLI", + "os": null, + "cmds": [ + "**/node", + "**/nodejs", + "**/ts-node", + "**/ts-node-*" + ], + "args": [{ "args": ["*/pnpm.cjs"], "position": 1}], + "envars": null + } + ] +} diff --git a/scripts/check-proposal-labels.js b/scripts/check-proposal-labels.js index 02a5f0bede3..87d89cce777 100644 --- a/scripts/check-proposal-labels.js +++ b/scripts/check-proposal-labels.js @@ -3,14 +3,14 @@ const childProcess = require('child_process') const ORIGIN = 'origin/' -let releaseBranch = process.env['GITHUB_BASE_REF'] // 'origin/v3.x' +let releaseBranch = process.env.GITHUB_BASE_REF // 'origin/v3.x' let releaseVersion = releaseBranch if (releaseBranch.startsWith(ORIGIN)) { releaseVersion = releaseBranch.substring(ORIGIN.length) } else { releaseBranch = ORIGIN + releaseBranch } -let currentBranch = process.env['GITHUB_HEAD_REF'] // 'ugaitz/workflow-to-verify-dont-land-on-v3.x' +let currentBranch = process.env.GITHUB_HEAD_REF // 'ugaitz/workflow-to-verify-dont-land-on-v3.x' if (!currentBranch.startsWith(ORIGIN)) { currentBranch = ORIGIN + currentBranch } @@ -56,11 +56,8 @@ ${withoutExclusionStderr} } const commitsHashesWithoutExclusions = withoutExclusionStdout.split('\n') if (commitsHashesWithExclusions.length !== commitsHashesWithoutExclusions.length) { - const commitsWithInvalidLabels = [] - commitsHashesWithoutExclusions.filter(c1 => { - if (!commitsHashesWithExclusions.some(c2 => c2 === c1)) { - commitsWithInvalidLabels.push(c1) - } + const commitsWithInvalidLabels = commitsHashesWithoutExclusions.filter(c1 => { + return !commitsHashesWithExclusions.some(c2 => c2 === c1) }) console.error('Some excluded label added in the release proposal', commitsWithInvalidLabels) process.exit(1) diff --git a/scripts/get-chrome-driver-download-url.js b/scripts/get-chrome-driver-download-url.js new file mode 100644 index 00000000000..99f98a9079b --- /dev/null +++ b/scripts/get-chrome-driver-download-url.js @@ -0,0 +1,20 @@ +const URL = 'https://googlechromelabs.github.io/chrome-for-testing/known-good-versions-with-downloads.json' + +// Get chrome driver download URL from a given chrome version, provided via CHROME_VERSION env var +async function getChromeDriverDownloadURL (chromePlatform = 'linux64') { + // CHROME_VERSION is the output of google-chrome --version, e.g. "Google Chrome 124.0.6367.60" + const chromeVersion = process.env.CHROME_VERSION + + const majorMinorPatch = chromeVersion.split(' ')[2].split('.').slice(0, 3).join('.').trim() + const res = await fetch(URL) + const json = await res.json() + + const versions = json.versions.filter(({ version }) => version.includes(majorMinorPatch)) + + const latest = versions[versions.length - 1] + + // eslint-disable-next-line + console.log(latest.downloads.chromedriver.find(({ platform }) => platform === chromePlatform).url) +} + +getChromeDriverDownloadURL(process.env.CHROME_PLATFORM) diff --git a/scripts/install_plugin_modules.js b/scripts/install_plugin_modules.js index b76365bda67..682e2d3c5ad 100644 --- a/scripts/install_plugin_modules.js +++ b/scripts/install_plugin_modules.js @@ -12,12 +12,12 @@ const externals = require('../packages/dd-trace/test/plugins/externals') const requirePackageJsonPath = require.resolve('../packages/dd-trace/src/require-package-json') +// Can remove aerospike after removing support for aerospike < 5.2.0 (for Node.js 22, v5.12.1 is required) // Can remove couchbase after removing support for couchbase <= 3.2.0 -const excludeList = os.arch() === 'arm64' ? ['couchbase', 'grpc', 'oracledb'] : [] +const excludeList = os.arch() === 'arm64' ? ['aerospike', 'couchbase', 'grpc', 'oracledb'] : [] const workspaces = new Set() const versionLists = {} const deps = {} -const names = [] const filter = process.env.hasOwnProperty('PLUGINS') && process.env.PLUGINS.split('|') Object.keys(externals).forEach(external => externals[external].forEach(thing => { @@ -29,15 +29,10 @@ Object.keys(externals).forEach(external => externals[external].forEach(thing => } })) -fs.readdirSync(path.join(__dirname, '../packages/datadog-instrumentations/src')) - .filter(file => file.endsWith('js')) - .forEach(file => { - file = file.replace('.js', '') - - if (!filter || filter.includes(file)) { - names.push(file) - } - }) +const names = fs.readdirSync(path.join(__dirname, '..', 'packages', 'datadog-instrumentations', 'src')) + .filter(file => file.endsWith('.js')) + .map(file => file.slice(0, -3)) + .filter(file => !filter || filter.includes(file)) run() @@ -80,12 +75,16 @@ async function assertVersions () { } async function assertInstrumentation (instrumentation, external) { - const versions = process.env.PACKAGE_VERSION_RANGE ? [process.env.PACKAGE_VERSION_RANGE] + const versions = process.env.PACKAGE_VERSION_RANGE && !external + ? [process.env.PACKAGE_VERSION_RANGE] : [].concat(instrumentation.versions || []) for (const version of versions) { if (version) { - await assertModules(instrumentation.name, semver.coerce(version).version, external) + if (version !== '*') { + await assertModules(instrumentation.name, semver.coerce(version).version, external) + } + await assertModules(instrumentation.name, version, external) } } @@ -134,7 +133,7 @@ async function assertPackage (name, version, dependency, external) { if (!external) { if (name === 'aerospike') { pkg.installConfig = { - 'hoistingLimits': 'workspaces' + hoistingLimits: 'workspaces' } } else { pkg.workspaces = { @@ -207,7 +206,11 @@ function assertWorkspace () { } function install () { - exec('yarn --ignore-engines', { cwd: folder() }) + try { + exec('yarn --ignore-engines', { cwd: folder() }) + } catch (e) { // retry in case of server error from registry + exec('yarn --ignore-engines', { cwd: folder() }) + } } function addFolder (name, version) { diff --git a/scripts/prepare-release-proposal.js b/scripts/prepare-release-proposal.js new file mode 100644 index 00000000000..507ac3d4708 --- /dev/null +++ b/scripts/prepare-release-proposal.js @@ -0,0 +1,104 @@ +#!/usr/bin/env node +/* eslint-disable no-console */ +'use strict' + +const semver = require('semver') +const packageJson = require('../package.json') +const path = require('path') +const { execSync } = require('child_process') +const { readFileSync, writeFileSync } = require('fs') + +function helpAndExit () { + console.log('usage: node prepare-release-proposal.js ') + console.log('Actions:') + console.log(' create-branch Create a branch for the release proposal') + console.log(' commit-branch-diffs Commit the branch diffs to the release proposal branch') + console.log(' update-package-json Update the package.json version to the release proposal version') + console.log(' help Show this help message and exit') + process.exit() +} + +function createReleaseBranch (args) { + if (typeof args === 'string') { + const newVersion = semver.inc(packageJson.version, args) + const branchName = `v${newVersion}-proposal` + execSync(`git checkout -b ${branchName}`, { stdio: 'ignore' }) + + console.log(branchName) + return + } + + switch (args[0]) { + case 'minor': + case 'patch': + createReleaseBranch(args[0]) + break + case 'help': + default: + console.log('usage: node prepare-release-proposal.js create-branch ') + console.log('Version types:') + console.log(' minor Create a branch for a minor release proposal') + console.log(' patch Create a branch for a patch release proposal') + break + } +} + +function commitBranchDiffs (args) { + if (args.length !== 1) { + console.log('usage: node prepare-release-proposal.js commit-branch-diffs ') + console.log('release-branches:') + console.log(' v4.x') + console.log(' v5.x') + return + } + const releaseBranch = args[0] + + const excludedLabels = [ + 'semver-major', + `dont-land-on-${releaseBranch}` + ] + + const commandCore = `branch-diff --user DataDog --repo dd-trace-js --exclude-label=${excludedLabels.join(',')}` + + const releaseNotesDraft = execSync(`${commandCore} ${releaseBranch} master`).toString() + + execSync(`${commandCore} --format=sha --reverse ${releaseBranch} master | xargs git cherry-pick`) + + console.log(releaseNotesDraft) +} + +function updatePackageJson (args) { + if (args.length !== 1) { + console.log('usage: node prepare-release-proposal.js update-package-json ') + console.log(' minor') + console.log(' patch') + return + } + + const newVersion = semver.inc(packageJson.version, args[0]) + const packageJsonPath = path.join(__dirname, '..', 'package.json') + + const packageJsonString = readFileSync(packageJsonPath).toString() + .replace(`"version": "${packageJson.version}"`, `"version": "${newVersion}"`) + + writeFileSync(packageJsonPath, packageJsonString) + + console.log(newVersion) +} + +const methodArgs = process.argv.slice(3) +switch (process.argv[2]) { + case 'create-branch': + createReleaseBranch(methodArgs) + break + case 'commit-branch-diffs': + commitBranchDiffs(methodArgs) + break + case 'update-package-json': + updatePackageJson(methodArgs) + break + case 'help': + default: + helpAndExit() + break +} diff --git a/scripts/publish_docs.js b/scripts/publish_docs.js index aa5664bef49..ff1921842ac 100644 --- a/scripts/publish_docs.js +++ b/scripts/publish_docs.js @@ -3,7 +3,7 @@ const exec = require('./helpers/exec') const title = require('./helpers/title') -title(`Publishing API documentation to GitHub Pages`) +title('Publishing API documentation to GitHub Pages') const msg = process.argv[2] @@ -11,7 +11,12 @@ if (!msg) { throw new Error('Please provide a reason for the change. Example: node scripts/publish_docs.js "fix typo"') } -exec('yarn install', { cwd: './docs' }) +try { + exec('yarn install', { cwd: './docs' }) +} catch (e) { // retry in case of error from registry + exec('yarn install', { cwd: './docs' }) +} + exec('rm -rf ./out', { cwd: './docs' }) exec('yarn type:doc') // run first because typedoc requires an empty directory exec('git init', { cwd: './docs/out' }) // cloning would overwrite generated docs diff --git a/scripts/st.js b/scripts/st.js new file mode 100644 index 00000000000..a44eb617e6f --- /dev/null +++ b/scripts/st.js @@ -0,0 +1,105 @@ +#!/usr/bin/env node +/* eslint-disable no-console, no-fallthrough */ +'use strict' + +const path = require('path') +const { writeFileSync } = require('fs') +const { execSync } = require('child_process') + +const ddtracePath = path.join(__dirname, '..') +const defaultTestPath = process.env.DD_ST_PATH || path.join(ddtracePath, '..', 'system-tests') + +const { buildAll, npm, testDir, testArgs } = parseArgs() + +const binariesPath = path.join(testDir, 'binaries') + +if (npm) { + console.log('Using NPM package:', npm) + + writeFileSync(path.join(binariesPath, 'nodejs-load-from-npm'), npm) +} else { + console.log('Using local repo') + + const packName = execSync(`npm pack ${ddtracePath}`, { + cwd: binariesPath, + stdio: [null, null, 'inherit'], + encoding: 'utf8' + }).slice(0, -1) // remove trailing newline + + writeFileSync(path.join(binariesPath, 'nodejs-load-from-npm'), `/binaries/${packName}`) +} + +try { + execSync(`./build.sh ${buildAll ? '' : '-i weblog'} && ./run.sh ${testArgs}`, { + cwd: testDir, + stdio: [null, 'inherit', 'inherit'] + }) +} catch (err) { + process.exit(err.status || 1) +} + +function parseArgs () { + const args = { + buildAll: false, + npm: null, + testDir: defaultTestPath, + testArgs: '' + } + + for (let i = 2; i < process.argv.length; i++) { + switch (process.argv[i]) { + case '-b': + case '--build-all': + args.buildAll = true + break + + case '-h': + case '--help': + helpAndExit() + break + + case '-n': + case '--npm': { + const arg = process.argv[i + 1] + if (!arg || arg[0] === '-') { + args.npm = 'dd-trace' + } else { + args.npm = arg + i++ + } + break + } + + case '-t': + case '--test-dir': { + const arg = process.argv[++i] + if (!arg || arg[0] === '-') helpAndExit() + args.testDir = arg + break + } + + case '--': + args.testArgs = process.argv.slice(i + 1).join(' ') + return args + + default: + console.log('Unknown option:', process.argv[i], '\n') + helpAndExit() + } + } + + return args +} + +function helpAndExit () { + console.log('Usage: node st.js [options...] [-- test-args]') + console.log('Options:') + console.log(' -b, --build-all Rebuild all images (default: only build weblog)') + console.log(' -h, --help Print this message') + console.log(' -n, --npm [package] Build a remote package instead of the local repo (default: "dd-trace")') + console.log(' Can be a package name (e.g. "dd-trace@4.2.0") or a git URL (e.g.') + console.log(' "git+https://github.com/DataDog/dd-trace-js.git#mybranch")') + console.log(' -t, --test-dir Specify the system-tests directory (default: "dd-trace/../system-tests/")') + console.log(' -- Passed to system-tests run.sh (e.g. "-- SCENARIO_NAME tests/path_to_test.py")') + process.exit() +} diff --git a/static-analysis.datadog.yml b/static-analysis.datadog.yml new file mode 100644 index 00000000000..a46fba39176 --- /dev/null +++ b/static-analysis.datadog.yml @@ -0,0 +1,4 @@ +rulesets: + - sit-ci-best-practices: + only: + - ".github/workflows" diff --git a/yarn.lock b/yarn.lock index f0778b55a7e..ea83a1fee4b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,270 +2,320 @@ # yarn lockfile v1 -"@ampproject/remapping@^2.1.0", "@ampproject/remapping@^2.2.0": - version "2.2.0" - resolved "https://registry.yarnpkg.com/@ampproject/remapping/-/remapping-2.2.0.tgz#56c133824780de3174aed5ab6834f3026790154d" - integrity sha512-qRmjj8nj9qmLTQXXmaR1cck3UXSRMPrbsLJAasZpF+t3riI71BXed5ebIOYwQntykeZuhjsdweEc9BxH5Jc26w== +"@ampproject/remapping@^2.2.0": + version "2.2.1" + resolved "https://registry.yarnpkg.com/@ampproject/remapping/-/remapping-2.2.1.tgz#99e8e11851128b8702cd57c33684f1d0f260b630" + integrity sha512-lFMjJTrFL3j7L9yBxwYfCq2k6qqwHyzuUl/XBnif78PWTJYyL/dfowQHWE3sp6U6ZzqWiiIZnpTMO96zhkjwtg== dependencies: - "@jridgewell/gen-mapping" "^0.1.0" + "@jridgewell/gen-mapping" "^0.3.0" "@jridgewell/trace-mapping" "^0.3.9" -"@babel/code-frame@^7.18.6": - version "7.18.6" - resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.18.6.tgz#3b25d38c89600baa2dcc219edfa88a74eb2c427a" - integrity sha512-TDCmlK5eOvH+eH7cdAFlNXeVJqWIQ7gW9tY1GJIpUtFb6CmjVyq2VM3u71bOyR8CRihcCgMUYoDNyLXao3+70Q== +"@apollo/cache-control-types@^1.0.3": + version "1.0.3" + resolved "https://registry.yarnpkg.com/@apollo/cache-control-types/-/cache-control-types-1.0.3.tgz#5da62cf64c3b4419dabfef4536b57a40c8ff0b47" + integrity sha512-F17/vCp7QVwom9eG7ToauIKdAxpSoadsJnqIfyryLFSkLSOEqu+eC5Z3N8OXcUVStuOMcNHlyraRsA6rRICu4g== + +"@apollo/protobufjs@1.2.7": + version "1.2.7" + resolved "https://registry.yarnpkg.com/@apollo/protobufjs/-/protobufjs-1.2.7.tgz#3a8675512817e4a046a897e5f4f16415f16a7d8a" + integrity sha512-Lahx5zntHPZia35myYDBRuF58tlwPskwHc5CWBZC/4bMKB6siTBWwtMrkqXcsNwQiFSzSx5hKdRPUmemrEp3Gg== dependencies: - "@babel/highlight" "^7.18.6" + "@protobufjs/aspromise" "^1.1.2" + "@protobufjs/base64" "^1.1.2" + "@protobufjs/codegen" "^2.0.4" + "@protobufjs/eventemitter" "^1.1.0" + "@protobufjs/fetch" "^1.1.0" + "@protobufjs/float" "^1.0.2" + "@protobufjs/inquire" "^1.1.0" + "@protobufjs/path" "^1.1.2" + "@protobufjs/pool" "^1.1.0" + "@protobufjs/utf8" "^1.1.0" + "@types/long" "^4.0.0" + long "^4.0.0" -"@babel/compat-data@^7.19.3": - version "7.19.3" - resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.19.3.tgz#707b939793f867f5a73b2666e6d9a3396eb03151" - integrity sha512-prBHMK4JYYK+wDjJF1q99KK4JLL+egWS4nmNqdlMUgCExMZ+iZW0hGhyC3VEbsPjvaN0TBhW//VIFwBrk8sEiw== +"@apollo/server-gateway-interface@^1.1.1": + version "1.1.1" + resolved "https://registry.yarnpkg.com/@apollo/server-gateway-interface/-/server-gateway-interface-1.1.1.tgz#a79632aa921edefcd532589943f6b97c96fa4d3c" + integrity sha512-pGwCl/po6+rxRmDMFgozKQo2pbsSwE91TpsDBAOgf74CRDPXHHtM88wbwjab0wMMZh95QfR45GGyDIdhY24bkQ== + dependencies: + "@apollo/usage-reporting-protobuf" "^4.1.1" + "@apollo/utils.fetcher" "^2.0.0" + "@apollo/utils.keyvaluecache" "^2.1.0" + "@apollo/utils.logger" "^2.0.0" + +"@apollo/server@^4.11.0": + version "4.11.0" + resolved "https://registry.yarnpkg.com/@apollo/server/-/server-4.11.0.tgz#21c0f10ad805192a5485e58ed5c5b3dbe2243174" + integrity sha512-SWDvbbs0wl2zYhKG6aGLxwTJ72xpqp0awb2lotNpfezd9VcAvzaUizzKQqocephin2uMoaA8MguoyBmgtPzNWw== + dependencies: + "@apollo/cache-control-types" "^1.0.3" + "@apollo/server-gateway-interface" "^1.1.1" + "@apollo/usage-reporting-protobuf" "^4.1.1" + "@apollo/utils.createhash" "^2.0.0" + "@apollo/utils.fetcher" "^2.0.0" + "@apollo/utils.isnodelike" "^2.0.0" + "@apollo/utils.keyvaluecache" "^2.1.0" + "@apollo/utils.logger" "^2.0.0" + "@apollo/utils.usagereporting" "^2.1.0" + "@apollo/utils.withrequired" "^2.0.0" + "@graphql-tools/schema" "^9.0.0" + "@types/express" "^4.17.13" + "@types/express-serve-static-core" "^4.17.30" + "@types/node-fetch" "^2.6.1" + async-retry "^1.2.1" + cors "^2.8.5" + express "^4.17.1" + loglevel "^1.6.8" + lru-cache "^7.10.1" + negotiator "^0.6.3" + node-abort-controller "^3.1.1" + node-fetch "^2.6.7" + uuid "^9.0.0" + whatwg-mimetype "^3.0.0" + +"@apollo/usage-reporting-protobuf@^4.1.0", "@apollo/usage-reporting-protobuf@^4.1.1": + version "4.1.1" + resolved "https://registry.yarnpkg.com/@apollo/usage-reporting-protobuf/-/usage-reporting-protobuf-4.1.1.tgz#407c3d18c7fbed7a264f3b9a3812620b93499de1" + integrity sha512-u40dIUePHaSKVshcedO7Wp+mPiZsaU6xjv9J+VyxpoU/zL6Jle+9zWeG98tr/+SZ0nZ4OXhrbb8SNr0rAPpIDA== + dependencies: + "@apollo/protobufjs" "1.2.7" -"@babel/compat-data@^7.20.5": - version "7.21.0" - resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.21.0.tgz#c241dc454e5b5917e40d37e525e2f4530c399298" - integrity sha512-gMuZsmsgxk/ENC3O/fRw5QY8A9/uxQbbCEypnLIiYYc/qVJtEV7ouxC3EllIIwNzMqAQee5tanFabWsUOutS7g== +"@apollo/utils.createhash@^2.0.0": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@apollo/utils.createhash/-/utils.createhash-2.0.1.tgz#9d982a166833ce08265ff70f8ef781d65109bdaa" + integrity sha512-fQO4/ZOP8LcXWvMNhKiee+2KuKyqIcfHrICA+M4lj/h/Lh1H10ICcUtk6N/chnEo5HXu0yejg64wshdaiFitJg== + dependencies: + "@apollo/utils.isnodelike" "^2.0.1" + sha.js "^2.4.11" -"@babel/core@^7.5.5": - version "7.21.0" - resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.21.0.tgz#1341aefdcc14ccc7553fcc688dd8986a2daffc13" - integrity sha512-PuxUbxcW6ZYe656yL3EAhpy7qXKq0DmYsrJLpbB8XrsCP9Nm+XCg9XFMb5vIDliPD7+U/+M+QJlH17XOcB7eXA== +"@apollo/utils.dropunuseddefinitions@^2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@apollo/utils.dropunuseddefinitions/-/utils.dropunuseddefinitions-2.0.1.tgz#916cd912cbd88769d3b0eab2d24f4674eeda8124" + integrity sha512-EsPIBqsSt2BwDsv8Wu76LK5R1KtsVkNoO4b0M5aK0hx+dGg9xJXuqlr7Fo34Dl+y83jmzn+UvEW+t1/GP2melA== + +"@apollo/utils.fetcher@^2.0.0": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@apollo/utils.fetcher/-/utils.fetcher-2.0.1.tgz#2f6e3edc8ce79fbe916110d9baaddad7e13d955f" + integrity sha512-jvvon885hEyWXd4H6zpWeN3tl88QcWnHp5gWF5OPF34uhvoR+DFqcNxs9vrRaBBSY3qda3Qe0bdud7tz2zGx1A== + +"@apollo/utils.isnodelike@^2.0.0", "@apollo/utils.isnodelike@^2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@apollo/utils.isnodelike/-/utils.isnodelike-2.0.1.tgz#08a7e50f08d2031122efa25af089d1c6ee609f31" + integrity sha512-w41XyepR+jBEuVpoRM715N2ZD0xMD413UiJx8w5xnAZD2ZkSJnMJBoIzauK83kJpSgNuR6ywbV29jG9NmxjK0Q== + +"@apollo/utils.keyvaluecache@^2.1.0": + version "2.1.1" + resolved "https://registry.yarnpkg.com/@apollo/utils.keyvaluecache/-/utils.keyvaluecache-2.1.1.tgz#f3f79a2f00520c6ab7a77a680a4e1fec4d19e1a6" + integrity sha512-qVo5PvUUMD8oB9oYvq4ViCjYAMWnZ5zZwEjNF37L2m1u528x5mueMlU+Cr1UinupCgdB78g+egA1G98rbJ03Vw== dependencies: - "@ampproject/remapping" "^2.2.0" - "@babel/code-frame" "^7.18.6" - "@babel/generator" "^7.21.0" - "@babel/helper-compilation-targets" "^7.20.7" - "@babel/helper-module-transforms" "^7.21.0" - "@babel/helpers" "^7.21.0" - "@babel/parser" "^7.21.0" - "@babel/template" "^7.20.7" - "@babel/traverse" "^7.21.0" - "@babel/types" "^7.21.0" - convert-source-map "^1.7.0" - debug "^4.1.0" - gensync "^1.0.0-beta.2" - json5 "^2.2.2" - semver "^6.3.0" + "@apollo/utils.logger" "^2.0.1" + lru-cache "^7.14.1" -"@babel/core@^7.7.5": - version "7.19.3" - resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.19.3.tgz#2519f62a51458f43b682d61583c3810e7dcee64c" - integrity sha512-WneDJxdsjEvyKtXKsaBGbDeiyOjR5vYq4HcShxnIbG0qixpoHjI3MqeZM9NDvsojNCEBItQE4juOo/bU6e72gQ== - dependencies: - "@ampproject/remapping" "^2.1.0" - "@babel/code-frame" "^7.18.6" - "@babel/generator" "^7.19.3" - "@babel/helper-compilation-targets" "^7.19.3" - "@babel/helper-module-transforms" "^7.19.0" - "@babel/helpers" "^7.19.0" - "@babel/parser" "^7.19.3" - "@babel/template" "^7.18.10" - "@babel/traverse" "^7.19.3" - "@babel/types" "^7.19.3" - convert-source-map "^1.7.0" - debug "^4.1.0" - gensync "^1.0.0-beta.2" - json5 "^2.2.1" - semver "^6.3.0" +"@apollo/utils.logger@^2.0.0", "@apollo/utils.logger@^2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@apollo/utils.logger/-/utils.logger-2.0.1.tgz#74faeb97d7ad9f22282dfb465bcb2e6873b8a625" + integrity sha512-YuplwLHaHf1oviidB7MxnCXAdHp3IqYV8n0momZ3JfLniae92eYqMIx+j5qJFX6WKJPs6q7bczmV4lXIsTu5Pg== + +"@apollo/utils.printwithreducedwhitespace@^2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@apollo/utils.printwithreducedwhitespace/-/utils.printwithreducedwhitespace-2.0.1.tgz#f4fadea0ae849af2c19c339cc5420d1ddfaa905e" + integrity sha512-9M4LUXV/fQBh8vZWlLvb/HyyhjJ77/I5ZKu+NBWV/BmYGyRmoEP9EVAy7LCVoY3t8BDcyCAGfxJaLFCSuQkPUg== -"@babel/generator@^7.19.3": - version "7.19.3" - resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.19.3.tgz#d7f4d1300485b4547cb6f94b27d10d237b42bf59" - integrity sha512-fqVZnmp1ncvZU757UzDheKZpfPgatqY59XtW2/j/18H7u76akb8xqvjw82f+i2UKd/ksYsSick/BCLQUUtJ/qQ== +"@apollo/utils.removealiases@2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@apollo/utils.removealiases/-/utils.removealiases-2.0.1.tgz#2873c93d72d086c60fc0d77e23d0f75e66a2598f" + integrity sha512-0joRc2HBO4u594Op1nev+mUF6yRnxoUH64xw8x3bX7n8QBDYdeYgY4tF0vJReTy+zdn2xv6fMsquATSgC722FA== + +"@apollo/utils.sortast@^2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@apollo/utils.sortast/-/utils.sortast-2.0.1.tgz#58c90bb8bd24726346b61fa51ba7fcf06e922ef7" + integrity sha512-eciIavsWpJ09za1pn37wpsCGrQNXUhM0TktnZmHwO+Zy9O4fu/WdB4+5BvVhFiZYOXvfjzJUcc+hsIV8RUOtMw== dependencies: - "@babel/types" "^7.19.3" - "@jridgewell/gen-mapping" "^0.3.2" - jsesc "^2.5.1" + lodash.sortby "^4.7.0" + +"@apollo/utils.stripsensitiveliterals@^2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@apollo/utils.stripsensitiveliterals/-/utils.stripsensitiveliterals-2.0.1.tgz#2f3350483be376a98229f90185eaf19888323132" + integrity sha512-QJs7HtzXS/JIPMKWimFnUMK7VjkGQTzqD9bKD1h3iuPAqLsxd0mUNVbkYOPTsDhUKgcvUOfOqOJWYohAKMvcSA== -"@babel/generator@^7.21.0", "@babel/generator@^7.21.1": - version "7.21.1" - resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.21.1.tgz#951cc626057bc0af2c35cd23e9c64d384dea83dd" - integrity sha512-1lT45bAYlQhFn/BHivJs43AiW2rg3/UbLyShGfF3C0KmHvO5fSghWd5kBJy30kpRRucGzXStvnnCFniCR2kXAA== +"@apollo/utils.usagereporting@^2.1.0": + version "2.1.0" + resolved "https://registry.yarnpkg.com/@apollo/utils.usagereporting/-/utils.usagereporting-2.1.0.tgz#11bca6a61fcbc6e6d812004503b38916e74313f4" + integrity sha512-LPSlBrn+S17oBy5eWkrRSGb98sWmnEzo3DPTZgp8IQc8sJe0prDgDuppGq4NeQlpoqEHz0hQeYHAOA0Z3aQsxQ== + dependencies: + "@apollo/usage-reporting-protobuf" "^4.1.0" + "@apollo/utils.dropunuseddefinitions" "^2.0.1" + "@apollo/utils.printwithreducedwhitespace" "^2.0.1" + "@apollo/utils.removealiases" "2.0.1" + "@apollo/utils.sortast" "^2.0.1" + "@apollo/utils.stripsensitiveliterals" "^2.0.1" + +"@apollo/utils.withrequired@^2.0.0": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@apollo/utils.withrequired/-/utils.withrequired-2.0.1.tgz#e72bc512582a6f26af150439f7eb7473b46ba874" + integrity sha512-YBDiuAX9i1lLc6GeTy1m7DGLFn/gMnvXqlalOIMjM7DeOgIacEjjfwPqb0M1CQ2v11HhR15d1NmxJoRCfrNqcA== + +"@babel/code-frame@^7.22.13", "@babel/code-frame@^7.23.5": + version "7.23.5" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.23.5.tgz#9009b69a8c602293476ad598ff53e4562e15c244" + integrity "sha1-kAm2moxgIpNHatWY/1PkVi4VwkQ= sha512-CgH3s1a96LipHCmSUmYFPwY7MNx8C3avkq7i4Wl3cfa662ldtUe4VM1TPXX70pfmrlWTb6jLqTYrZyT2ZTJBgA==" dependencies: - "@babel/types" "^7.21.0" + "@babel/highlight" "^7.23.4" + chalk "^2.4.2" + +"@babel/compat-data@^7.20.5", "@babel/compat-data@^7.22.9": + version "7.23.5" + resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.23.5.tgz#ffb878728bb6bdcb6f4510aa51b1be9afb8cfd98" + integrity "sha1-/7h4cou2vctvRRCqUbG+mvuM/Zg= sha512-uU27kfDRlhfKl+w1U6vp16IuvSLtjAxdArVXPa9BvLkrr7CYIsxH5adpHObeAGY/41+syctUWOZ140a2Rvkgjw==" + +"@babel/core@^7.5.5", "@babel/core@^7.7.5": + version "7.23.5" + resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.23.5.tgz#6e23f2acbcb77ad283c5ed141f824fd9f70101c7" + integrity "sha1-biPyrLy3etKDxe0UH4JP2fcBAcc= sha512-Cwc2XjUrG4ilcfOw4wBAK+enbdgwAcAJCfGUItPBKR7Mjw4aEfAFYrLxeRp4jWgtNIKn3n2AlBOfwwafl+42/g==" + dependencies: + "@ampproject/remapping" "^2.2.0" + "@babel/code-frame" "^7.23.5" + "@babel/generator" "^7.23.5" + "@babel/helper-compilation-targets" "^7.22.15" + "@babel/helper-module-transforms" "^7.23.3" + "@babel/helpers" "^7.23.5" + "@babel/parser" "^7.23.5" + "@babel/template" "^7.22.15" + "@babel/traverse" "^7.23.5" + "@babel/types" "^7.23.5" + convert-source-map "^2.0.0" + debug "^4.1.0" + gensync "^1.0.0-beta.2" + json5 "^2.2.3" + semver "^6.3.1" + +"@babel/generator@^7.23.5": + version "7.23.5" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.23.5.tgz#17d0a1ea6b62f351d281350a5f80b87a810c4755" + integrity "sha1-F9Ch6mti81HSgTUKX4C4eoEMR1U= sha512-BPssCHrBD+0YrxviOa3QzpqwhNIXKEtOa2jQrm4FlmkC2apYgRnQcmPWiGZDlGxiNtltnUFolMe8497Esry+jA==" + dependencies: + "@babel/types" "^7.23.5" "@jridgewell/gen-mapping" "^0.3.2" "@jridgewell/trace-mapping" "^0.3.17" jsesc "^2.5.1" -"@babel/helper-annotate-as-pure@^7.18.6": - version "7.18.6" - resolved "https://registry.yarnpkg.com/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.18.6.tgz#eaa49f6f80d5a33f9a5dd2276e6d6e451be0a6bb" - integrity sha512-duORpUiYrEpzKIop6iNbjnwKLAKnJ47csTyRACyEmWj0QdUrm5aqNJGHSSEQSUAvNW0ojX0dOmK9dZduvkfeXA== +"@babel/helper-annotate-as-pure@^7.22.5": + version "7.22.5" + resolved "https://registry.yarnpkg.com/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.22.5.tgz#e7f06737b197d580a01edf75d97e2c8be99d3882" + integrity sha512-LvBTxu8bQSQkcyKOU+a1btnNFQ1dMAd0R6PyW3arXes06F6QLWLIrd681bxRPIXlrMGR3XYnW9JyML7dP3qgxg== dependencies: - "@babel/types" "^7.18.6" + "@babel/types" "^7.22.5" -"@babel/helper-compilation-targets@^7.19.3": - version "7.19.3" - resolved "https://registry.yarnpkg.com/@babel/helper-compilation-targets/-/helper-compilation-targets-7.19.3.tgz#a10a04588125675d7c7ae299af86fa1b2ee038ca" - integrity sha512-65ESqLGyGmLvgR0mst5AdW1FkNlj9rQsCKduzEoEPhBCDFGXvz2jW6bXFG6i0/MrV2s7hhXjjb2yAzcPuQlLwg== +"@babel/helper-compilation-targets@^7.20.7", "@babel/helper-compilation-targets@^7.22.15": + version "7.22.15" + resolved "https://registry.yarnpkg.com/@babel/helper-compilation-targets/-/helper-compilation-targets-7.22.15.tgz#0698fc44551a26cf29f18d4662d5bf545a6cfc52" + integrity "sha1-Bpj8RFUaJs8p8Y1GYtW/VFps/FI= sha512-y6EEzULok0Qvz8yyLkCvVX+02ic+By2UdOhylwUOvOn9dvYc9mKICJuuU1n1XBI02YWsNsnrY1kc6DVbjcXbtw==" dependencies: - "@babel/compat-data" "^7.19.3" - "@babel/helper-validator-option" "^7.18.6" - browserslist "^4.21.3" - semver "^6.3.0" - -"@babel/helper-compilation-targets@^7.20.7": - version "7.20.7" - resolved "https://registry.yarnpkg.com/@babel/helper-compilation-targets/-/helper-compilation-targets-7.20.7.tgz#a6cd33e93629f5eb473b021aac05df62c4cd09bb" - integrity sha512-4tGORmfQcrc+bvrjb5y3dG9Mx1IOZjsHqQVUz7XCNHO+iTmqxWnVg3KRygjGmpRLJGdQSKuvFinbIb0CnZwHAQ== - dependencies: - "@babel/compat-data" "^7.20.5" - "@babel/helper-validator-option" "^7.18.6" - browserslist "^4.21.3" + "@babel/compat-data" "^7.22.9" + "@babel/helper-validator-option" "^7.22.15" + browserslist "^4.21.9" lru-cache "^5.1.1" - semver "^6.3.0" - -"@babel/helper-environment-visitor@^7.18.9": - version "7.18.9" - resolved "https://registry.yarnpkg.com/@babel/helper-environment-visitor/-/helper-environment-visitor-7.18.9.tgz#0c0cee9b35d2ca190478756865bb3528422f51be" - integrity sha512-3r/aACDJ3fhQ/EVgFy0hpj8oHyHpQc+LPtJoY9SzTThAsStm4Ptegq92vqKoE3vD706ZVFWITnMnxucw+S9Ipg== - -"@babel/helper-function-name@^7.19.0": - version "7.19.0" - resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.19.0.tgz#941574ed5390682e872e52d3f38ce9d1bef4648c" - integrity sha512-WAwHBINyrpqywkUH0nTnNgI5ina5TFn85HKS0pbPDfxFfhyR/aNQEn4hGi1P1JyT//I0t4OgXUlofzWILRvS5w== - dependencies: - "@babel/template" "^7.18.10" - "@babel/types" "^7.19.0" - -"@babel/helper-function-name@^7.21.0": - version "7.21.0" - resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.21.0.tgz#d552829b10ea9f120969304023cd0645fa00b1b4" - integrity sha512-HfK1aMRanKHpxemaY2gqBmL04iAPOPRj7DxtNbiDOrJK+gdwkiNRVpCpUJYbUT+aZyemKN8brqTOxzCaG6ExRg== - dependencies: - "@babel/template" "^7.20.7" - "@babel/types" "^7.21.0" - -"@babel/helper-hoist-variables@^7.18.6": - version "7.18.6" - resolved "https://registry.yarnpkg.com/@babel/helper-hoist-variables/-/helper-hoist-variables-7.18.6.tgz#d4d2c8fb4baeaa5c68b99cc8245c56554f926678" - integrity sha512-UlJQPkFqFULIcyW5sbzgbkxn2FKRgwWiRexcuaR8RNJRy8+LLveqPjwZV/bwrLZCN0eUHD/x8D0heK1ozuoo6Q== - dependencies: - "@babel/types" "^7.18.6" - -"@babel/helper-module-imports@^7.18.6": - version "7.18.6" - resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.18.6.tgz#1e3ebdbbd08aad1437b428c50204db13c5a3ca6e" - integrity sha512-0NFvs3VkuSYbFi1x2Vd6tKrywq+z/cLeYC/RJNFrIX/30Bf5aiGYbtvGXolEktzJH8o5E5KJ3tT+nkxuuZFVlA== - dependencies: - "@babel/types" "^7.18.6" - -"@babel/helper-module-transforms@^7.19.0": - version "7.19.0" - resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.19.0.tgz#309b230f04e22c58c6a2c0c0c7e50b216d350c30" - integrity sha512-3HBZ377Fe14RbLIA+ac3sY4PTgpxHVkFrESaWhoI5PuyXPBBX8+C34qblV9G89ZtycGJCmCI/Ut+VUDK4bltNQ== - dependencies: - "@babel/helper-environment-visitor" "^7.18.9" - "@babel/helper-module-imports" "^7.18.6" - "@babel/helper-simple-access" "^7.18.6" - "@babel/helper-split-export-declaration" "^7.18.6" - "@babel/helper-validator-identifier" "^7.18.6" - "@babel/template" "^7.18.10" - "@babel/traverse" "^7.19.0" - "@babel/types" "^7.19.0" - -"@babel/helper-module-transforms@^7.21.0": - version "7.21.2" - resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.21.2.tgz#160caafa4978ac8c00ac66636cb0fa37b024e2d2" - integrity sha512-79yj2AR4U/Oqq/WOV7Lx6hUjau1Zfo4cI+JLAVYeMV5XIlbOhmjEk5ulbTc9fMpmlojzZHkUUxAiK+UKn+hNQQ== - dependencies: - "@babel/helper-environment-visitor" "^7.18.9" - "@babel/helper-module-imports" "^7.18.6" - "@babel/helper-simple-access" "^7.20.2" - "@babel/helper-split-export-declaration" "^7.18.6" - "@babel/helper-validator-identifier" "^7.19.1" - "@babel/template" "^7.20.7" - "@babel/traverse" "^7.21.2" - "@babel/types" "^7.21.2" - -"@babel/helper-plugin-utils@^7.18.6", "@babel/helper-plugin-utils@^7.20.2", "@babel/helper-plugin-utils@^7.8.0": - version "7.20.2" - resolved "https://registry.yarnpkg.com/@babel/helper-plugin-utils/-/helper-plugin-utils-7.20.2.tgz#d1b9000752b18d0877cff85a5c376ce5c3121629" - integrity sha512-8RvlJG2mj4huQ4pZ+rU9lqKi9ZKiRmuvGuM2HlWmkmgOhbs6zEAw6IEiJ5cQqGbDzGZOhwuOQNtZMi/ENLjZoQ== - -"@babel/helper-simple-access@^7.18.6": - version "7.18.6" - resolved "https://registry.yarnpkg.com/@babel/helper-simple-access/-/helper-simple-access-7.18.6.tgz#d6d8f51f4ac2978068df934b569f08f29788c7ea" - integrity sha512-iNpIgTgyAvDQpDj76POqg+YEt8fPxx3yaNBg3S30dxNKm2SWfYhD0TGrK/Eu9wHpUW63VQU894TsTg+GLbUa1g== - dependencies: - "@babel/types" "^7.18.6" - -"@babel/helper-simple-access@^7.20.2": - version "7.20.2" - resolved "https://registry.yarnpkg.com/@babel/helper-simple-access/-/helper-simple-access-7.20.2.tgz#0ab452687fe0c2cfb1e2b9e0015de07fc2d62dd9" - integrity sha512-+0woI/WPq59IrqDYbVGfshjT5Dmk/nnbdpcF8SnMhhXObpTq2KNBdLFRFrkVdbDOyUmHBCxzm5FHV1rACIkIbA== - dependencies: - "@babel/types" "^7.20.2" - -"@babel/helper-split-export-declaration@^7.18.6": - version "7.18.6" - resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.18.6.tgz#7367949bc75b20c6d5a5d4a97bba2824ae8ef075" - integrity sha512-bde1etTx6ZyTmobl9LLMMQsaizFVZrquTEHOqKeQESMKo4PlObf+8+JA25ZsIpZhT/WEd39+vOdLXAFG/nELpA== - dependencies: - "@babel/types" "^7.18.6" - -"@babel/helper-string-parser@^7.18.10": - version "7.18.10" - resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.18.10.tgz#181f22d28ebe1b3857fa575f5c290b1aaf659b56" - integrity sha512-XtIfWmeNY3i4t7t4D2t02q50HvqHybPqW2ki1kosnvWCwuCMeo81Jf0gwr85jy/neUdg5XDdeFE/80DXiO+njw== - -"@babel/helper-string-parser@^7.19.4": - version "7.19.4" - resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.19.4.tgz#38d3acb654b4701a9b77fb0615a96f775c3a9e63" - integrity sha512-nHtDoQcuqFmwYNYPz3Rah5ph2p8PFeFCsZk9A/48dPc/rGocJ5J3hAAZ7pb76VWX3fZKu+uEr/FhH5jLx7umrw== - -"@babel/helper-validator-identifier@^7.18.6", "@babel/helper-validator-identifier@^7.19.1": - version "7.19.1" - resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.19.1.tgz#7eea834cf32901ffdc1a7ee555e2f9c27e249ca2" - integrity sha512-awrNfaMtnHUr653GgGEs++LlAvW6w+DcPrOliSMXWCKo597CwL5Acf/wWdNkf/tfEQE3mjkeD1YOVZOUV/od1w== - -"@babel/helper-validator-option@^7.18.6": - version "7.18.6" - resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.18.6.tgz#bf0d2b5a509b1f336099e4ff36e1a63aa5db4db8" - integrity sha512-XO7gESt5ouv/LRJdrVjkShckw6STTaB7l9BrpBaAHDeF5YZT+01PCwmR0SJHnkW6i8OwW/EVWRShfi4j2x+KQw== - -"@babel/helpers@^7.19.0": - version "7.19.0" - resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.19.0.tgz#f30534657faf246ae96551d88dd31e9d1fa1fc18" - integrity sha512-DRBCKGwIEdqY3+rPJgG/dKfQy9+08rHIAJx8q2p+HSWP87s2HCrQmaAMMyMll2kIXKCW0cO1RdQskx15Xakftg== - dependencies: - "@babel/template" "^7.18.10" - "@babel/traverse" "^7.19.0" - "@babel/types" "^7.19.0" - -"@babel/helpers@^7.21.0": - version "7.21.0" - resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.21.0.tgz#9dd184fb5599862037917cdc9eecb84577dc4e7e" - integrity sha512-XXve0CBtOW0pd7MRzzmoyuSj0e3SEzj8pgyFxnTT1NJZL38BD1MK7yYrm8yefRPIDvNNe14xR4FdbHwpInD4rA== - dependencies: - "@babel/template" "^7.20.7" - "@babel/traverse" "^7.21.0" - "@babel/types" "^7.21.0" - -"@babel/highlight@^7.18.6": - version "7.18.6" - resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.18.6.tgz#81158601e93e2563795adcbfbdf5d64be3f2ecdf" - integrity sha512-u7stbOuYjaPezCuLj29hNW1v64M2Md2qupEKP1fHc7WdOA3DgLh37suiSrZYY7haUB7iBeQZ9P1uiRF359do3g== - dependencies: - "@babel/helper-validator-identifier" "^7.18.6" - chalk "^2.0.0" + semver "^6.3.1" + +"@babel/helper-environment-visitor@^7.22.20": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.20.tgz#96159db61d34a29dba454c959f5ae4a649ba9167" + integrity "sha1-lhWdth00op26RUyVn1rkpkm6kWc= sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA==" + +"@babel/helper-function-name@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.23.0.tgz#1f9a3cdbd5b2698a670c30d2735f9af95ed52759" + integrity "sha1-H5o829WyaYpnDDDSc1+a+V7VJ1k= sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw==" + dependencies: + "@babel/template" "^7.22.15" + "@babel/types" "^7.23.0" + +"@babel/helper-hoist-variables@^7.22.5": + version "7.22.5" + resolved "https://registry.yarnpkg.com/@babel/helper-hoist-variables/-/helper-hoist-variables-7.22.5.tgz#c01a007dac05c085914e8fb652b339db50d823bb" + integrity sha512-wGjk9QZVzvknA6yKIUURb8zY3grXCcOZt+/7Wcy8O2uctxhplmUPkOdlgoNhmdVee2c92JXbf1xpMtVNbfoxRw== + dependencies: + "@babel/types" "^7.22.5" + +"@babel/helper-module-imports@^7.22.15": + version "7.22.15" + resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.22.15.tgz#16146307acdc40cc00c3b2c647713076464bdbf0" + integrity "sha1-FhRjB6zcQMwAw7LGR3EwdkZL2/A= sha512-0pYVBnDKZO2fnSPCrgM/6WMc7eS20Fbok+0r88fp+YtWVLZrp4CkafFGIp+W0VKw4a22sgebPT99y+FDNMdP4w==" + dependencies: + "@babel/types" "^7.22.15" + +"@babel/helper-module-transforms@^7.23.3": + version "7.23.3" + resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.23.3.tgz#d7d12c3c5d30af5b3c0fcab2a6d5217773e2d0f1" + integrity "sha1-19EsPF0wr1s8D8qyptUhd3Pi0PE= sha512-7bBs4ED9OmswdfDzpz4MpWgSrV7FXlc3zIagvLFjS5H+Mk7Snr21vQ6QwrsoCGMfNC4e4LQPdoULEt4ykz0SRQ==" + dependencies: + "@babel/helper-environment-visitor" "^7.22.20" + "@babel/helper-module-imports" "^7.22.15" + "@babel/helper-simple-access" "^7.22.5" + "@babel/helper-split-export-declaration" "^7.22.6" + "@babel/helper-validator-identifier" "^7.22.20" + +"@babel/helper-plugin-utils@^7.20.2", "@babel/helper-plugin-utils@^7.22.5", "@babel/helper-plugin-utils@^7.8.0": + version "7.22.5" + resolved "https://registry.yarnpkg.com/@babel/helper-plugin-utils/-/helper-plugin-utils-7.22.5.tgz#dd7ee3735e8a313b9f7b05a773d892e88e6d7295" + integrity sha512-uLls06UVKgFG9QD4OeFYLEGteMIAa5kpTPcFL28yuCIIzsf6ZyKZMllKVOCZFhiZ5ptnwX4mtKdWCBE/uT4amg== + +"@babel/helper-simple-access@^7.22.5": + version "7.22.5" + resolved "https://registry.yarnpkg.com/@babel/helper-simple-access/-/helper-simple-access-7.22.5.tgz#4938357dc7d782b80ed6dbb03a0fba3d22b1d5de" + integrity sha512-n0H99E/K+Bika3++WNL17POvo4rKWZ7lZEp1Q+fStVbUi8nxPQEBOlTmCOxW/0JsS56SKKQ+ojAe2pHKJHN35w== + dependencies: + "@babel/types" "^7.22.5" + +"@babel/helper-split-export-declaration@^7.22.6": + version "7.22.6" + resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.22.6.tgz#322c61b7310c0997fe4c323955667f18fcefb91c" + integrity sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g== + dependencies: + "@babel/types" "^7.22.5" + +"@babel/helper-string-parser@^7.23.4": + version "7.23.4" + resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.23.4.tgz#9478c707febcbbe1ddb38a3d91a2e054ae622d83" + integrity "sha1-lHjHB/68u+Hds4o9kaLgVK5iLYM= sha512-803gmbQdqwdf4olxrX4AJyFBV/RTr3rSmOj0rKwesmzlfhYNDEs+/iOcznzpNWlJlIlTJC2QfPFcHB6DlzdVLQ==" + +"@babel/helper-validator-identifier@^7.22.20": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz#c4ae002c61d2879e724581d96665583dbc1dc0e0" + integrity "sha1-xK4ALGHSh55yRYHZZmVYPbwdwOA= sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A==" + +"@babel/helper-validator-option@^7.22.15": + version "7.23.5" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.23.5.tgz#907a3fbd4523426285365d1206c423c4c5520307" + integrity "sha1-kHo/vUUjQmKFNl0SBsQjxMVSAwc= sha512-85ttAOMLsr53VgXkTbkx8oA6YTfT4q7/HzXSLEYmjcSTJPMPQtvq1BD79Byep5xMUYbGRzEpDsjUf3dyp54IKw==" + +"@babel/helpers@^7.23.5": + version "7.23.5" + resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.23.5.tgz#52f522840df8f1a848d06ea6a79b79eefa72401e" + integrity "sha1-UvUihA348ahI0G6mp5t57vpyQB4= sha512-oO7us8FzTEsG3U6ag9MfdF1iA/7Z6dz+MtFhifZk8C8o453rGJFFWUP1t+ULM9TUIAzC9uxXEiXjOiVMyd7QPg==" + dependencies: + "@babel/template" "^7.22.15" + "@babel/traverse" "^7.23.5" + "@babel/types" "^7.23.5" + +"@babel/highlight@^7.23.4": + version "7.23.4" + resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.23.4.tgz#edaadf4d8232e1a961432db785091207ead0621b" + integrity "sha1-7arfTYIy4alhQy23hQkSB+rQYhs= sha512-acGdbYSfp2WheJoJm/EBBBLh/ID8KDc64ISZ9DYtBmC8/Q204PZJLHyzeB5qMzJ5trcOkybd78M4x2KWsUq++A==" + dependencies: + "@babel/helper-validator-identifier" "^7.22.20" + chalk "^2.4.2" js-tokens "^4.0.0" -"@babel/parser@^7.18.10", "@babel/parser@^7.19.3": - version "7.19.3" - resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.19.3.tgz#8dd36d17c53ff347f9e55c328710321b49479a9a" - integrity sha512-pJ9xOlNWHiy9+FuFP09DEAFbAn4JskgRsVcc169w2xRBC3FRGuQEwjeIMMND9L2zc0iEhO/tGv4Zq+km+hxNpQ== - -"@babel/parser@^7.20.7", "@babel/parser@^7.21.0", "@babel/parser@^7.21.2": - version "7.21.2" - resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.21.2.tgz#dacafadfc6d7654c3051a66d6fe55b6cb2f2a0b3" - integrity sha512-URpaIJQwEkEC2T9Kn+Ai6Xe/02iNaVCuT/PtoRz3GPVJVDpPd7mLo+VddTbhCRU9TXqW5mSrQfXZyi8kDKOVpQ== +"@babel/parser@^7.22.15", "@babel/parser@^7.23.5": + version "7.23.5" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.23.5.tgz#37dee97c4752af148e1d38c34b856b2507660563" + integrity "sha1-N97pfEdSrxSOHTjDS4VrJQdmBWM= sha512-hOOqoiNXrmGdFbhgCzu6GiURxUgM27Xwd/aPuu8RfHEZPBzL1Z54okAHAQjXfcQNwvrlkAmAp4SlRTZ45vlthQ==" "@babel/plugin-proposal-object-rest-spread@^7.5.5": version "7.20.7" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-object-rest-spread/-/plugin-proposal-object-rest-spread-7.20.7.tgz#aa662940ef425779c75534a5c41e9d936edc390a" - integrity sha512-d2S98yCiLxDVmBmE8UjGcfPvNEUbA1U5q5WxaWFUGRzJSVAZqm5W6MbPct0jxnegUZ0niLeNX+IOzEs7wYg9Dg== + resolved "https://registry.npmjs.org/@babel/plugin-proposal-object-rest-spread/-/plugin-proposal-object-rest-spread-7.20.7.tgz" + integrity "sha1-qmYpQO9CV3nHVTSlxB6dk27cOQo= sha512-d2S98yCiLxDVmBmE8UjGcfPvNEUbA1U5q5WxaWFUGRzJSVAZqm5W6MbPct0jxnegUZ0niLeNX+IOzEs7wYg9Dg==" dependencies: "@babel/compat-data" "^7.20.5" "@babel/helper-compilation-targets" "^7.20.7" @@ -273,162 +323,128 @@ "@babel/plugin-syntax-object-rest-spread" "^7.8.3" "@babel/plugin-transform-parameters" "^7.20.7" -"@babel/plugin-syntax-jsx@^7.18.6": - version "7.18.6" - resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.18.6.tgz#a8feef63b010150abd97f1649ec296e849943ca0" - integrity sha512-6mmljtAedFGTWu2p/8WIORGwy+61PLgOMPOdazc7YoJ9ZCWUyFy3A6CpPkRKLKD1ToAesxX8KGEViAiLo9N+7Q== +"@babel/plugin-syntax-jsx@^7.23.3": + version "7.23.3" + resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.23.3.tgz#8f2e4f8a9b5f9aa16067e142c1ac9cd9f810f473" + integrity "sha1-jy5PiptfmqFgZ+FCwayc2fgQ9HM= sha512-EB2MELswq55OHUoRZLGg/zC7QWUKfNLpE57m/S2yr1uEneIgsTgrSzXP3NXEsMkVn76OlaVVnzN+ugObuYGwhg==" dependencies: - "@babel/helper-plugin-utils" "^7.18.6" + "@babel/helper-plugin-utils" "^7.22.5" "@babel/plugin-syntax-object-rest-spread@^7.8.3": version "7.8.3" - resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz#60e225edcbd98a640332a2e72dd3e66f1af55871" + resolved "https://registry.npmjs.org/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz" integrity sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA== dependencies: "@babel/helper-plugin-utils" "^7.8.0" "@babel/plugin-transform-destructuring@^7.5.0": - version "7.20.7" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-destructuring/-/plugin-transform-destructuring-7.20.7.tgz#8bda578f71620c7de7c93af590154ba331415454" - integrity sha512-Xwg403sRrZb81IVB79ZPqNQME23yhugYVqgTxAhT99h485F4f+GMELFhhOsscDUB7HCswepKeCKLn/GZvUKoBA== + version "7.23.3" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-destructuring/-/plugin-transform-destructuring-7.23.3.tgz#8c9ee68228b12ae3dff986e56ed1ba4f3c446311" + integrity "sha1-jJ7mgiixKuPf+YblbtG6TzxEYxE= sha512-n225npDqjDIr967cMScVKHXJs7rout1q+tt50inyBCPkyZ8KxeI6d+GIbSBTT/w/9WdlWDOej3V9HE5Lgk57gw==" dependencies: - "@babel/helper-plugin-utils" "^7.20.2" + "@babel/helper-plugin-utils" "^7.22.5" "@babel/plugin-transform-parameters@^7.20.7": - version "7.20.7" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-parameters/-/plugin-transform-parameters-7.20.7.tgz#0ee349e9d1bc96e78e3b37a7af423a4078a7083f" - integrity sha512-WiWBIkeHKVOSYPO0pWkxGPfKeWrCJyD3NJ53+Lrp/QMSZbsVPovrVl2aWZ19D/LTVnaDv5Ap7GJ/B2CTOZdrfA== + version "7.23.3" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-parameters/-/plugin-transform-parameters-7.23.3.tgz#83ef5d1baf4b1072fa6e54b2b0999a7b2527e2af" + integrity "sha1-g+9dG69LEHL6blSysJmaeyUn4q8= sha512-09lMt6UsUb3/34BbECKVbVwrT9bO6lILWln237z7sLaWnMsTi7Yc9fhX5DLpkJzAGfaReXI22wP41SZmnAA3Vw==" dependencies: - "@babel/helper-plugin-utils" "^7.20.2" + "@babel/helper-plugin-utils" "^7.22.5" "@babel/plugin-transform-react-jsx@^7.3.0": - version "7.21.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-jsx/-/plugin-transform-react-jsx-7.21.0.tgz#656b42c2fdea0a6d8762075d58ef9d4e3c4ab8a2" - integrity sha512-6OAWljMvQrZjR2DaNhVfRz6dkCAVV+ymcLUmaf8bccGOHn2v5rHJK3tTpij0BuhdYWP4LLaqj5lwcdlpAAPuvg== - dependencies: - "@babel/helper-annotate-as-pure" "^7.18.6" - "@babel/helper-module-imports" "^7.18.6" - "@babel/helper-plugin-utils" "^7.20.2" - "@babel/plugin-syntax-jsx" "^7.18.6" - "@babel/types" "^7.21.0" - -"@babel/template@^7.18.10": - version "7.18.10" - resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.18.10.tgz#6f9134835970d1dbf0835c0d100c9f38de0c5e71" - integrity sha512-TI+rCtooWHr3QJ27kJxfjutghu44DLnasDMwpDqCXVTal9RLp3RSYNh4NdBrRP2cQAoG9A8juOQl6P6oZG4JxA== - dependencies: - "@babel/code-frame" "^7.18.6" - "@babel/parser" "^7.18.10" - "@babel/types" "^7.18.10" - -"@babel/template@^7.20.7": - version "7.20.7" - resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.20.7.tgz#a15090c2839a83b02aa996c0b4994005841fd5a8" - integrity sha512-8SegXApWe6VoNw0r9JHpSteLKTpTiLZ4rMlGIm9JQ18KiCtyQiAMEazujAHrUS5flrcqYZa75ukev3P6QmUwUw== - dependencies: - "@babel/code-frame" "^7.18.6" - "@babel/parser" "^7.20.7" - "@babel/types" "^7.20.7" - -"@babel/traverse@^7.19.0", "@babel/traverse@^7.19.3": - version "7.19.3" - resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.19.3.tgz#3a3c5348d4988ba60884e8494b0592b2f15a04b4" - integrity sha512-qh5yf6149zhq2sgIXmwjnsvmnNQC2iw70UFjp4olxucKrWd/dvlUsBI88VSLUsnMNF7/vnOiA+nk1+yLoCqROQ== - dependencies: - "@babel/code-frame" "^7.18.6" - "@babel/generator" "^7.19.3" - "@babel/helper-environment-visitor" "^7.18.9" - "@babel/helper-function-name" "^7.19.0" - "@babel/helper-hoist-variables" "^7.18.6" - "@babel/helper-split-export-declaration" "^7.18.6" - "@babel/parser" "^7.19.3" - "@babel/types" "^7.19.3" - debug "^4.1.0" - globals "^11.1.0" - -"@babel/traverse@^7.21.0", "@babel/traverse@^7.21.2": - version "7.21.2" - resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.21.2.tgz#ac7e1f27658750892e815e60ae90f382a46d8e75" - integrity sha512-ts5FFU/dSUPS13tv8XiEObDu9K+iagEKME9kAbaP7r0Y9KtZJZ+NGndDvWoRAYNpeWafbpFeki3q9QoMD6gxyw== - dependencies: - "@babel/code-frame" "^7.18.6" - "@babel/generator" "^7.21.1" - "@babel/helper-environment-visitor" "^7.18.9" - "@babel/helper-function-name" "^7.21.0" - "@babel/helper-hoist-variables" "^7.18.6" - "@babel/helper-split-export-declaration" "^7.18.6" - "@babel/parser" "^7.21.2" - "@babel/types" "^7.21.2" + version "7.23.4" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-jsx/-/plugin-transform-react-jsx-7.23.4.tgz#393f99185110cea87184ea47bcb4a7b0c2e39312" + integrity "sha1-OT+ZGFEQzqhxhOpHvLSnsMLjkxI= sha512-5xOpoPguCZCRbo/JeHlloSkTA8Bld1J/E1/kLfD1nsuiW1m8tduTA1ERCgIZokDflX/IBzKcqR3l7VlRgiIfHA==" + dependencies: + "@babel/helper-annotate-as-pure" "^7.22.5" + "@babel/helper-module-imports" "^7.22.15" + "@babel/helper-plugin-utils" "^7.22.5" + "@babel/plugin-syntax-jsx" "^7.23.3" + "@babel/types" "^7.23.4" + +"@babel/template@^7.22.15": + version "7.22.15" + resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.22.15.tgz#09576efc3830f0430f4548ef971dde1350ef2f38" + integrity "sha1-CVdu/Dgw8EMPRUjvlx3eE1DvLzg= sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w==" + dependencies: + "@babel/code-frame" "^7.22.13" + "@babel/parser" "^7.22.15" + "@babel/types" "^7.22.15" + +"@babel/traverse@^7.23.5": + version "7.23.5" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.23.5.tgz#f546bf9aba9ef2b042c0e00d245990c15508e7ec" + integrity "sha1-9Ua/mrqe8rBCwOANJFmQwVUI5+w= sha512-czx7Xy5a6sapWWRx61m1Ke1Ra4vczu1mCTtJam5zRTBOonfdJ+S/B6HYmGYu3fJtr8GGET3si6IhgWVBhJ/m8w==" + dependencies: + "@babel/code-frame" "^7.23.5" + "@babel/generator" "^7.23.5" + "@babel/helper-environment-visitor" "^7.22.20" + "@babel/helper-function-name" "^7.23.0" + "@babel/helper-hoist-variables" "^7.22.5" + "@babel/helper-split-export-declaration" "^7.22.6" + "@babel/parser" "^7.23.5" + "@babel/types" "^7.23.5" debug "^4.1.0" globals "^11.1.0" -"@babel/types@^7.18.10", "@babel/types@^7.18.6", "@babel/types@^7.19.0", "@babel/types@^7.19.3": - version "7.19.3" - resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.19.3.tgz#fc420e6bbe54880bce6779ffaf315f5e43ec9624" - integrity sha512-hGCaQzIY22DJlDh9CH7NOxgKkFjBk0Cw9xDO1Xmh2151ti7wiGfQ3LauXzL4HP1fmFlTX6XjpRETTpUcv7wQLw== +"@babel/types@^7.22.15", "@babel/types@^7.22.5", "@babel/types@^7.23.0", "@babel/types@^7.23.4", "@babel/types@^7.23.5": + version "7.23.5" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.23.5.tgz#48d730a00c95109fa4393352705954d74fb5b602" + integrity "sha1-SNcwoAyVEJ+kOTNScFlU10+1tgI= sha512-ON5kSOJwVO6xXVRTvOI0eOnWe7VdUcIpsovGo9U/Br4Ie4UVFQTboO2cYnDhAGU6Fp+UxSiT+pMft0SMHfuq6w==" dependencies: - "@babel/helper-string-parser" "^7.18.10" - "@babel/helper-validator-identifier" "^7.19.1" - to-fast-properties "^2.0.0" - -"@babel/types@^7.20.2", "@babel/types@^7.20.7", "@babel/types@^7.21.0", "@babel/types@^7.21.2": - version "7.21.2" - resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.21.2.tgz#92246f6e00f91755893c2876ad653db70c8310d1" - integrity sha512-3wRZSs7jiFaB8AjxiiD+VqN5DTG2iRvJGQ+qYFrs/654lg6kGTQWIOFjlBo5RaXuAZjBmP3+OQH4dmhqiiyYxw== - dependencies: - "@babel/helper-string-parser" "^7.19.4" - "@babel/helper-validator-identifier" "^7.19.1" + "@babel/helper-string-parser" "^7.23.4" + "@babel/helper-validator-identifier" "^7.22.20" to-fast-properties "^2.0.0" "@colors/colors@1.5.0": version "1.5.0" - resolved "https://registry.yarnpkg.com/@colors/colors/-/colors-1.5.0.tgz#bb504579c1cae923e6576a4f5da43d25f97bdbd9" - integrity sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ== + resolved "https://registry.npmjs.org/@colors/colors/-/colors-1.5.0.tgz" + integrity "sha1-u1BFecHK6SPmV2pPXaQ9Jfl729k= sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==" -"@datadog/native-appsec@5.0.0": - version "5.0.0" - resolved "https://registry.yarnpkg.com/@datadog/native-appsec/-/native-appsec-5.0.0.tgz#e42e77f42062532ad7defa3a79090dc8b020c22b" - integrity sha512-Ks8a4L49N40w+TJjj2e9ncGssUIEjo4wnmUFjPBRvlLGuVj1VJLxCx7ztpd8eTycM5QQlzggCDOP6CMEVmeZbA== +"@datadog/native-appsec@8.2.1": + version "8.2.1" + resolved "https://registry.yarnpkg.com/@datadog/native-appsec/-/native-appsec-8.2.1.tgz#e84f9ec7e5dddea2531970117744264a685da15a" + integrity sha512-PnSlb4DC+EngEfXvZLYVBUueMnxxQV0dTpwbRQmyC6rcIFBzBCPxUl6O0hZaxCNmT1dgllpif+P1efrSi85e0Q== dependencies: node-gyp-build "^3.9.0" -"@datadog/native-iast-rewriter@2.2.1": - version "2.2.1" - resolved "https://registry.yarnpkg.com/@datadog/native-iast-rewriter/-/native-iast-rewriter-2.2.1.tgz#3c74c5a8caa0b876e091e9c5a95256add0d73e1c" - integrity sha512-DyZlE8gNa5AoOFNKGRJU4RYF/Y/tJzv4bIAMuVBbEnMA0xhiIYqpYQG8T3OKkALl3VSEeBMjYwuOR2fCrJ6gzA== +"@datadog/native-iast-rewriter@2.5.0": + version "2.5.0" + resolved "https://registry.yarnpkg.com/@datadog/native-iast-rewriter/-/native-iast-rewriter-2.5.0.tgz#b613defe86e78168f750d1f1662d4ffb3cf002e6" + integrity sha512-WRu34A3Wwp6oafX8KWNAbedtDaaJO+nzfYQht7pcJKjyC2ggfPeF7SoP+eDo9wTn4/nQwEOscSR4hkJqTRlpXQ== dependencies: lru-cache "^7.14.0" node-gyp-build "^4.5.0" -"@datadog/native-iast-taint-tracking@1.6.4": - version "1.6.4" - resolved "https://registry.yarnpkg.com/@datadog/native-iast-taint-tracking/-/native-iast-taint-tracking-1.6.4.tgz#16c21ad7c36a53420c0d3c5a3720731809cc7e98" - integrity sha512-Owxk7hQ4Dxwv4zJAoMjRga0IvE6lhvxnNc8pJCHsemCWBXchjr/9bqg05Zy5JnMbKUWn4XuZeJD6RFZpRa8bfw== +"@datadog/native-iast-taint-tracking@3.2.0": + version "3.2.0" + resolved "https://registry.yarnpkg.com/@datadog/native-iast-taint-tracking/-/native-iast-taint-tracking-3.2.0.tgz#9fb6823d82f934e12c06ea1baa7399ca80deb2ec" + integrity sha512-Mc6FzCoyvU5yXLMsMS9yKnEqJMWoImAukJXolNWCTm+JQYCMf2yMsJ8pBAm7KyZKliamM9rCn7h7Tr2H3lXwjA== dependencies: node-gyp-build "^3.9.0" "@datadog/native-metrics@^2.0.0": version "2.0.0" - resolved "https://registry.yarnpkg.com/@datadog/native-metrics/-/native-metrics-2.0.0.tgz#65bf03313ee419956361e097551db36173e85712" + resolved "https://registry.npmjs.org/@datadog/native-metrics/-/native-metrics-2.0.0.tgz" integrity sha512-YklGVwUtmKGYqFf1MNZuOHvTYdKuR4+Af1XkWcMD8BwOAjxmd9Z+97328rCOY8TFUJzlGUPaXzB8j2qgG/BMwA== dependencies: node-addon-api "^6.1.0" node-gyp-build "^3.9.0" -"@datadog/pprof@4.1.0": - version "4.1.0" - resolved "https://registry.yarnpkg.com/@datadog/pprof/-/pprof-4.1.0.tgz#db86a720f1dfecbcab8838bc1f148eb0a402af55" - integrity sha512-g7EWI185nwSuFwlmnAGDPxbPsqe+ipOoDB2oP841WMNRaJBPRdg5J90c+6ucmyltuC9VpTrmzzqcachkOTzZEQ== +"@datadog/pprof@5.4.1": + version "5.4.1" + resolved "https://registry.yarnpkg.com/@datadog/pprof/-/pprof-5.4.1.tgz#08c9bcf5d8efb2eeafdfc9f5bb5402f79fb41266" + integrity sha512-IvpL96e/cuh8ugP5O8Czdup7XQOLHeIDgM5pac5W7Lc1YzGe5zTtebKFpitvb1CPw1YY+1qFx0pWGgKP2kOfHg== dependencies: delay "^5.0.0" node-gyp-build "<4.0" p-limit "^3.1.0" - pprof-format "^2.0.7" + pprof-format "^2.1.0" source-map "^0.7.4" "@datadog/sketches-js@^2.1.0": version "2.1.0" - resolved "https://registry.yarnpkg.com/@datadog/sketches-js/-/sketches-js-2.1.0.tgz#8c7e8028a5fc22ad102fa542b0a446c956830455" + resolved "https://registry.npmjs.org/@datadog/sketches-js/-/sketches-js-2.1.0.tgz" integrity sha512-smLocSfrt3s53H/XSVP3/1kP42oqvrkjUPtyaFd1F79ux24oE31BKt+q0c6lsa6hOYrFzsIwyc5GXAI5JmfOew== "@esbuild/android-arm64@0.16.12": @@ -448,7 +464,7 @@ "@esbuild/darwin-arm64@0.16.12": version "0.16.12" - resolved "https://registry.yarnpkg.com/@esbuild/darwin-arm64/-/darwin-arm64-0.16.12.tgz#ac6c5d85cabf20de5047b55eab7f3c252d9aae71" + resolved "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.16.12.tgz" integrity sha512-Dpe5hOAQiQRH20YkFAg+wOpcd4PEuXud+aGgKBQa/VriPJA8zuVlgCOSTwna1CgYl05lf6o5els4dtuyk1qJxQ== "@esbuild/darwin-x64@0.16.12": @@ -541,48 +557,91 @@ resolved "https://registry.yarnpkg.com/@esbuild/win32-x64/-/win32-x64-0.16.12.tgz#31197bb509049b63c059c4808ac58e66fdff7479" integrity sha512-iPYKN78t3op2+erv2frW568j1q0RpqX6JOLZ7oPPaAV1VaF7dDstOrNw37PVOYoTWE11pV4A1XUitpdEFNIsPg== -"@eslint/eslintrc@^1.3.2": - version "1.3.2" - resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-1.3.2.tgz#58b69582f3b7271d8fa67fe5251767a5b38ea356" - integrity sha512-AXYd23w1S/bv3fTs3Lz0vjiYemS08jWkI3hYyS9I1ry+0f+Yjs1wm+sU0BS8qDOPrBIkp4qHYC16I8uVtpLajQ== +"@eslint-community/eslint-utils@^4.1.2", "@eslint-community/eslint-utils@^4.2.0", "@eslint-community/eslint-utils@^4.4.0": + version "4.4.0" + resolved "https://registry.yarnpkg.com/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz#a23514e8fb9af1269d5f7788aa556798d61c6b59" + integrity sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA== + dependencies: + eslint-visitor-keys "^3.3.0" + +"@eslint-community/regexpp@^4.11.0", "@eslint-community/regexpp@^4.6.1": + version "4.11.0" + resolved "https://registry.yarnpkg.com/@eslint-community/regexpp/-/regexpp-4.11.0.tgz#b0ffd0312b4a3fd2d6f77237e7248a5ad3a680ae" + integrity sha512-G/M/tIiMrTAxEWRfLfQJMmGNX28IxBg4PBz8XqQhqUHLFI6TL2htpIB1iQCj144V5ee/JaKyT9/WZ0MGZWfA7A== + +"@eslint/eslintrc@^2.1.4": + version "2.1.4" + resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-2.1.4.tgz#388a269f0f25c1b6adc317b5a2c55714894c70ad" + integrity sha512-269Z39MS6wVJtsoUl10L60WdkhJVdPG24Q4eZTH3nnF6lpvSShEK3wQjDX9JRWAUPvPh7COouPpU9IrqaZFvtQ== dependencies: ajv "^6.12.4" debug "^4.3.2" - espree "^9.4.0" - globals "^13.15.0" + espree "^9.6.0" + globals "^13.19.0" ignore "^5.2.0" import-fresh "^3.2.1" js-yaml "^4.1.0" minimatch "^3.1.2" strip-json-comments "^3.1.1" -"@humanwhocodes/config-array@^0.10.5": - version "0.10.5" - resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.10.5.tgz#bb679745224745fff1e9a41961c1d45a49f81c04" - integrity sha512-XVVDtp+dVvRxMoxSiSfasYaG02VEe1qH5cKgMQJWhol6HwzbcqoCMJi8dAGoYAO57jhUyhI6cWuRiTcRaDaYug== +"@eslint/js@8.57.0": + version "8.57.0" + resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.57.0.tgz#a5417ae8427873f1dd08b70b3574b453e67b5f7f" + integrity sha512-Ys+3g2TaW7gADOJzPt83SJtCDhMjndcDMFVQ/Tj9iA1BfJzFKD9mAUXT3OenpuPHbI6P/myECxRJrofUsDx/5g== + +"@graphql-tools/merge@^8.4.1": + version "8.4.2" + resolved "https://registry.yarnpkg.com/@graphql-tools/merge/-/merge-8.4.2.tgz#95778bbe26b635e8d2f60ce9856b388f11fe8288" + integrity sha512-XbrHAaj8yDuINph+sAfuq3QCZ/tKblrTLOpirK0+CAgNlZUCHs0Fa+xtMUURgwCVThLle1AF7svJCxFizygLsw== dependencies: - "@humanwhocodes/object-schema" "^1.2.1" - debug "^4.1.1" - minimatch "^3.0.4" + "@graphql-tools/utils" "^9.2.1" + tslib "^2.4.0" -"@humanwhocodes/gitignore-to-minimatch@^1.0.2": - version "1.0.2" - resolved "https://registry.yarnpkg.com/@humanwhocodes/gitignore-to-minimatch/-/gitignore-to-minimatch-1.0.2.tgz#316b0a63b91c10e53f242efb4ace5c3b34e8728d" - integrity sha512-rSqmMJDdLFUsyxR6FMtD00nfQKKLFb1kv+qBbOVKqErvloEIJLo5bDTJTQNTYgeyp78JsA7u/NPi5jT1GR/MuA== +"@graphql-tools/schema@^9.0.0": + version "9.0.19" + resolved "https://registry.yarnpkg.com/@graphql-tools/schema/-/schema-9.0.19.tgz#c4ad373b5e1b8a0cf365163435b7d236ebdd06e7" + integrity sha512-oBRPoNBtCkk0zbUsyP4GaIzCt8C0aCI4ycIRUL67KK5pOHljKLBBtGT+Jr6hkzA74C8Gco8bpZPe7aWFjiaK2w== + dependencies: + "@graphql-tools/merge" "^8.4.1" + "@graphql-tools/utils" "^9.2.1" + tslib "^2.4.0" + value-or-promise "^1.0.12" + +"@graphql-tools/utils@^9.2.1": + version "9.2.1" + resolved "https://registry.yarnpkg.com/@graphql-tools/utils/-/utils-9.2.1.tgz#1b3df0ef166cfa3eae706e3518b17d5922721c57" + integrity sha512-WUw506Ql6xzmOORlriNrD6Ugx+HjVgYxt9KCXD9mHAak+eaXSwuGGPyE60hy9xaDEoXKBsG7SkG69ybitaVl6A== + dependencies: + "@graphql-typed-document-node/core" "^3.1.1" + tslib "^2.4.0" + +"@graphql-typed-document-node/core@^3.1.1": + version "3.2.0" + resolved "https://registry.yarnpkg.com/@graphql-typed-document-node/core/-/core-3.2.0.tgz#5f3d96ec6b2354ad6d8a28bf216a1d97b5426861" + integrity sha512-mB9oAsNCm9aM3/SOv4YtBMqZbYj10R7dkq8byBqxGY/ncFwhf2oQzMV+LCRlWoDSEBJ3COiR1yeDvMtsoOsuFQ== + +"@humanwhocodes/config-array@^0.11.14": + version "0.11.14" + resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.11.14.tgz#d78e481a039f7566ecc9660b4ea7fe6b1fec442b" + integrity sha512-3T8LkOmg45BV5FICb15QQMsyUSWrQ8AygVfC7ZG32zOalnqrilm018ZVCw0eapXux8FtA33q8PSRSstjee3jSg== + dependencies: + "@humanwhocodes/object-schema" "^2.0.2" + debug "^4.3.1" + minimatch "^3.0.5" "@humanwhocodes/module-importer@^1.0.1": version "1.0.1" - resolved "https://registry.yarnpkg.com/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz#af5b2691a22b44be847b0ca81641c5fb6ad0172c" + resolved "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz" integrity sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA== -"@humanwhocodes/object-schema@^1.2.1": - version "1.2.1" - resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz#b520529ec21d8e5945a1851dfd1c32e94e39ff45" - integrity sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA== +"@humanwhocodes/object-schema@^2.0.2": + version "2.0.3" + resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-2.0.3.tgz#4a2868d75d6d6963e423bcf90b7fd1be343409d3" + integrity sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA== "@isaacs/import-jsx@^4.0.1": version "4.0.1" - resolved "https://registry.yarnpkg.com/@isaacs/import-jsx/-/import-jsx-4.0.1.tgz#493cab5fc543a0703dba7c3f5947d6499028a169" + resolved "https://registry.npmjs.org/@isaacs/import-jsx/-/import-jsx-4.0.1.tgz" integrity sha512-l34FEsEqpdYdGcQjRCxWy+7rHY6euUbOBz9FI+Mq6oQeVhNegHcXFSJxVxrJvOpO31NbnDjS74quKXDlPDearA== dependencies: "@babel/core" "^7.5.5" @@ -597,7 +656,7 @@ "@istanbuljs/load-nyc-config@^1.0.0": version "1.1.0" - resolved "https://registry.yarnpkg.com/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz#fd3db1d59ecf7cf121e80650bb86712f9b55eced" + resolved "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz" integrity sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ== dependencies: camelcase "^5.3.1" @@ -608,52 +667,44 @@ "@istanbuljs/schema@^0.1.2": version "0.1.3" - resolved "https://registry.yarnpkg.com/@istanbuljs/schema/-/schema-0.1.3.tgz#e45e384e4b8ec16bce2fd903af78450f6bf7ec98" + resolved "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz" integrity sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA== -"@jridgewell/gen-mapping@^0.1.0": - version "0.1.1" - resolved "https://registry.yarnpkg.com/@jridgewell/gen-mapping/-/gen-mapping-0.1.1.tgz#e5d2e450306a9491e3bd77e323e38d7aff315996" - integrity sha512-sQXCasFk+U8lWYEe66WxRDOE9PjVz4vSM51fTu3Hw+ClTpUSQb718772vH3pyS5pShp6lvQM7SxgIDXXXmOX7w== - dependencies: - "@jridgewell/set-array" "^1.0.0" - "@jridgewell/sourcemap-codec" "^1.4.10" - -"@jridgewell/gen-mapping@^0.3.2": - version "0.3.2" - resolved "https://registry.yarnpkg.com/@jridgewell/gen-mapping/-/gen-mapping-0.3.2.tgz#c1aedc61e853f2bb9f5dfe6d4442d3b565b253b9" - integrity sha512-mh65xKQAzI6iBcFzwv28KVWSmCkdRBWoOh+bYQGW3+6OZvbbN3TqMGo5hqYxQniRcH9F2VZIoJCm4pa3BPDK/A== +"@jridgewell/gen-mapping@^0.3.0", "@jridgewell/gen-mapping@^0.3.2": + version "0.3.3" + resolved "https://registry.yarnpkg.com/@jridgewell/gen-mapping/-/gen-mapping-0.3.3.tgz#7e02e6eb5df901aaedb08514203b096614024098" + integrity sha512-HLhSWOLRi875zjjMG/r+Nv0oCW8umGb0BgEhyX3dDX3egwZtB8PqLnjz3yedt8R5StBrzcg4aBpnh8UA9D1BoQ== dependencies: "@jridgewell/set-array" "^1.0.1" "@jridgewell/sourcemap-codec" "^1.4.10" "@jridgewell/trace-mapping" "^0.3.9" -"@jridgewell/resolve-uri@3.1.0", "@jridgewell/resolve-uri@^3.0.3": - version "3.1.0" - resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz#2203b118c157721addfe69d47b70465463066d78" - integrity sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w== +"@jridgewell/resolve-uri@^3.0.3", "@jridgewell/resolve-uri@^3.1.0": + version "3.1.2" + resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz#7a0ee601f60f99a20c7c7c5ff0c80388c1189bd6" + integrity sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw== -"@jridgewell/set-array@^1.0.0", "@jridgewell/set-array@^1.0.1": +"@jridgewell/set-array@^1.0.1": version "1.1.2" - resolved "https://registry.yarnpkg.com/@jridgewell/set-array/-/set-array-1.1.2.tgz#7c6cf998d6d20b914c0a55a91ae928ff25965e72" + resolved "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.1.2.tgz" integrity sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw== -"@jridgewell/sourcemap-codec@1.4.14", "@jridgewell/sourcemap-codec@^1.4.10": - version "1.4.14" - resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz#add4c98d341472a289190b424efbdb096991bb24" - integrity sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw== +"@jridgewell/sourcemap-codec@^1.4.10", "@jridgewell/sourcemap-codec@^1.4.14": + version "1.4.15" + resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz#d7c6e6755c78567a951e04ab52ef0fd26de59f32" + integrity sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg== "@jridgewell/trace-mapping@^0.3.17": - version "0.3.17" - resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.17.tgz#793041277af9073b0951a7fe0f0d8c4c98c36985" - integrity sha512-MCNzAp77qzKca9+W/+I0+sEpaUnZoeasnghNeVc41VZCEKaCH73Vq3BZZ/SzWIgrqE4H4ceI+p+b6C0mHf9T4g== + version "0.3.20" + resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.20.tgz#72e45707cf240fa6b081d0366f8265b0cd10197f" + integrity "sha1-cuRXB88kD6awgdA2b4JlsM0QGX8= sha512-R8LcPeWZol2zR8mmH3JeKQ6QRCFb7XgUhV9ZlGhHLGyg4wpPiPZNQOOWhFZhxKw8u//yTbNGI42Bx/3paXEQ+Q==" dependencies: - "@jridgewell/resolve-uri" "3.1.0" - "@jridgewell/sourcemap-codec" "1.4.14" + "@jridgewell/resolve-uri" "^3.1.0" + "@jridgewell/sourcemap-codec" "^1.4.14" "@jridgewell/trace-mapping@^0.3.9": version "0.3.15" - resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.15.tgz#aba35c48a38d3fd84b37e66c9c0423f9744f9774" + resolved "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.15.tgz" integrity sha512-oWZNOULl+UbhsgB51uuZzglikfIKSUBO/M9W2OfEjn7cmqoAiCgmv9lyACTUacZwBz0ITnJ2NqjU8Tx0DHL88g== dependencies: "@jridgewell/resolve-uri" "^3.0.3" @@ -661,18 +712,18 @@ "@nodelib/fs.scandir@2.1.5": version "2.1.5" - resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5" + resolved "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz" integrity sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g== dependencies: "@nodelib/fs.stat" "2.0.5" run-parallel "^1.1.9" -"@nodelib/fs.stat@2.0.5", "@nodelib/fs.stat@^2.0.2": +"@nodelib/fs.stat@2.0.5": version "2.0.5" - resolved "https://registry.yarnpkg.com/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz#5bd262af94e9d25bd1e71b05deed44876a222e8b" + resolved "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz" integrity sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A== -"@nodelib/fs.walk@^1.2.3": +"@nodelib/fs.walk@^1.2.8": version "1.2.8" resolved "https://registry.yarnpkg.com/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz#e95737e8bb6746ddedf69c556953494f196fe69a" integrity sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg== @@ -680,46 +731,46 @@ "@nodelib/fs.scandir" "2.1.5" fastq "^1.6.0" -"@opentelemetry/api@^1.0.0": - version "1.4.1" - resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.4.1.tgz#ff22eb2e5d476fbc2450a196e40dd243cc20c28f" - integrity sha512-O2yRJce1GOc6PAy3QxFM4NzFiWzvScDC1/5ihYBL6BUEVdq0XMWN01sppE+H6bBXbaFYipjwFLEWLg5PaSOThA== +"@opentelemetry/api@>=1.0.0 <1.9.0": + version "1.8.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.8.0.tgz#5aa7abb48f23f693068ed2999ae627d2f7d902ec" + integrity sha512-I/s6F7yKUDdtMsoBWXJe8Qz40Tui5vsuKCWJEWVL+5q9sSWRzzx6v2KeNsOBEwd94j0eWkpWCH4yB6rZg9Mf0w== "@opentelemetry/core@^1.14.0": version "1.14.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/core/-/core-1.14.0.tgz#64e876b29cb736c984d54164cd47433f513eafd3" - integrity sha512-MnMZ+sxsnlzloeuXL2nm5QcNczt/iO82UOeQQDHhV83F2fP3sgntW2evvtoxJki0MBLxEsh5ADD7PR/Hn5uzjw== + resolved "https://registry.npmjs.org/@opentelemetry/core/-/core-1.14.0.tgz" + integrity "sha1-ZOh2spy3NsmE1UFkzUdDP1E+r9M= sha512-MnMZ+sxsnlzloeuXL2nm5QcNczt/iO82UOeQQDHhV83F2fP3sgntW2evvtoxJki0MBLxEsh5ADD7PR/Hn5uzjw==" dependencies: "@opentelemetry/semantic-conventions" "1.14.0" "@opentelemetry/semantic-conventions@1.14.0": version "1.14.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/semantic-conventions/-/semantic-conventions-1.14.0.tgz#6a729b7f372ce30f77a3f217c09bc216f863fccb" - integrity sha512-rJfCY8rCWz3cb4KI6pEofnytvMPuj3YLQwoscCCYZ5DkdiPjo15IQ0US7+mjcWy9H3fcZIzf2pbJZ7ck/h4tug== + resolved "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.14.0.tgz" + integrity "sha1-anKbfzcs4w93o/IXwJvCFvhj/Ms= sha512-rJfCY8rCWz3cb4KI6pEofnytvMPuj3YLQwoscCCYZ5DkdiPjo15IQ0US7+mjcWy9H3fcZIzf2pbJZ7ck/h4tug==" "@protobufjs/aspromise@^1.1.1", "@protobufjs/aspromise@^1.1.2": version "1.1.2" - resolved "https://registry.yarnpkg.com/@protobufjs/aspromise/-/aspromise-1.1.2.tgz#9b8b0cc663d669a7d8f6f5d0893a14d348f30fbf" + resolved "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz" integrity sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ== "@protobufjs/base64@^1.1.2": version "1.1.2" - resolved "https://registry.yarnpkg.com/@protobufjs/base64/-/base64-1.1.2.tgz#4c85730e59b9a1f1f349047dbf24296034bb2735" + resolved "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz" integrity sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg== "@protobufjs/codegen@^2.0.4": version "2.0.4" - resolved "https://registry.yarnpkg.com/@protobufjs/codegen/-/codegen-2.0.4.tgz#7ef37f0d010fb028ad1ad59722e506d9262815cb" + resolved "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz" integrity sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg== "@protobufjs/eventemitter@^1.1.0": version "1.1.0" - resolved "https://registry.yarnpkg.com/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz#355cbc98bafad5978f9ed095f397621f1d066b70" + resolved "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz" integrity sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q== "@protobufjs/fetch@^1.1.0": version "1.1.0" - resolved "https://registry.yarnpkg.com/@protobufjs/fetch/-/fetch-1.1.0.tgz#ba99fb598614af65700c1619ff06d454b0d84c45" + resolved "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz" integrity sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ== dependencies: "@protobufjs/aspromise" "^1.1.1" @@ -727,147 +778,241 @@ "@protobufjs/float@^1.0.2": version "1.0.2" - resolved "https://registry.yarnpkg.com/@protobufjs/float/-/float-1.0.2.tgz#5e9e1abdcb73fc0a7cb8b291df78c8cbd97b87d1" + resolved "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz" integrity sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ== "@protobufjs/inquire@^1.1.0": version "1.1.0" - resolved "https://registry.yarnpkg.com/@protobufjs/inquire/-/inquire-1.1.0.tgz#ff200e3e7cf2429e2dcafc1140828e8cc638f089" + resolved "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz" integrity sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q== "@protobufjs/path@^1.1.2": version "1.1.2" - resolved "https://registry.yarnpkg.com/@protobufjs/path/-/path-1.1.2.tgz#6cc2b20c5c9ad6ad0dccfd21ca7673d8d7fbf68d" + resolved "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz" integrity sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA== "@protobufjs/pool@^1.1.0": version "1.1.0" - resolved "https://registry.yarnpkg.com/@protobufjs/pool/-/pool-1.1.0.tgz#09fd15f2d6d3abfa9b65bc366506d6ad7846ff54" + resolved "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz" integrity sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw== "@protobufjs/utf8@^1.1.0": version "1.1.0" - resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570" + resolved "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz" integrity sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw== -"@sinonjs/commons@^2.0.0": - version "2.0.0" - resolved "https://registry.yarnpkg.com/@sinonjs/commons/-/commons-2.0.0.tgz#fd4ca5b063554307e8327b4564bd56d3b73924a3" - integrity sha512-uLa0j859mMrg2slwQYdO/AkrOfmH+X6LTVmNTS9CqexuE2IvVORIkSpJLqePAbEnKJ77aMmCwr1NUZ57120Xcg== - dependencies: - type-detect "4.0.8" - -"@sinonjs/commons@^3.0.0": - version "3.0.0" - resolved "https://registry.yarnpkg.com/@sinonjs/commons/-/commons-3.0.0.tgz#beb434fe875d965265e04722ccfc21df7f755d72" - integrity sha512-jXBtWAF4vmdNmZgD5FoKsVLv3rPgDnLgPbU84LIJ3otV44vJlDRokVng5v8NFJdCf/da9legHcKaRuZs4L7faA== +"@sinonjs/commons@^3.0.0", "@sinonjs/commons@^3.0.1": + version "3.0.1" + resolved "https://registry.yarnpkg.com/@sinonjs/commons/-/commons-3.0.1.tgz#1029357e44ca901a615585f6d27738dbc89084cd" + integrity sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ== dependencies: type-detect "4.0.8" -"@sinonjs/fake-timers@^10.0.2", "@sinonjs/fake-timers@^10.3.0": +"@sinonjs/fake-timers@^10.3.0": version "10.3.0" resolved "https://registry.yarnpkg.com/@sinonjs/fake-timers/-/fake-timers-10.3.0.tgz#55fdff1ecab9f354019129daf4df0dd4d923ea66" integrity sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA== dependencies: "@sinonjs/commons" "^3.0.0" +"@sinonjs/fake-timers@^11.2.2": + version "11.3.1" + resolved "https://registry.yarnpkg.com/@sinonjs/fake-timers/-/fake-timers-11.3.1.tgz#51d6e8d83ca261ff02c0ab0e68e9db23d5cd5999" + integrity sha512-EVJO7nW5M/F5Tur0Rf2z/QoMo+1Ia963RiMtapiQrEWvY0iBUvADo8Beegwjpnle5BHkyHuoxSTW3jF43H1XRA== + dependencies: + "@sinonjs/commons" "^3.0.1" + "@sinonjs/samsam@^8.0.0": - version "8.0.0" - resolved "https://registry.yarnpkg.com/@sinonjs/samsam/-/samsam-8.0.0.tgz#0d488c91efb3fa1442e26abea81759dfc8b5ac60" - integrity sha512-Bp8KUVlLp8ibJZrnvq2foVhP0IVX2CIprMJPK0vqGqgrDa0OHVKeZyBykqskkrdxV6yKBPmGasO8LVjAKR3Gew== + version "8.0.2" + resolved "https://registry.yarnpkg.com/@sinonjs/samsam/-/samsam-8.0.2.tgz#e4386bf668ff36c95949e55a38dc5f5892fc2689" + integrity sha512-v46t/fwnhejRSFTGqbpn9u+LQ9xJDse10gNnPgAcxgdoCDMXj/G2asWAC/8Qs+BAZDicX+MNZouXT1A7c83kVw== dependencies: - "@sinonjs/commons" "^2.0.0" + "@sinonjs/commons" "^3.0.1" lodash.get "^4.4.2" - type-detect "^4.0.8" + type-detect "^4.1.0" -"@sinonjs/text-encoding@^0.7.1": - version "0.7.2" - resolved "https://registry.yarnpkg.com/@sinonjs/text-encoding/-/text-encoding-0.7.2.tgz#5981a8db18b56ba38ef0efb7d995b12aa7b51918" - integrity sha512-sXXKG+uL9IrKqViTtao2Ws6dy0znu9sOaP1di/jKGW1M6VssO8vlpXCQcpZ+jisQ1tTFAC5Jo/EOzFbggBagFQ== +"@sinonjs/text-encoding@^0.7.2": + version "0.7.3" + resolved "https://registry.yarnpkg.com/@sinonjs/text-encoding/-/text-encoding-0.7.3.tgz#282046f03e886e352b2d5f5da5eb755e01457f3f" + integrity sha512-DE427ROAphMQzU4ENbliGYrBSYPXF+TtLg9S8vzeA+OF4ZKzoDdzfL8sxuMUGS/lgRhM6j1URSk9ghf7Xo1tyA== + +"@types/body-parser@*": + version "1.19.5" + resolved "https://registry.yarnpkg.com/@types/body-parser/-/body-parser-1.19.5.tgz#04ce9a3b677dc8bd681a17da1ab9835dc9d3ede4" + integrity sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg== + dependencies: + "@types/connect" "*" + "@types/node" "*" + +"@types/connect@*": + version "3.4.38" + resolved "https://registry.yarnpkg.com/@types/connect/-/connect-3.4.38.tgz#5ba7f3bc4fbbdeaff8dded952e5ff2cc53f8d858" + integrity sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug== + dependencies: + "@types/node" "*" + +"@types/express-serve-static-core@^4.17.30", "@types/express-serve-static-core@^4.17.33": + version "4.19.6" + resolved "https://registry.yarnpkg.com/@types/express-serve-static-core/-/express-serve-static-core-4.19.6.tgz#e01324c2a024ff367d92c66f48553ced0ab50267" + integrity sha512-N4LZ2xG7DatVqhCZzOGb1Yi5lMbXSZcmdLDe9EzSndPV2HpWYWzRbaerl2n27irrm94EPpprqa8KpskPT085+A== + dependencies: + "@types/node" "*" + "@types/qs" "*" + "@types/range-parser" "*" + "@types/send" "*" + +"@types/express@^4.17.13": + version "4.17.21" + resolved "https://registry.yarnpkg.com/@types/express/-/express-4.17.21.tgz#c26d4a151e60efe0084b23dc3369ebc631ed192d" + integrity sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ== + dependencies: + "@types/body-parser" "*" + "@types/express-serve-static-core" "^4.17.33" + "@types/qs" "*" + "@types/serve-static" "*" + +"@types/http-errors@*": + version "2.0.4" + resolved "https://registry.yarnpkg.com/@types/http-errors/-/http-errors-2.0.4.tgz#7eb47726c391b7345a6ec35ad7f4de469cf5ba4f" + integrity sha512-D0CFMMtydbJAegzOyHjtiKPLlvnm3iTZyZRSZoLq2mRhDdmLfIWOCYPfQJ4cu2erKghU++QvjcUjp/5h7hESpA== "@types/json5@^0.0.29": version "0.0.29" - resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee" + resolved "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz" integrity sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ== +"@types/long@^4.0.0": + version "4.0.2" + resolved "https://registry.yarnpkg.com/@types/long/-/long-4.0.2.tgz#b74129719fc8d11c01868010082d483b7545591a" + integrity sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA== + +"@types/mime@^1": + version "1.3.5" + resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.5.tgz#1ef302e01cf7d2b5a0fa526790c9123bf1d06690" + integrity sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w== + +"@types/node-fetch@^2.6.1": + version "2.6.11" + resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.11.tgz#9b39b78665dae0e82a08f02f4967d62c66f95d24" + integrity sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g== + dependencies: + "@types/node" "*" + form-data "^4.0.0" + +"@types/node@*": + version "22.7.8" + resolved "https://registry.yarnpkg.com/@types/node/-/node-22.7.8.tgz#a1dbf0dc5f71bdd2642fc89caef65d58747ce825" + integrity sha512-a922jJy31vqR5sk+kAdIENJjHblqcZ4RmERviFsER4WJcEONqxKcjNOlk0q7OUfrF5sddT+vng070cdfMlrPLg== + dependencies: + undici-types "~6.19.2" + "@types/node@>=13.7.0": - version "18.11.9" - resolved "https://registry.yarnpkg.com/@types/node/-/node-18.11.9.tgz#02d013de7058cea16d36168ef2fc653464cfbad4" - integrity sha512-CRpX21/kGdzjOpFsZSkcrXMGIBWMGNIHXXBVFSH+ggkftxg+XYP20TESbh+zFvFj3EQOl5byk0HTRn1IL6hbqg== + version "20.14.11" + resolved "https://registry.yarnpkg.com/@types/node/-/node-20.14.11.tgz#09b300423343460455043ddd4d0ded6ac579b74b" + integrity sha512-kprQpL8MMeszbz6ojB5/tU8PLN4kesnN8Gjzw349rDlNgsSzg90lAVj3llK99Dh7JON+t9AuscPPFW6mPbTnSA== + dependencies: + undici-types "~5.26.4" -"@types/node@>=16": - version "18.7.23" - resolved "https://registry.yarnpkg.com/@types/node/-/node-18.7.23.tgz#75c580983846181ebe5f4abc40fe9dfb2d65665f" - integrity sha512-DWNcCHolDq0ZKGizjx2DZjR/PqsYwAcYUJmfMWqtVU2MBMG5Mo+xFZrhGId5r/O5HOuMPyQEcM6KUBp5lBZZBg== +"@types/node@^16.18.103": + version "16.18.103" + resolved "https://registry.yarnpkg.com/@types/node/-/node-16.18.103.tgz#5557c7c32a766fddbec4b933b1d5c365f89b20a4" + integrity sha512-gOAcUSik1nR/CRC3BsK8kr6tbmNIOTpvb1sT+v5Nmmys+Ho8YtnIHP90wEsVK4hTcHndOqPVIlehEGEA5y31bA== "@types/prop-types@*": version "15.7.5" - resolved "https://registry.yarnpkg.com/@types/prop-types/-/prop-types-15.7.5.tgz#5f19d2b85a98e9558036f6a3cacc8819420f05cf" - integrity sha512-JCB8C6SnDoQf0cNycqd/35A7MjcnK+ZTqE7judS6o7utxUCg6imJg3QK2qzHKszlTjcj2cn+NwMB2i96ubpj7w== + resolved "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.5.tgz" + integrity "sha1-XxnSuFqY6VWANvajysyIGUIPBc8= sha512-JCB8C6SnDoQf0cNycqd/35A7MjcnK+ZTqE7judS6o7utxUCg6imJg3QK2qzHKszlTjcj2cn+NwMB2i96ubpj7w==" + +"@types/qs@*": + version "6.9.16" + resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.16.tgz#52bba125a07c0482d26747d5d4947a64daf8f794" + integrity sha512-7i+zxXdPD0T4cKDuxCUXJ4wHcsJLwENa6Z3dCu8cfCK743OGy5Nu1RmAGqDPsoTDINVEcdXKRvR/zre+P2Ku1A== + +"@types/range-parser@*": + version "1.2.7" + resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.7.tgz#50ae4353eaaddc04044279812f52c8c65857dbcb" + integrity sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ== "@types/react@^17.0.52": - version "17.0.58" - resolved "https://registry.yarnpkg.com/@types/react/-/react-17.0.58.tgz#c8bbc82114e5c29001548ebe8ed6c4ba4d3c9fb0" - integrity sha512-c1GzVY97P0fGxwGxhYq989j4XwlcHQoto6wQISOC2v6wm3h0PORRWJFHlkRjfGsiG3y1609WdQ+J+tKxvrEd6A== + version "17.0.71" + resolved "https://registry.yarnpkg.com/@types/react/-/react-17.0.71.tgz#3673d446ad482b1564e44bf853b3ab5bcbc942c4" + integrity "sha1-NnPURq1IKxVk5Ev4U7OrW8vJQsQ= sha512-lfqOu9mp16nmaGRrS8deS2Taqhd5Ih0o92Te5Ws6I1py4ytHBcXLqh0YIqVsViqwVI5f+haiFM6hju814BzcmA==" dependencies: "@types/prop-types" "*" "@types/scheduler" "*" csstype "^3.0.2" "@types/scheduler@*": - version "0.16.2" - resolved "https://registry.yarnpkg.com/@types/scheduler/-/scheduler-0.16.2.tgz#1a62f89525723dde24ba1b01b092bf5df8ad4d39" - integrity sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew== + version "0.16.8" + resolved "https://registry.yarnpkg.com/@types/scheduler/-/scheduler-0.16.8.tgz#ce5ace04cfeabe7ef87c0091e50752e36707deff" + integrity "sha1-zlrOBM/qvn74fACR5QdS42cH3v8= sha512-WZLiwShhwLRmeV6zH+GkbOFT6Z6VklCItrDioxUnv+u4Ll+8vKeFySoFyK/0ctcRpOmwAicELfmys1sDc/Rw+A==" + +"@types/send@*": + version "0.17.4" + resolved "https://registry.yarnpkg.com/@types/send/-/send-0.17.4.tgz#6619cd24e7270793702e4e6a4b958a9010cfc57a" + integrity sha512-x2EM6TJOybec7c52BX0ZspPodMsQUd5L6PRwOunVyVUhXiBSKf3AezDL8Dgvgt5o0UfKNfuA0eMLr2wLT4AiBA== + dependencies: + "@types/mime" "^1" + "@types/node" "*" + +"@types/serve-static@*": + version "1.15.7" + resolved "https://registry.yarnpkg.com/@types/serve-static/-/serve-static-1.15.7.tgz#22174bbd74fb97fe303109738e9b5c2f3064f714" + integrity sha512-W8Ym+h8nhuRwaKPaDw34QUkwsGi6Rc4yYqvKFo5rm2FUEhCFbzVWrxXUxuKK8TASjWsysJY0nsmNCGhCOIsrOw== + dependencies: + "@types/http-errors" "*" + "@types/node" "*" + "@types/send" "*" "@types/yoga-layout@1.9.2": version "1.9.2" - resolved "https://registry.yarnpkg.com/@types/yoga-layout/-/yoga-layout-1.9.2.tgz#efaf9e991a7390dc081a0b679185979a83a9639a" + resolved "https://registry.npmjs.org/@types/yoga-layout/-/yoga-layout-1.9.2.tgz" integrity sha512-S9q47ByT2pPvD65IvrWp7qppVMpk9WGMbVq9wbWZOHg6tnXSD4vyhao6nOSBwwfDdV2p3Kx9evA9vI+XWTfDvw== "@ungap/promise-all-settled@1.1.2": version "1.1.2" - resolved "https://registry.yarnpkg.com/@ungap/promise-all-settled/-/promise-all-settled-1.1.2.tgz#aa58042711d6e3275dd37dc597e5d31e8c290a44" + resolved "https://registry.npmjs.org/@ungap/promise-all-settled/-/promise-all-settled-1.1.2.tgz" integrity sha512-sL/cEvJWAnClXw0wHk85/2L0G6Sj8UB0Ctc1TEMbKSsmpRosqhwj9gWgFRZSrBr2f9tiXISwNhCPmlfqUqyb9Q== +"@ungap/structured-clone@^1.2.0": + version "1.2.0" + resolved "https://registry.yarnpkg.com/@ungap/structured-clone/-/structured-clone-1.2.0.tgz#756641adb587851b5ccb3e095daf27ae581c8406" + integrity sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ== + accepts@~1.3.8: version "1.3.8" - resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.8.tgz#0bf0be125b67014adcb0b0921e62db7bffe16b2e" + resolved "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz" integrity sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw== dependencies: mime-types "~2.1.34" negotiator "0.6.3" -acorn-import-assertions@^1.9.0: - version "1.9.0" - resolved "https://registry.yarnpkg.com/acorn-import-assertions/-/acorn-import-assertions-1.9.0.tgz#507276249d684797c84e0734ef84860334cfb1ac" - integrity sha512-cmMwop9x+8KFhxvKrKfPYmN6/pKTYYHBqLa0DfvVZcKMJWNyWLnaqND7dx/qn66R7ewM1UX5XMaDVP5wlVTaVA== +acorn-import-attributes@^1.9.5: + version "1.9.5" + resolved "https://registry.yarnpkg.com/acorn-import-attributes/-/acorn-import-attributes-1.9.5.tgz#7eb1557b1ba05ef18b5ed0ec67591bfab04688ef" + integrity sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ== acorn-jsx@^5.3.2: version "5.3.2" - resolved "https://registry.yarnpkg.com/acorn-jsx/-/acorn-jsx-5.3.2.tgz#7ed5bb55908b3b2f1bc55c6af1653bada7f07937" + resolved "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz" integrity sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ== -acorn@^8.8.0: - version "8.8.0" - resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.8.0.tgz#88c0187620435c7f6015803f5539dae05a9dbea8" - integrity sha512-QOxyigPVrpZ2GXT+PFyZTl6TtOFc5egxHIP9IlQ+RbupQuX4RkT/Bee4/kQuC02Xkzg84JcT7oLYtDIQxp+v7w== - -acorn@^8.8.2: - version "8.10.0" - resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.10.0.tgz#8be5b3907a67221a81ab23c7889c4c5526b62ec5" - integrity sha512-F0SAmZ8iUtS//m8DmCTA0jlh6TDKkHQyK6xc6V4KDTyZKA9dnvX9/3sRTVQrWm79glUAZbnmmNcdYwUIHWVybw== +acorn@^8.8.2, acorn@^8.9.0: + version "8.12.1" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.12.1.tgz#71616bdccbe25e27a54439e0046e89ca76df2248" + integrity sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg== aggregate-error@^3.0.0: version "3.1.0" - resolved "https://registry.yarnpkg.com/aggregate-error/-/aggregate-error-3.1.0.tgz#92670ff50f5359bdb7a3e0d40d0ec30c5737687a" + resolved "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz" integrity sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA== dependencies: clean-stack "^2.0.0" indent-string "^4.0.0" -ajv@^6.10.0, ajv@^6.12.4: +ajv@^6.12.4: version "6.12.6" - resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4" + resolved "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz" integrity sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g== dependencies: fast-deep-equal "^3.1.1" @@ -877,56 +1022,48 @@ ajv@^6.10.0, ajv@^6.12.4: ansi-colors@4.1.1: version "4.1.1" - resolved "https://registry.yarnpkg.com/ansi-colors/-/ansi-colors-4.1.1.tgz#cbb9ae256bf750af1eab344f229aa27fe94ba348" + resolved "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.1.tgz" integrity sha512-JoX0apGbHaUJBNl6yF+p6JAFYZ666/hhCGKN5t9QFjbJQKUU/g8MNbFDbvfrgKXvI1QpZplPOnwIo99lX/AAmA== ansi-escapes@^4.2.1: version "4.3.2" - resolved "https://registry.yarnpkg.com/ansi-escapes/-/ansi-escapes-4.3.2.tgz#6b2291d1db7d98b6521d5f1efa42d0f3a9feb65e" + resolved "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz" integrity sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ== dependencies: type-fest "^0.21.3" ansi-regex@^3.0.0: version "3.0.1" - resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-3.0.1.tgz#123d6479e92ad45ad897d4054e3c7ca7db4944e1" + resolved "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.1.tgz" integrity sha512-+O9Jct8wf++lXxxFc4hc8LsjaSq0HFzzL7cVsw8pRDIPdjKD2mT4ytDZlLuSBZ4cLKZFXIrMGO7DbQCtMJJMKw== ansi-regex@^5.0.1: version "5.0.1" - resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304" + resolved "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz" integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ== ansi-styles@^3.2.1: version "3.2.1" - resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-3.2.1.tgz#41fbb20243e50b12be0f04b8dedbf07520ce841d" + resolved "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz" integrity sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA== dependencies: color-convert "^1.9.0" ansi-styles@^4.0.0, ansi-styles@^4.1.0: version "4.3.0" - resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-4.3.0.tgz#edd803628ae71c04c85ae7a0906edad34b648937" + resolved "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz" integrity sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg== dependencies: color-convert "^2.0.1" ansicolors@~0.3.2: version "0.3.2" - resolved "https://registry.yarnpkg.com/ansicolors/-/ansicolors-0.3.2.tgz#665597de86a9ffe3aa9bfbe6cae5c6ea426b4979" + resolved "https://registry.npmjs.org/ansicolors/-/ansicolors-0.3.2.tgz" integrity sha512-QXu7BPrP29VllRxH8GwB7x5iX5qWKAAMLqKQGWTeLWVlNHNOpVMJ91dsxQAIWXpjuW5wqvxu3Jd/nRjrJ+0pqg== -anymatch@~3.1.1: - version "3.1.2" - resolved "https://registry.yarnpkg.com/anymatch/-/anymatch-3.1.2.tgz#c0557c096af32f106198f4f4e2a383537e378716" - integrity sha512-P43ePfOAIupkguHUycrc4qJ9kz8ZiuOUijaETwX7THt0Y/GNK7v0aa8rY816xWjZ7rJdA5XdMcpVFTKMq+RvWg== - dependencies: - normalize-path "^3.0.0" - picomatch "^2.0.4" - anymatch@~3.1.2: version "3.1.3" - resolved "https://registry.yarnpkg.com/anymatch/-/anymatch-3.1.3.tgz#790c58b19ba1720a84205b57c618d5ad8524973e" + resolved "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz" integrity sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw== dependencies: normalize-path "^3.0.0" @@ -934,109 +1071,126 @@ anymatch@~3.1.2: append-field@^1.0.0: version "1.0.0" - resolved "https://registry.yarnpkg.com/append-field/-/append-field-1.0.0.tgz#1e3440e915f0b1203d23748e78edd7b9b5b43e56" + resolved "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz" integrity sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw== append-transform@^2.0.0: version "2.0.0" - resolved "https://registry.yarnpkg.com/append-transform/-/append-transform-2.0.0.tgz#99d9d29c7b38391e6f428d28ce136551f0b77e12" + resolved "https://registry.npmjs.org/append-transform/-/append-transform-2.0.0.tgz" integrity sha512-7yeyCEurROLQJFv5Xj4lEGTy0borxepjFv1g22oAdqFu//SrAlDl1O1Nxx15SH1RoliUml6p8dwJW9jvZughhg== dependencies: default-require-extensions "^3.0.0" archy@^1.0.0: version "1.0.0" - resolved "https://registry.yarnpkg.com/archy/-/archy-1.0.0.tgz#f9c8c13757cc1dd7bc379ac77b2c62a5c2868c40" + resolved "https://registry.npmjs.org/archy/-/archy-1.0.0.tgz" integrity sha512-Xg+9RwCg/0p32teKdGMPTPnVXKD0w3DfHnFTficozsAgsvq2XenPJq/MYpzzQ/v8zrOyJn6Ds39VA4JIDwFfqw== argparse@^1.0.7: version "1.0.10" - resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911" + resolved "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz" integrity sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg== dependencies: sprintf-js "~1.0.2" argparse@^2.0.1: version "2.0.1" - resolved "https://registry.yarnpkg.com/argparse/-/argparse-2.0.1.tgz#246f50f3ca78a3240f6c997e8a9bd1eac49e4b38" + resolved "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz" integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q== -array-buffer-byte-length@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/array-buffer-byte-length/-/array-buffer-byte-length-1.0.0.tgz#fabe8bc193fea865f317fe7807085ee0dee5aead" - integrity sha512-LPuwb2P+NrQw3XhxGc36+XSvuBPopovXYTR9Ew++Du9Yb/bx5AzBfrIsBoj0EZUifjQU+sHL21sseZ3jerWO/A== +array-buffer-byte-length@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/array-buffer-byte-length/-/array-buffer-byte-length-1.0.1.tgz#1e5583ec16763540a27ae52eed99ff899223568f" + integrity sha512-ahC5W1xgou+KTXix4sAO8Ki12Q+jf4i0+tmk3sC+zgcynshkHxzpXdImBehiUYKKKDwvfFiJl1tZt6ewscS1Mg== dependencies: - call-bind "^1.0.2" - is-array-buffer "^3.0.1" + call-bind "^1.0.5" + is-array-buffer "^3.0.4" array-flatten@1.1.1: version "1.1.1" - resolved "https://registry.yarnpkg.com/array-flatten/-/array-flatten-1.1.1.tgz#9a5f699051b1e7073328f2a008968b64ea2955d2" + resolved "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz" integrity sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg== -array-includes@^3.1.4: - version "3.1.5" - resolved "https://registry.yarnpkg.com/array-includes/-/array-includes-3.1.5.tgz#2c320010db8d31031fd2a5f6b3bbd4b1aad31bdb" - integrity sha512-iSDYZMMyTPkiFasVqfuAQnWAYcvO/SeBSCGKePoEthjp4LEMTe4uLc7b025o4jAZpHhihh8xPo99TNWUWWkGDQ== +array-includes@^3.1.7: + version "3.1.8" + resolved "https://registry.yarnpkg.com/array-includes/-/array-includes-3.1.8.tgz#5e370cbe172fdd5dd6530c1d4aadda25281ba97d" + integrity sha512-itaWrbYbqpGXkGhZPGUulwnhVf5Hpy1xiCFsGqyIGglbBxmG5vSjxQen3/WGOjPpNEv1RtBLKxbmVXm8HpJStQ== dependencies: - call-bind "^1.0.2" - define-properties "^1.1.4" - es-abstract "^1.19.5" - get-intrinsic "^1.1.1" + call-bind "^1.0.7" + define-properties "^1.2.1" + es-abstract "^1.23.2" + es-object-atoms "^1.0.0" + get-intrinsic "^1.2.4" is-string "^1.0.7" -array-union@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/array-union/-/array-union-2.1.0.tgz#b798420adbeb1de828d84acd8a2e23d3efe85e8d" - integrity sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw== - -array.prototype.every@^1.1.4: - version "1.1.4" - resolved "https://registry.yarnpkg.com/array.prototype.every/-/array.prototype.every-1.1.4.tgz#2762daecd9cec87cb63f3ca6be576817074a684e" - integrity sha512-Aui35iRZk1HHLRAyF7QP0KAnOnduaQ6fo6k1NVWfRc0xTs2AZ70ytlXvOmkC6Di4JmUs2Wv3DYzGtCQFSk5uGg== +array.prototype.findlastindex@^1.2.3: + version "1.2.5" + resolved "https://registry.yarnpkg.com/array.prototype.findlastindex/-/array.prototype.findlastindex-1.2.5.tgz#8c35a755c72908719453f87145ca011e39334d0d" + integrity sha512-zfETvRFA8o7EiNn++N5f/kaCw221hrpGsDmcpndVupkPzEc1Wuf3VgC0qby1BbHs7f5DVYjgtEU2LLh5bqeGfQ== dependencies: - call-bind "^1.0.2" - define-properties "^1.1.4" - es-abstract "^1.20.4" - is-string "^1.0.7" + call-bind "^1.0.7" + define-properties "^1.2.1" + es-abstract "^1.23.2" + es-errors "^1.3.0" + es-object-atoms "^1.0.0" + es-shim-unscopables "^1.0.2" -array.prototype.flat@^1.2.5: - version "1.3.0" - resolved "https://registry.yarnpkg.com/array.prototype.flat/-/array.prototype.flat-1.3.0.tgz#0b0c1567bf57b38b56b4c97b8aa72ab45e4adc7b" - integrity sha512-12IUEkHsAhA4DY5s0FPgNXIdc8VRSqD9Zp78a5au9abH/SOBrsp082JOWFNTjkMozh8mqcdiKuaLGhPeYztxSw== +array.prototype.flat@^1.3.2: + version "1.3.2" + resolved "https://registry.yarnpkg.com/array.prototype.flat/-/array.prototype.flat-1.3.2.tgz#1476217df8cff17d72ee8f3ba06738db5b387d18" + integrity sha512-djYB+Zx2vLewY8RWlNCUdHjDXs2XOgm602S9E7P/UpHgfeHL00cRiIF+IN/G/aUJ7kGPb6yO/ErDI5V2s8iycA== dependencies: call-bind "^1.0.2" - define-properties "^1.1.3" - es-abstract "^1.19.2" + define-properties "^1.2.0" + es-abstract "^1.22.1" es-shim-unscopables "^1.0.0" -arraybuffer.prototype.slice@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/arraybuffer.prototype.slice/-/arraybuffer.prototype.slice-1.0.1.tgz#9b5ea3868a6eebc30273da577eb888381c0044bb" - integrity sha512-09x0ZWFEjj4WD8PDbykUwo3t9arLn8NIzmmYEJFpYekOAQjpkGSyrQhNoRTcwwcFRu+ycWF78QZ63oWTqSjBcw== +array.prototype.flatmap@^1.3.2: + version "1.3.2" + resolved "https://registry.yarnpkg.com/array.prototype.flatmap/-/array.prototype.flatmap-1.3.2.tgz#c9a7c6831db8e719d6ce639190146c24bbd3e527" + integrity sha512-Ewyx0c9PmpcsByhSW4r+9zDU7sGjFc86qf/kKtuSCRdhfbk0SNLLkaT5qvcHnRGgc5NP/ly/y+qkXkqONX54CQ== dependencies: - array-buffer-byte-length "^1.0.0" call-bind "^1.0.2" define-properties "^1.2.0" - get-intrinsic "^1.2.1" - is-array-buffer "^3.0.2" + es-abstract "^1.22.1" + es-shim-unscopables "^1.0.0" + +arraybuffer.prototype.slice@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/arraybuffer.prototype.slice/-/arraybuffer.prototype.slice-1.0.3.tgz#097972f4255e41bc3425e37dc3f6421cf9aefde6" + integrity sha512-bMxMKAjg13EBSVscxTaYA4mRc5t1UAXa2kXiGTNfZ079HIWXEkKmkgFrh/nJqamaLSrXO5H4WFFkPEaLJWbs3A== + dependencies: + array-buffer-byte-length "^1.0.1" + call-bind "^1.0.5" + define-properties "^1.2.1" + es-abstract "^1.22.3" + es-errors "^1.2.1" + get-intrinsic "^1.2.3" + is-array-buffer "^3.0.4" is-shared-array-buffer "^1.0.2" assertion-error@^1.1.0: version "1.1.0" - resolved "https://registry.yarnpkg.com/assertion-error/-/assertion-error-1.1.0.tgz#e60b6b0e8f301bd97e5375215bda406c85118c0b" + resolved "https://registry.npmjs.org/assertion-error/-/assertion-error-1.1.0.tgz" integrity sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw== astral-regex@^2.0.0: version "2.0.0" - resolved "https://registry.yarnpkg.com/astral-regex/-/astral-regex-2.0.0.tgz#483143c567aeed4785759c0865786dc77d7d2e31" + resolved "https://registry.npmjs.org/astral-regex/-/astral-regex-2.0.0.tgz" integrity sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ== async-hook-domain@^2.0.4: version "2.0.4" - resolved "https://registry.yarnpkg.com/async-hook-domain/-/async-hook-domain-2.0.4.tgz#5a24910982c04394ea33dd442860f80cce2d972c" + resolved "https://registry.npmjs.org/async-hook-domain/-/async-hook-domain-2.0.4.tgz" integrity sha512-14LjCmlK1PK8eDtTezR6WX8TMaYNIzBIsd2D1sGoGjgx0BuNMMoSdk7i/drlbtamy0AWv9yv2tkB+ASdmeqFIw== +async-retry@^1.2.1: + version "1.3.3" + resolved "https://registry.yarnpkg.com/async-retry/-/async-retry-1.3.3.tgz#0e7f36c04d8478e7a58bdbed80cedf977785f280" + integrity sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw== + dependencies: + retry "0.13.1" + asynckit@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" @@ -1044,12 +1198,12 @@ asynckit@^0.4.0: auto-bind@4.0.0: version "4.0.0" - resolved "https://registry.yarnpkg.com/auto-bind/-/auto-bind-4.0.0.tgz#e3589fc6c2da8f7ca43ba9f84fa52a744fc997fb" + resolved "https://registry.npmjs.org/auto-bind/-/auto-bind-4.0.0.tgz" integrity sha512-Hdw8qdNiqdJ8LqT0iK0sVzkFbzg6fhnQqqfWhBDxcHZvU75+B+ayzTy8x+k5Ix0Y92XOhOUlx74ps+bA6BeYMQ== autocannon@^4.5.2: version "4.6.0" - resolved "https://registry.yarnpkg.com/autocannon/-/autocannon-4.6.0.tgz#01c18e211444bd523c97da4ff7ff83cd25035333" + resolved "https://registry.npmjs.org/autocannon/-/autocannon-4.6.0.tgz" integrity sha512-pWHEBJh9bkQeDXYj1NL2BBYeXTaLkbRiy3NZ7vNR1bq7vWxHP8R+iCmDyBCtuh2PMJiWlGlikXa1p0LUUY3Tdw== dependencies: chalk "^3.0.0" @@ -1072,15 +1226,17 @@ autocannon@^4.5.2: retimer "^2.0.0" timestring "^6.0.0" -available-typed-arrays@^1.0.5: - version "1.0.5" - resolved "https://registry.yarnpkg.com/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz#92f95616501069d07d10edb2fc37d3e1c65123b7" - integrity sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw== +available-typed-arrays@^1.0.7: + version "1.0.7" + resolved "https://registry.yarnpkg.com/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz#a5cc375d6a03c2efc87a553f3e0b1522def14846" + integrity sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ== + dependencies: + possible-typed-array-names "^1.0.0" aws-sdk@^2.1446.0: version "2.1477.0" - resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.1477.0.tgz#ec878ea5584fee217eb02ec8f6ebfd9ace47f908" - integrity sha512-DLsrKosrKRe5P1E+BcJAVpOXkma4oUOrcyBUridDmUhdf9k3jj5dnL1roFuDpTmNDDhK8a1tUgY3wmXoKQtv7A== + resolved "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1477.0.tgz" + integrity "sha1-7IeOpVhP7iF+sC7I9uv9ms5H+Qg= sha512-DLsrKosrKRe5P1E+BcJAVpOXkma4oUOrcyBUridDmUhdf9k3jj5dnL1roFuDpTmNDDhK8a1tUgY3wmXoKQtv7A==" dependencies: buffer "4.9.2" events "1.1.1" @@ -1093,26 +1249,28 @@ aws-sdk@^2.1446.0: uuid "8.0.0" xml2js "0.5.0" -axios@^0.21.2: - version "0.21.4" - resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575" - integrity sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg== +axios@^1.7.4: + version "1.7.4" + resolved "https://registry.yarnpkg.com/axios/-/axios-1.7.4.tgz#4c8ded1b43683c8dd362973c393f3ede24052aa2" + integrity sha512-DukmaFRnY6AzAALSH4J2M3k6PkaC+MfaAGdEERRWcC9q3/TWQwLpHR8ZRLKTdQ3aBDL64EdluRDjJqKw+BPZEw== dependencies: - follow-redirects "^1.14.0" + follow-redirects "^1.15.6" + form-data "^4.0.0" + proxy-from-env "^1.1.0" balanced-match@^1.0.0: version "1.0.2" - resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee" + resolved "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz" integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== base64-js@^1.0.2, base64-js@^1.2.0: version "1.5.1" - resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" + resolved "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz" integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== benchmark@^2.1.4: version "2.1.4" - resolved "https://registry.yarnpkg.com/benchmark/-/benchmark-2.1.4.tgz#09f3de31c916425d498cc2ee565a0ebf3c2a5629" + resolved "https://registry.npmjs.org/benchmark/-/benchmark-2.1.4.tgz" integrity sha512-l9MlfN4M1K/H2fbhfMy3B7vJd6AGKJVQn2h6Sg/Yx+KckoUA7ewS5Vv6TjSq18ooE1kS9hhAlQRH3AkXIh/aOQ== dependencies: lodash "^4.17.4" @@ -1120,36 +1278,18 @@ benchmark@^2.1.4: binary-extensions@^2.0.0: version "2.2.0" - resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.2.0.tgz#75f502eeaf9ffde42fc98829645be4ea76bd9e2d" + resolved "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz" integrity sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA== bind-obj-methods@^3.0.0: version "3.0.0" - resolved "https://registry.yarnpkg.com/bind-obj-methods/-/bind-obj-methods-3.0.0.tgz#65b66544d9d668d80dfefe2089dd347ad1dbcaed" + resolved "https://registry.npmjs.org/bind-obj-methods/-/bind-obj-methods-3.0.0.tgz" integrity sha512-nLEaaz3/sEzNSyPWRsN9HNsqwk1AUyECtGj+XwGdIi3xABnEqecvXtIJ0wehQXuuER5uZ/5fTs2usONgYjG+iw== -body-parser@1.20.1: - version "1.20.1" - resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.1.tgz#b1812a8912c195cd371a3ee5e66faa2338a5c668" - integrity sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw== - dependencies: - bytes "3.1.2" - content-type "~1.0.4" - debug "2.6.9" - depd "2.0.0" - destroy "1.2.0" - http-errors "2.0.0" - iconv-lite "0.4.24" - on-finished "2.4.1" - qs "6.11.0" - raw-body "2.5.1" - type-is "~1.6.18" - unpipe "1.0.0" - -body-parser@^1.20.2: - version "1.20.2" - resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.2.tgz#6feb0e21c4724d06de7ff38da36dad4f57a747fd" - integrity sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA== +body-parser@1.20.3, body-parser@^1.20.3: + version "1.20.3" + resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.3.tgz#1953431221c6fb5cd63c4b36d53fab0928e548c6" + integrity sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g== dependencies: bytes "3.1.2" content-type "~1.0.5" @@ -1159,77 +1299,82 @@ body-parser@^1.20.2: http-errors "2.0.0" iconv-lite "0.4.24" on-finished "2.4.1" - qs "6.11.0" + qs "6.13.0" raw-body "2.5.2" type-is "~1.6.18" unpipe "1.0.0" brace-expansion@^1.1.7: version "1.1.11" - resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd" + resolved "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz" integrity sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA== dependencies: balanced-match "^1.0.0" concat-map "0.0.1" -braces@^3.0.2, braces@~3.0.2: - version "3.0.2" - resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107" - integrity sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A== +braces@~3.0.2: + version "3.0.3" + resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.3.tgz#490332f40919452272d55a8480adc0c441358789" + integrity sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA== dependencies: - fill-range "^7.0.1" + fill-range "^7.1.1" browser-stdout@1.3.1: version "1.3.1" - resolved "https://registry.yarnpkg.com/browser-stdout/-/browser-stdout-1.3.1.tgz#baa559ee14ced73452229bad7326467c61fabd60" + resolved "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz" integrity sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw== -browserslist@^4.21.3: - version "4.21.4" - resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.21.4.tgz#e7496bbc67b9e39dd0f98565feccdcb0d4ff6987" - integrity sha512-CBHJJdDmgjl3daYjN5Cp5kbTf1mUhZoS+beLklHIvkOWscs83YAhLlF3Wsh/lciQYAcbBJgTOD44VtG31ZM4Hw== +browserslist@^4.21.9: + version "4.22.2" + resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.22.2.tgz#704c4943072bd81ea18997f3bd2180e89c77874b" + integrity "sha1-cExJQwcr2B6hiZfzvSGA6Jx3h0s= sha512-0UgcrvQmBDvZHFGdYUehrCNIazki7/lUP3kkoi/r3YB2amZbFM9J43ZRkJTXBUZK4gmx56+Sqk9+Vs9mwZx9+A==" dependencies: - caniuse-lite "^1.0.30001400" - electron-to-chromium "^1.4.251" - node-releases "^2.0.6" - update-browserslist-db "^1.0.9" + caniuse-lite "^1.0.30001565" + electron-to-chromium "^1.4.601" + node-releases "^2.0.14" + update-browserslist-db "^1.0.13" buffer-from@^1.0.0: version "1.1.2" - resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5" + resolved "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz" integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ== buffer@4.9.2: version "4.9.2" - resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.2.tgz#230ead344002988644841ab0244af8c44bbe3ef8" - integrity sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg== + resolved "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz" + integrity "sha1-Iw6tNEACmIZEhBqwJEr4xEu+Pvg= sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==" dependencies: base64-js "^1.0.2" ieee754 "^1.1.4" isarray "^1.0.0" +builtin-modules@^3.3.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/builtin-modules/-/builtin-modules-3.3.0.tgz#cae62812b89801e9656336e46223e030386be7b6" + integrity sha512-zhaCDicdLuWN5UbN5IMnFqNMhNfo919sH85y2/ea+5Yg9TsTkeZxpL+JLbp6cgYFS4sRLp3YV4S6yDuqVWHYOw== + builtins@^5.0.1: version "5.0.1" - resolved "https://registry.yarnpkg.com/builtins/-/builtins-5.0.1.tgz#87f6db9ab0458be728564fa81d876d8d74552fa9" + resolved "https://registry.npmjs.org/builtins/-/builtins-5.0.1.tgz" integrity sha512-qwVpFEHNfhYJIzNRBvd2C1kyo6jz3ZSMPyyuR47OPdiKWlbYnZNyDWuyR175qDnAJLiCo5fBBqPb3RiXgWlkOQ== dependencies: semver "^7.0.0" busboy@^1.0.0: version "1.6.0" - resolved "https://registry.yarnpkg.com/busboy/-/busboy-1.6.0.tgz#966ea36a9502e43cdb9146962523b92f531f6893" + resolved "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz" integrity sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA== dependencies: streamsearch "^1.1.0" bytes@3.1.2: version "3.1.2" - resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.2.tgz#8b0beeb98605adf1b128fa4386403c009e0221a5" + resolved "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz" integrity sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg== caching-transform@^4.0.0: version "4.0.0" - resolved "https://registry.yarnpkg.com/caching-transform/-/caching-transform-4.0.0.tgz#00d297a4206d71e2163c39eaffa8157ac0651f0f" + resolved "https://registry.npmjs.org/caching-transform/-/caching-transform-4.0.0.tgz" integrity sha512-kpqOvwXnjjN44D89K5ccQC+RUrsy7jB/XLlRrx0D7/2HNcTPqzsb6XgYoErwko6QsV184CA2YgS1fxDiiDZMWA== dependencies: hasha "^5.0.0" @@ -1237,60 +1382,54 @@ caching-transform@^4.0.0: package-hash "^4.0.0" write-file-atomic "^3.0.0" -call-bind@^1.0.0, call-bind@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.2.tgz#b1d4e89e688119c3c9a903ad30abb2f6a919be3c" - integrity sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA== - dependencies: - function-bind "^1.1.1" - get-intrinsic "^1.0.2" - -call-bind@^1.0.4: - version "1.0.5" - resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.5.tgz#6fa2b7845ce0ea49bf4d8b9ef64727a2c2e2e513" - integrity sha512-C3nQxfFZxFRVoJoGKKI8y3MOEo129NQ+FgQ08iye+Mk4zNZZGdjfs06bVTr+DBSlA66Q2VEcMki/cUCP4SercQ== +call-bind@^1.0.0, call-bind@^1.0.2, call-bind@^1.0.5, call-bind@^1.0.6, call-bind@^1.0.7: + version "1.0.7" + resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.7.tgz#06016599c40c56498c18769d2730be242b6fa3b9" + integrity sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w== dependencies: + es-define-property "^1.0.0" + es-errors "^1.3.0" function-bind "^1.1.2" - get-intrinsic "^1.2.1" - set-function-length "^1.1.1" + get-intrinsic "^1.2.4" + set-function-length "^1.2.1" caller-callsite@^4.1.0: version "4.1.0" - resolved "https://registry.yarnpkg.com/caller-callsite/-/caller-callsite-4.1.0.tgz#3e33cb1d910e7b09332d59a3503b9af7462f7295" + resolved "https://registry.npmjs.org/caller-callsite/-/caller-callsite-4.1.0.tgz" integrity sha512-99nnnGlJexTc41xwQTr+mWl15OI5PPczUJzM4YRE7QjkefMKCXGa5gfQjCOuVrD+1TjI/fevIDHg2nz3iYN5Ig== dependencies: callsites "^3.1.0" caller-path@^3.0.1: version "3.0.1" - resolved "https://registry.yarnpkg.com/caller-path/-/caller-path-3.0.1.tgz#bc932ecec3f943e10c2f8922146e23b132f932e4" + resolved "https://registry.npmjs.org/caller-path/-/caller-path-3.0.1.tgz" integrity sha512-fhmztL4wURO/BzwJUJ4aVRdnKEFskPBbrJ8fNgl7XdUiD1ygzzlt+nhPgUBSRq2ciEVubo6x+W8vJQzm55QLLQ== dependencies: caller-callsite "^4.1.0" callsites@^3.0.0, callsites@^3.1.0: version "3.1.0" - resolved "https://registry.yarnpkg.com/callsites/-/callsites-3.1.0.tgz#b3630abd8943432f54b3f0519238e33cd7df2f73" + resolved "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz" integrity sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ== camelcase@^5.0.0, camelcase@^5.3.1: version "5.3.1" - resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-5.3.1.tgz#e3c9b31569e106811df242f715725a1f4c494320" + resolved "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz" integrity sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg== camelcase@^6.0.0: version "6.3.0" - resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-6.3.0.tgz#5685b95eb209ac9c0c177467778c9c84df58ba9a" + resolved "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz" integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA== -caniuse-lite@^1.0.30001400: - version "1.0.30001412" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001412.tgz#30f67d55a865da43e0aeec003f073ea8764d5d7c" - integrity sha512-+TeEIee1gS5bYOiuf+PS/kp2mrXic37Hl66VY6EAfxasIk5fELTktK2oOezYed12H8w7jt3s512PpulQidPjwA== +caniuse-lite@^1.0.30001565: + version "1.0.30001566" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001566.tgz#61a8e17caf3752e3e426d4239c549ebbb37fef0d" + integrity "sha1-YajhfK83UuPkJtQjnFSeu7N/7w0= sha512-ggIhCsTxmITBAMmK8yZjEhCO5/47jKXPu6Dha/wuCS4JePVL+3uiDEBuhu2aIoT+bqTOR8L76Ip1ARL9xYsEJA==" cardinal@^2.1.1: version "2.1.1" - resolved "https://registry.yarnpkg.com/cardinal/-/cardinal-2.1.1.tgz#7cc1055d822d212954d07b085dea251cc7bc5505" + resolved "https://registry.npmjs.org/cardinal/-/cardinal-2.1.1.tgz" integrity sha512-JSr5eOgoEymtYHBjNWyjrMqet9Am2miJhlfKNdqLp6zoeAh0KN5dRAcxlecj5mAJrmQomgiOBj35xHLrFjqBpw== dependencies: ansicolors "~0.3.2" @@ -1298,8 +1437,8 @@ cardinal@^2.1.1: chai@^4.3.7: version "4.3.7" - resolved "https://registry.yarnpkg.com/chai/-/chai-4.3.7.tgz#ec63f6df01829088e8bf55fca839bcd464a8ec51" - integrity sha512-HLnAzZ2iupm25PlN0xFreAlBA5zaBSv3og0DdeGA4Ar6h6rJ3A0rolRUKJhSF2V10GZKDgWF/VmAEsNWjCRB+A== + resolved "https://registry.npmjs.org/chai/-/chai-4.3.7.tgz" + integrity "sha1-7GP23wGCkIjov1X8qDm81GSo7FE= sha512-HLnAzZ2iupm25PlN0xFreAlBA5zaBSv3og0DdeGA4Ar6h6rJ3A0rolRUKJhSF2V10GZKDgWF/VmAEsNWjCRB+A==" dependencies: assertion-error "^1.1.0" check-error "^1.0.2" @@ -1309,9 +1448,9 @@ chai@^4.3.7: pathval "^1.1.1" type-detect "^4.0.5" -chalk@^2.0.0: +chalk@^2.4.2: version "2.4.2" - resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" + resolved "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz" integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== dependencies: ansi-styles "^3.2.1" @@ -1320,7 +1459,7 @@ chalk@^2.0.0: chalk@^3.0.0: version "3.0.0" - resolved "https://registry.yarnpkg.com/chalk/-/chalk-3.0.0.tgz#3f73c2bf526591f574cc492c51e2456349f844e4" + resolved "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz" integrity sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg== dependencies: ansi-styles "^4.1.0" @@ -1328,7 +1467,7 @@ chalk@^3.0.0: chalk@^4.0.0, chalk@^4.1.0: version "4.1.2" - resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01" + resolved "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz" integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== dependencies: ansi-styles "^4.1.0" @@ -1336,39 +1475,24 @@ chalk@^4.0.0, chalk@^4.1.0: chalk@^5.3.0: version "5.3.0" - resolved "https://registry.yarnpkg.com/chalk/-/chalk-5.3.0.tgz#67c20a7ebef70e7f3970a01f90fa210cb6860385" - integrity sha512-dLitG79d+GV1Nb/VYcCDFivJeK1hiukt9QjRNVOsUtTy1rR1YJsmpGGTZ3qJos+uw7WmWF4wUwBd9jxjocFC2w== + resolved "https://registry.npmjs.org/chalk/-/chalk-5.3.0.tgz" + integrity "sha1-Z8IKfr73Dn85cKAfkPohDLaGA4U= sha512-dLitG79d+GV1Nb/VYcCDFivJeK1hiukt9QjRNVOsUtTy1rR1YJsmpGGTZ3qJos+uw7WmWF4wUwBd9jxjocFC2w==" check-error@^1.0.2: version "1.0.2" - resolved "https://registry.yarnpkg.com/check-error/-/check-error-1.0.2.tgz#574d312edd88bb5dd8912e9286dd6c0aed4aac82" + resolved "https://registry.npmjs.org/check-error/-/check-error-1.0.2.tgz" integrity sha512-BrgHpW9NURQgzoNyjfq0Wu6VFO6D7IZEmJNdtgNqpzGG8RuNFHt2jQxWlAs4HMe119chBnv+34syEZtc6IhLtA== checksum@^1.0.0: version "1.0.0" - resolved "https://registry.yarnpkg.com/checksum/-/checksum-1.0.0.tgz#39d9b5eef273a6a53203900f6780ee8881ab39e5" - integrity sha512-68bHejnM/sBQhjXcXd2mFusICnqAwikZ9RVMURIacWh7moNjgOdHKimS6yk30Np/PwfR00dceY4b1GwWanu5cg== + resolved "https://registry.npmjs.org/checksum/-/checksum-1.0.0.tgz" + integrity "sha1-Odm17vJzpqUyA5APZ4DuiIGrOeU= sha512-68bHejnM/sBQhjXcXd2mFusICnqAwikZ9RVMURIacWh7moNjgOdHKimS6yk30Np/PwfR00dceY4b1GwWanu5cg==" dependencies: optimist "~0.3.5" -chokidar@3.5.1: - version "3.5.1" - resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.5.1.tgz#ee9ce7bbebd2b79f49f304799d5468e31e14e68a" - integrity sha512-9+s+Od+W0VJJzawDma/gvBNQqkTiqYTWLuZoyAsivsI4AaWTCzHG06/TMjsf1cYe9Cb97UCEhjz7HvnPk2p/tw== - dependencies: - anymatch "~3.1.1" - braces "~3.0.2" - glob-parent "~5.1.0" - is-binary-path "~2.1.0" - is-glob "~4.0.1" - normalize-path "~3.0.0" - readdirp "~3.5.0" - optionalDependencies: - fsevents "~2.3.1" - -chokidar@^3.3.0: +chokidar@3.5.3, chokidar@^3.3.0: version "3.5.3" - resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.5.3.tgz#1cf37c8707b932bd1af1ae22c0432e2acd1903bd" + resolved "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz" integrity sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw== dependencies: anymatch "~3.1.2" @@ -1383,34 +1507,34 @@ chokidar@^3.3.0: ci-info@^2.0.0: version "2.0.0" - resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-2.0.0.tgz#67a9e964be31a51e15e5010d58e6f12834002f46" + resolved "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz" integrity sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ== cjs-module-lexer@^1.2.2: version "1.2.3" - resolved "https://registry.yarnpkg.com/cjs-module-lexer/-/cjs-module-lexer-1.2.3.tgz#6c370ab19f8a3394e318fe682686ec0ac684d107" - integrity sha512-0TNiGstbQmCFwt4akjjBg5pLRTSyj/PkWQ1ZoO2zntmg9yLqSRxwEa4iCfQLGjqhiqBfOJa7W/E8wfGrTDmlZQ== + resolved "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.2.3.tgz" + integrity "sha1-bDcKsZ+KM5TjGP5oJobsCsaE0Qc= sha512-0TNiGstbQmCFwt4akjjBg5pLRTSyj/PkWQ1ZoO2zntmg9yLqSRxwEa4iCfQLGjqhiqBfOJa7W/E8wfGrTDmlZQ==" clean-stack@^2.0.0: version "2.2.0" - resolved "https://registry.yarnpkg.com/clean-stack/-/clean-stack-2.2.0.tgz#ee8472dbb129e727b31e8a10a427dee9dfe4008b" + resolved "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz" integrity sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A== cli-boxes@^2.2.0: version "2.2.1" - resolved "https://registry.yarnpkg.com/cli-boxes/-/cli-boxes-2.2.1.tgz#ddd5035d25094fce220e9cab40a45840a440318f" + resolved "https://registry.npmjs.org/cli-boxes/-/cli-boxes-2.2.1.tgz" integrity sha512-y4coMcylgSCdVinjiDBuR8PCC2bLjyGTwEmPb9NHR/QaNU6EUOXcTY/s6VjGMD6ENSEaeQYHCY0GNGS5jfMwPw== cli-cursor@^3.1.0: version "3.1.0" - resolved "https://registry.yarnpkg.com/cli-cursor/-/cli-cursor-3.1.0.tgz#264305a7ae490d1d03bf0c9ba7c925d1753af307" + resolved "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz" integrity sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw== dependencies: restore-cursor "^3.1.0" cli-table3@^0.5.1: version "0.5.1" - resolved "https://registry.yarnpkg.com/cli-table3/-/cli-table3-0.5.1.tgz#0252372d94dfc40dbd8df06005f48f31f656f202" + resolved "https://registry.npmjs.org/cli-table3/-/cli-table3-0.5.1.tgz" integrity sha512-7Qg2Jrep1S/+Q3EceiZtQcDPWxhAvBw+ERf1162v4sikJrvojMHFqXt8QIVha8UlH9rgU0BeWPytZ9/TzYqlUw== dependencies: object-assign "^4.1.0" @@ -1420,8 +1544,8 @@ cli-table3@^0.5.1: cli-table3@^0.6.3: version "0.6.3" - resolved "https://registry.yarnpkg.com/cli-table3/-/cli-table3-0.6.3.tgz#61ab765aac156b52f222954ffc607a6f01dbeeb2" - integrity sha512-w5Jac5SykAeZJKntOxJCrm63Eg5/4dhMWIcuTbo9rpE+brgaSZo0RuNJZeOyMgsUdhDeojvgyQLmjI+K50ZGyg== + resolved "https://registry.npmjs.org/cli-table3/-/cli-table3-0.6.3.tgz" + integrity "sha1-Yat2WqwVa1LyIpVP/GB6bwHb7rI= sha512-w5Jac5SykAeZJKntOxJCrm63Eg5/4dhMWIcuTbo9rpE+brgaSZo0RuNJZeOyMgsUdhDeojvgyQLmjI+K50ZGyg==" dependencies: string-width "^4.2.0" optionalDependencies: @@ -1429,7 +1553,7 @@ cli-table3@^0.6.3: cli-truncate@^2.1.0: version "2.1.0" - resolved "https://registry.yarnpkg.com/cli-truncate/-/cli-truncate-2.1.0.tgz#c39e28bf05edcde5be3b98992a22deed5a2b93c7" + resolved "https://registry.npmjs.org/cli-truncate/-/cli-truncate-2.1.0.tgz" integrity sha512-n8fOixwDD6b/ObinzTrp1ZKFzbgvKZvuz/TvejnLn1aQfC6r52XEx85FmuC+3HI+JM7coBRXUvNqEU2PHVrHpg== dependencies: slice-ansi "^3.0.0" @@ -1437,7 +1561,7 @@ cli-truncate@^2.1.0: cliui@^6.0.0: version "6.0.0" - resolved "https://registry.yarnpkg.com/cliui/-/cliui-6.0.0.tgz#511d702c0c4e41ca156d7d0e96021f23e13225b1" + resolved "https://registry.npmjs.org/cliui/-/cliui-6.0.0.tgz" integrity sha512-t6wbgtoCXvAzst7QgXxJYqPt0usEfbgQdftEPbLL/cvv6HPE5VgvqCuAIDR0NgU52ds6rFwqrgakNLrHEjCbrQ== dependencies: string-width "^4.2.0" @@ -1446,7 +1570,7 @@ cliui@^6.0.0: cliui@^7.0.2, cliui@^7.0.4: version "7.0.4" - resolved "https://registry.yarnpkg.com/cliui/-/cliui-7.0.4.tgz#a0265ee655476fc807aea9df3df8df7783808b4f" + resolved "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz" integrity sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ== dependencies: string-width "^4.2.0" @@ -1455,80 +1579,80 @@ cliui@^7.0.2, cliui@^7.0.4: clone@^2.1.2: version "2.1.2" - resolved "https://registry.yarnpkg.com/clone/-/clone-2.1.2.tgz#1b7f4b9f591f1e8f83670401600345a02887435f" + resolved "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz" integrity sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w== code-excerpt@^3.0.0: version "3.0.0" - resolved "https://registry.yarnpkg.com/code-excerpt/-/code-excerpt-3.0.0.tgz#fcfb6748c03dba8431c19f5474747fad3f250f10" + resolved "https://registry.npmjs.org/code-excerpt/-/code-excerpt-3.0.0.tgz" integrity sha512-VHNTVhd7KsLGOqfX3SyeO8RyYPMp1GJOg194VITk04WMYCv4plV68YWe6TJZxd9MhobjtpMRnVky01gqZsalaw== dependencies: convert-to-spaces "^1.0.1" color-convert@^1.9.0: version "1.9.3" - resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8" + resolved "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz" integrity sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg== dependencies: color-name "1.1.3" color-convert@^2.0.1: version "2.0.1" - resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-2.0.1.tgz#72d3a68d598c9bdb3af2ad1e84f21d896abd4de3" + resolved "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz" integrity sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ== dependencies: color-name "~1.1.4" color-name@1.1.3: version "1.1.3" - resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25" + resolved "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz" integrity sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw== color-name@~1.1.4: version "1.1.4" - resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" + resolved "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz" integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== color-support@^1.1.0, color-support@^1.1.1: version "1.1.3" - resolved "https://registry.yarnpkg.com/color-support/-/color-support-1.1.3.tgz#93834379a1cc9a0c61f82f52f0d04322251bd5a2" + resolved "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz" integrity sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg== colorette@2.0.19: version "2.0.19" - resolved "https://registry.yarnpkg.com/colorette/-/colorette-2.0.19.tgz#cdf044f47ad41a0f4b56b3a0d5b4e6e1a2d5a798" + resolved "https://registry.npmjs.org/colorette/-/colorette-2.0.19.tgz" integrity sha512-3tlv/dIP7FWvj3BsbHrGLJ6l/oKh1O3TcgBqMn+yyCagOxc23fyzDS6HypQbgxWbkpDnf52p1LuR4eWDQ/K9WQ== colors@^1.1.2: version "1.4.0" - resolved "https://registry.yarnpkg.com/colors/-/colors-1.4.0.tgz#c50491479d4c1bdaed2c9ced32cf7c7dc2360f78" + resolved "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz" integrity sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA== -combined-stream@^1.0.6: +combined-stream@^1.0.6, combined-stream@^1.0.8: version "1.0.8" - resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" + resolved "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz" integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg== dependencies: delayed-stream "~1.0.0" commander@^9.1.0: version "9.5.0" - resolved "https://registry.yarnpkg.com/commander/-/commander-9.5.0.tgz#bc08d1eb5cedf7ccb797a96199d41c7bc3e60d30" + resolved "https://registry.npmjs.org/commander/-/commander-9.5.0.tgz" integrity sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ== commondir@^1.0.1: version "1.0.1" - resolved "https://registry.yarnpkg.com/commondir/-/commondir-1.0.1.tgz#ddd800da0c66127393cca5950ea968a3aaf1253b" + resolved "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz" integrity sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg== concat-map@0.0.1: version "0.0.1" - resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" + resolved "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz" integrity sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg== concat-stream@^1.5.2: version "1.6.2" - resolved "https://registry.yarnpkg.com/concat-stream/-/concat-stream-1.6.2.tgz#904bdf194cd3122fc675c77fc4ac3d4ff0fd1a34" + resolved "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz" integrity sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw== dependencies: buffer-from "^1.0.0" @@ -1538,56 +1662,64 @@ concat-stream@^1.5.2: content-disposition@0.5.4: version "0.5.4" - resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.4.tgz#8b82b4efac82512a02bb0b1dcec9d2c5e8eb5bfe" + resolved "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz" integrity sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ== dependencies: safe-buffer "5.2.1" -content-type@~1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.4.tgz#e138cc75e040c727b1966fe5e5f8c9aee256fe3b" - integrity sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA== - -content-type@~1.0.5: +content-type@~1.0.4, content-type@~1.0.5: version "1.0.5" - resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.5.tgz#8b773162656d1d1086784c8f23a54ce6d73d7918" - integrity sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA== + resolved "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz" + integrity "sha1-i3cxYmVtHRCGeEyPI6VM5tc9eRg= sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==" convert-source-map@^1.7.0: version "1.8.0" - resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-1.8.0.tgz#f3373c32d21b4d780dd8004514684fb791ca4369" + resolved "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.8.0.tgz" integrity sha512-+OQdjP49zViI/6i7nIJpA8rAl4sV/JdPfU9nZs3VqOwGIgizICvuN2ru6fMd+4llL0tar18UYJXfZ/TWtmhUjA== dependencies: safe-buffer "~5.1.1" +convert-source-map@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-2.0.0.tgz#4b560f649fc4e918dd0ab75cf4961e8bc882d82a" + integrity sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg== + convert-to-spaces@^1.0.1: version "1.0.2" - resolved "https://registry.yarnpkg.com/convert-to-spaces/-/convert-to-spaces-1.0.2.tgz#7e3e48bbe6d997b1417ddca2868204b4d3d85715" + resolved "https://registry.npmjs.org/convert-to-spaces/-/convert-to-spaces-1.0.2.tgz" integrity sha512-cj09EBuObp9gZNQCzc7hByQyrs6jVGE+o9kSJmeUoj+GiPiJvi5LYqEH/Hmme4+MTLHM+Ejtq+FChpjjEnsPdQ== cookie-signature@1.0.6: version "1.0.6" - resolved "https://registry.yarnpkg.com/cookie-signature/-/cookie-signature-1.0.6.tgz#e303a882b342cc3ee8ca513a79999734dab3ae2c" + resolved "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz" integrity sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ== -cookie@0.5.0: - version "0.5.0" - resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.5.0.tgz#d1f5d71adec6558c58f389987c366aa47e994f8b" - integrity sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw== +cookie@0.7.1: + version "0.7.1" + resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.7.1.tgz#2f73c42142d5d5cf71310a74fc4ae61670e5dbc9" + integrity sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w== core-util-is@~1.0.0: version "1.0.3" - resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.3.tgz#a6042d3634c2b27e9328f837b965fac83808db85" + resolved "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz" integrity sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ== +cors@^2.8.5: + version "2.8.5" + resolved "https://registry.yarnpkg.com/cors/-/cors-2.8.5.tgz#eac11da51592dd86b9f06f6e7ac293b3df875d29" + integrity sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g== + dependencies: + object-assign "^4" + vary "^1" + cross-argv@^1.0.0: version "1.0.0" - resolved "https://registry.yarnpkg.com/cross-argv/-/cross-argv-1.0.0.tgz#e7221e9ff73092a80496c699c8c45efb20f6486c" + resolved "https://registry.npmjs.org/cross-argv/-/cross-argv-1.0.0.tgz" integrity sha512-uAVe/bgNHlPdP1VE4Sk08u9pAJ7o1x/tVQtX77T5zlhYhuwOWtVkPBEtHdvF5cq48VzeCG5i1zN4dQc8pwLYrw== cross-spawn@^7.0.0, cross-spawn@^7.0.2, cross-spawn@^7.0.3: version "7.0.3" - resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" + resolved "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz" integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w== dependencies: path-key "^3.1.0" @@ -1596,88 +1728,98 @@ cross-spawn@^7.0.0, cross-spawn@^7.0.2, cross-spawn@^7.0.3: crypto-randomuuid@^1.0.0: version "1.0.0" - resolved "https://registry.yarnpkg.com/crypto-randomuuid/-/crypto-randomuuid-1.0.0.tgz#acf583e5e085e867ae23e107ff70279024f9e9e7" + resolved "https://registry.npmjs.org/crypto-randomuuid/-/crypto-randomuuid-1.0.0.tgz" integrity sha512-/RC5F4l1SCqD/jazwUF6+t34Cd8zTSAGZ7rvvZu1whZUhD2a5MOGKjSGowoGcpj/cbVZk1ZODIooJEQQq3nNAA== csstype@^3.0.2: - version "3.1.1" - resolved "https://registry.yarnpkg.com/csstype/-/csstype-3.1.1.tgz#841b532c45c758ee546a11d5bd7b7b473c8c30b9" - integrity sha512-DJR/VvkAvSZW9bTouZue2sSxDwdTN92uHjqeKVm+0dAqdfNykRzQ95tay8aXMBAAPpUiq4Qcug2L7neoRh2Egw== + version "3.1.3" + resolved "https://registry.yarnpkg.com/csstype/-/csstype-3.1.3.tgz#d80ff294d114fb0e6ac500fbf85b60137d7eff81" + integrity "sha1-2A/ylNEU+w5qxQD7+FtgE31+/4E= sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==" -dc-polyfill@^0.1.2: - version "0.1.2" - resolved "https://registry.yarnpkg.com/dc-polyfill/-/dc-polyfill-0.1.2.tgz#99a2f120759317b9976999aa715183a1c44b1327" - integrity sha512-AJ4TWwkeOKF7+Wj301wdyK8L0D9SE8Fr7+eMein8UP8+Iyb1xuL3rXWXavsTEM1+vOqDLciYho4cpsvNY0RDGQ== +data-view-buffer@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/data-view-buffer/-/data-view-buffer-1.0.1.tgz#8ea6326efec17a2e42620696e671d7d5a8bc66b2" + integrity sha512-0lht7OugA5x3iJLOWFhWK/5ehONdprk0ISXqVFn/NFrDu+cuc8iADFrGQz5BnRK7LLU3JmkbXSxaqX+/mXYtUA== + dependencies: + call-bind "^1.0.6" + es-errors "^1.3.0" + is-data-view "^1.0.1" + +data-view-byte-length@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/data-view-byte-length/-/data-view-byte-length-1.0.1.tgz#90721ca95ff280677eb793749fce1011347669e2" + integrity sha512-4J7wRJD3ABAzr8wP+OcIcqq2dlUKp4DVflx++hs5h5ZKydWMI6/D/fAot+yh6g2tHh8fLFTvNOaVN357NvSrOQ== + dependencies: + call-bind "^1.0.7" + es-errors "^1.3.0" + is-data-view "^1.0.1" -debug@2.6.9, debug@^2.6.9: +data-view-byte-offset@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/data-view-byte-offset/-/data-view-byte-offset-1.0.0.tgz#5e0bbfb4828ed2d1b9b400cd8a7d119bca0ff18a" + integrity sha512-t/Ygsytq+R995EJ5PZlD4Cu56sWa8InXySaViRzw9apusqsOO2bQP+SbYzAhR0pFKoB+43lYy8rWban9JSuXnA== + dependencies: + call-bind "^1.0.6" + es-errors "^1.3.0" + is-data-view "^1.0.1" + +dc-polyfill@^0.1.4: + version "0.1.4" + resolved "https://registry.yarnpkg.com/dc-polyfill/-/dc-polyfill-0.1.4.tgz#4118cec81a8fab9a5729c41c285c715ffa42495a" + integrity sha512-8iwEduR2jR9wWYggeaYtYZWRiUe3XZPyAQtMTL1otv8X3kfR8xUIVb4l5awHEeyDrH6Je7N324lKzMKlMMN6Yw== + +debug@2.6.9: version "2.6.9" - resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" + resolved "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz" integrity sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA== dependencies: ms "2.0.0" -debug@4.3.1: - version "4.3.1" - resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.1.tgz#f0d229c505e0c6d8c49ac553d1b13dc183f6b2ee" - integrity sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ== +debug@4.3.3: + version "4.3.3" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.3.tgz#04266e0b70a98d4462e6e288e38259213332b664" + integrity sha512-/zxw5+vh1Tfv+4Qn7a5nsbcJKPaSvCDhojn6FEl9vupwK2VCSDtEiEtqr8DFtzYFOdz63LBkxec7DYuc2jon6Q== dependencies: ms "2.1.2" -debug@4.3.4, debug@^4.1.0, debug@^4.1.1, debug@^4.3.2: +debug@4.3.4: version "4.3.4" - resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865" + resolved "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz" integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== dependencies: ms "2.1.2" debug@^3.2.7: version "3.2.7" - resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.7.tgz#72580b7e9145fb39b6676f9c5e5fb100b934179a" + resolved "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz" integrity sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ== dependencies: ms "^2.1.1" +debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2: + version "4.3.5" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.5.tgz#e83444eceb9fedd4a1da56d671ae2446a01a6e1e" + integrity sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg== + dependencies: + ms "2.1.2" + decamelize@^1.2.0: version "1.2.0" - resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" + resolved "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz" integrity sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA== decamelize@^4.0.0: version "4.0.0" - resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-4.0.0.tgz#aa472d7bf660eb15f3494efd531cab7f2a709837" + resolved "https://registry.npmjs.org/decamelize/-/decamelize-4.0.0.tgz" integrity sha512-9iE1PgSik9HeIIw2JO94IidnE3eBoQrFJ3w7sFuzSX4DpmZ3v5sZpUiV5Swcf6mQEF+Y0ru8Neo+p+nyh2J+hQ== deep-eql@^4.1.2: version "4.1.3" - resolved "https://registry.yarnpkg.com/deep-eql/-/deep-eql-4.1.3.tgz#7c7775513092f7df98d8df9996dd085eb668cc6d" - integrity sha512-WaEtAOpRA1MQ0eohqZjpGD8zdI0Ovsm8mmFhaDN8dvDZzyoUMcYDnf5Y6iu7HTXxf8JDS23qWa4a+hKCDyOPzw== + resolved "https://registry.npmjs.org/deep-eql/-/deep-eql-4.1.3.tgz" + integrity "sha1-fHd1UTCS99+Y2N+Zlt0IXrZozG0= sha512-WaEtAOpRA1MQ0eohqZjpGD8zdI0Ovsm8mmFhaDN8dvDZzyoUMcYDnf5Y6iu7HTXxf8JDS23qWa4a+hKCDyOPzw==" dependencies: type-detect "^4.0.0" -deep-equal@^2.2.2: - version "2.2.2" - resolved "https://registry.yarnpkg.com/deep-equal/-/deep-equal-2.2.2.tgz#9b2635da569a13ba8e1cc159c2f744071b115daa" - integrity sha512-xjVyBf0w5vH0I42jdAZzOKVldmPgSulmiyPRywoyq7HXC9qdgo17kxJE+rdnif5Tz6+pIrpJI8dCpMNLIGkUiA== - dependencies: - array-buffer-byte-length "^1.0.0" - call-bind "^1.0.2" - es-get-iterator "^1.1.3" - get-intrinsic "^1.2.1" - is-arguments "^1.1.1" - is-array-buffer "^3.0.2" - is-date-object "^1.0.5" - is-regex "^1.1.4" - is-shared-array-buffer "^1.0.2" - isarray "^2.0.5" - object-is "^1.1.5" - object-keys "^1.1.1" - object.assign "^4.1.4" - regexp.prototype.flags "^1.5.0" - side-channel "^1.0.4" - which-boxed-primitive "^1.0.2" - which-collection "^1.0.1" - which-typed-array "^1.1.9" - deep-is@^0.1.3: version "0.1.4" resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.4.tgz#a6f2dce612fadd2ef1f519b73551f17e85199831" @@ -1685,44 +1827,32 @@ deep-is@^0.1.3: default-require-extensions@^3.0.0: version "3.0.0" - resolved "https://registry.yarnpkg.com/default-require-extensions/-/default-require-extensions-3.0.0.tgz#e03f93aac9b2b6443fc52e5e4a37b3ad9ad8df96" + resolved "https://registry.npmjs.org/default-require-extensions/-/default-require-extensions-3.0.0.tgz" integrity sha512-ek6DpXq/SCpvjhpFsLFRVtIxJCRw6fUR42lYMVZuUMK7n8eMz4Uh5clckdBjEpLhn/gEBZo7hDJnJcwdKLKQjg== dependencies: strip-bom "^4.0.0" -define-data-property@^1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/define-data-property/-/define-data-property-1.1.1.tgz#c35f7cd0ab09883480d12ac5cb213715587800b3" - integrity sha512-E7uGkTzkk1d0ByLeSc6ZsFS79Axg+m1P/VsgYsxHgiuc3tFSj+MjMIwe90FC4lOAZzNBdY7kkO2P2wKdsQ1vgQ== - dependencies: - get-intrinsic "^1.2.1" - gopd "^1.0.1" - has-property-descriptors "^1.0.0" - -define-properties@^1.1.3, define-properties@^1.1.4: +define-data-property@^1.0.1, define-data-property@^1.1.4: version "1.1.4" - resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.1.4.tgz#0b14d7bd7fbeb2f3572c3a7eda80ea5d57fb05b1" - integrity sha512-uckOqKcfaVvtBdsVkdPv3XjveQJsNQqmhXgRi8uhvWWuPYZCNlzT8qAyblUgNoXdHdjMTzAqeGjAoli8f+bzPA== + resolved "https://registry.yarnpkg.com/define-data-property/-/define-data-property-1.1.4.tgz#894dc141bb7d3060ae4366f6a0107e68fbe48c5e" + integrity sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A== dependencies: - has-property-descriptors "^1.0.0" - object-keys "^1.1.1" + es-define-property "^1.0.0" + es-errors "^1.3.0" + gopd "^1.0.1" -define-properties@^1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.2.0.tgz#52988570670c9eacedd8064f4a990f2405849bd5" - integrity sha512-xvqAVKGfT1+UAvPwKTVw/njhdQ8ZhXK4lI0bCIuCMrp2up9nPnaDftrLtmpTazqd1o+UY4zgzU+avtMbDP+ldA== +define-properties@^1.2.0, define-properties@^1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.2.1.tgz#10781cc616eb951a80a034bafcaa7377f6af2b6c" + integrity sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg== dependencies: + define-data-property "^1.0.1" has-property-descriptors "^1.0.0" object-keys "^1.1.1" -defined@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/defined/-/defined-1.0.1.tgz#c0b9db27bfaffd95d6f61399419b893df0f91ebf" - integrity sha512-hsBd2qSVCRE+5PmNdHt1uzyrFu5d3RwmFDKzyNZMFq/EwDNJF7Ee5+D5oEKF0hU6LhtoUF1macFvOe4AskQC1Q== - delay@^5.0.0: version "5.0.0" - resolved "https://registry.yarnpkg.com/delay/-/delay-5.0.0.tgz#137045ef1b96e5071060dd5be60bf9334436bd1d" + resolved "https://registry.npmjs.org/delay/-/delay-5.0.0.tgz" integrity sha512-ReEBKkIfe4ya47wlPYf/gu5ib6yUG0/Aez0JQZQz94kiWtRQvZIQbTiehsnwHvLSWJnQdhVeqYue7Id1dKr0qw== delayed-stream@~1.0.0: @@ -1732,196 +1862,168 @@ delayed-stream@~1.0.0: depd@2.0.0: version "2.0.0" - resolved "https://registry.yarnpkg.com/depd/-/depd-2.0.0.tgz#b696163cc757560d09cf22cc8fad1571b79e76df" + resolved "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz" integrity sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw== destroy@1.2.0: version "1.2.0" - resolved "https://registry.yarnpkg.com/destroy/-/destroy-1.2.0.tgz#4803735509ad8be552934c67df614f94e66fa015" + resolved "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz" integrity sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg== detect-newline@^3.0.0: version "3.1.0" - resolved "https://registry.yarnpkg.com/detect-newline/-/detect-newline-3.1.0.tgz#576f5dfc63ae1a192ff192d8ad3af6308991b651" - integrity sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA== + resolved "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz" + integrity "sha1-V29d/GOuGhkv8ZLYrTr2MImRtlE= sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==" diff@5.0.0: version "5.0.0" - resolved "https://registry.yarnpkg.com/diff/-/diff-5.0.0.tgz#7ed6ad76d859d030787ec35855f5b1daf31d852b" + resolved "https://registry.npmjs.org/diff/-/diff-5.0.0.tgz" integrity sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w== diff@^4.0.1, diff@^4.0.2: version "4.0.2" - resolved "https://registry.yarnpkg.com/diff/-/diff-4.0.2.tgz#60f3aecb89d5fae520c11aa19efc2bb982aade7d" + resolved "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz" integrity sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A== diff@^5.1.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/diff/-/diff-5.1.0.tgz#bc52d298c5ea8df9194800224445ed43ffc87e40" - integrity sha512-D+mk+qE8VC/PAUrlAU34N+VfXev0ghe5ywmpqrawphmVZc1bEfn56uo9qpyGp1p4xpzOHkSW4ztBd6L7Xx4ACw== - -dir-glob@^3.0.1: - version "3.0.1" - resolved "https://registry.yarnpkg.com/dir-glob/-/dir-glob-3.0.1.tgz#56dbf73d992a4a93ba1584f4534063fd2e41717f" - integrity sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA== - dependencies: - path-type "^4.0.0" + version "5.2.0" + resolved "https://registry.yarnpkg.com/diff/-/diff-5.2.0.tgz#26ded047cd1179b78b9537d5ef725503ce1ae531" + integrity sha512-uIFDxqpRZGZ6ThOk84hEfqWoHx2devRFvpTZcTHur85vImfaxUbTW9Ryh4CpCuDnToOP1CEtXKIgytHBPVff5A== doctrine@^2.1.0: version "2.1.0" - resolved "https://registry.yarnpkg.com/doctrine/-/doctrine-2.1.0.tgz#5cd01fc101621b42c4cd7f5d1a66243716d3f39d" + resolved "https://registry.npmjs.org/doctrine/-/doctrine-2.1.0.tgz" integrity sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw== dependencies: esutils "^2.0.2" doctrine@^3.0.0: version "3.0.0" - resolved "https://registry.yarnpkg.com/doctrine/-/doctrine-3.0.0.tgz#addebead72a6574db783639dc87a121773973961" + resolved "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz" integrity sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w== dependencies: esutils "^2.0.2" dotenv@16.3.1: version "16.3.1" - resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.3.1.tgz#369034de7d7e5b120972693352a3bf112172cc3e" - integrity sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ== - -dotignore@^0.1.2: - version "0.1.2" - resolved "https://registry.yarnpkg.com/dotignore/-/dotignore-0.1.2.tgz#f942f2200d28c3a76fbdd6f0ee9f3257c8a2e905" - integrity sha512-UGGGWfSauusaVJC+8fgV+NVvBXkCTmVv7sk6nojDZZvuOUNGUy0Zk4UpHQD6EDjS0jpBwcACvH4eofvyzBcRDw== - dependencies: - minimatch "^3.0.4" + resolved "https://registry.npmjs.org/dotenv/-/dotenv-16.3.1.tgz" + integrity "sha1-NpA03n1+WxIJcmkzUqO/ESFyzD4= sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ==" ee-first@1.1.1: version "1.1.1" - resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d" + resolved "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz" integrity sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow== -electron-to-chromium@^1.4.251: - version "1.4.264" - resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.264.tgz#2f68a062c38b7a04bf57f3e6954b868672fbdcd3" - integrity sha512-AZ6ZRkucHOQT8wke50MktxtmcWZr67kE17X/nAXFf62NIdMdgY6xfsaJD5Szoy84lnkuPWH+4tTNE3s2+bPCiw== +electron-to-chromium@^1.4.601: + version "1.4.608" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.608.tgz#ff567c51dde4892ae330860c7d9f19571e9e1d69" + integrity "sha1-/1Z8Ud3kiSrjMIYMfZ8ZVx6eHWk= sha512-J2f/3iIIm3Mo0npneITZ2UPe4B1bg8fTNrFjD8715F/k1BvbviRuqYGkET1PgprrczXYTHFvotbBOmUp6KE0uA==" emoji-regex@^8.0.0: version "8.0.0" - resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37" + resolved "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz" integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A== encodeurl@~1.0.2: version "1.0.2" - resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59" + resolved "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz" integrity sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w== -es-abstract@^1.19.0, es-abstract@^1.19.1, es-abstract@^1.19.2, es-abstract@^1.19.5: - version "1.20.3" - resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.20.3.tgz#90b143ff7aedc8b3d189bcfac7f1e3e3f81e9da1" - integrity sha512-AyrnaKVpMzljIdwjzrj+LxGmj8ik2LckwXacHqrJJ/jxz6dDDBcZ7I7nlHM0FvEW8MfbWJwOd+yT2XzYW49Frw== - dependencies: - call-bind "^1.0.2" - es-to-primitive "^1.2.1" - function-bind "^1.1.1" - function.prototype.name "^1.1.5" - get-intrinsic "^1.1.3" - get-symbol-description "^1.0.0" - has "^1.0.3" - has-property-descriptors "^1.0.0" - has-symbols "^1.0.3" - internal-slot "^1.0.3" - is-callable "^1.2.6" - is-negative-zero "^2.0.2" - is-regex "^1.1.4" - is-shared-array-buffer "^1.0.2" - is-string "^1.0.7" - is-weakref "^1.0.2" - object-inspect "^1.12.2" - object-keys "^1.1.1" - object.assign "^4.1.4" - regexp.prototype.flags "^1.4.3" - safe-regex-test "^1.0.0" - string.prototype.trimend "^1.0.5" - string.prototype.trimstart "^1.0.5" - unbox-primitive "^1.0.2" - -es-abstract@^1.20.4: - version "1.22.1" - resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.22.1.tgz#8b4e5fc5cefd7f1660f0f8e1a52900dfbc9d9ccc" - integrity sha512-ioRRcXMO6OFyRpyzV3kE1IIBd4WG5/kltnzdxSCqoP8CMGs/Li+M1uF5o7lOkZVFjDs+NLesthnF66Pg/0q0Lw== - dependencies: - array-buffer-byte-length "^1.0.0" - arraybuffer.prototype.slice "^1.0.1" - available-typed-arrays "^1.0.5" - call-bind "^1.0.2" - es-set-tostringtag "^2.0.1" +encodeurl@~2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-2.0.0.tgz#7b8ea898077d7e409d3ac45474ea38eaf0857a58" + integrity sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg== + +es-abstract@^1.22.1, es-abstract@^1.22.3, es-abstract@^1.23.0, es-abstract@^1.23.2: + version "1.23.3" + resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.23.3.tgz#8f0c5a35cd215312573c5a27c87dfd6c881a0aa0" + integrity sha512-e+HfNH61Bj1X9/jLc5v1owaLYuHdeHHSQlkhCBiTK8rBvKaULl/beGMxwrMXjpYrv4pz22BlY570vVePA2ho4A== + dependencies: + array-buffer-byte-length "^1.0.1" + arraybuffer.prototype.slice "^1.0.3" + available-typed-arrays "^1.0.7" + call-bind "^1.0.7" + data-view-buffer "^1.0.1" + data-view-byte-length "^1.0.1" + data-view-byte-offset "^1.0.0" + es-define-property "^1.0.0" + es-errors "^1.3.0" + es-object-atoms "^1.0.0" + es-set-tostringtag "^2.0.3" es-to-primitive "^1.2.1" - function.prototype.name "^1.1.5" - get-intrinsic "^1.2.1" - get-symbol-description "^1.0.0" + function.prototype.name "^1.1.6" + get-intrinsic "^1.2.4" + get-symbol-description "^1.0.2" globalthis "^1.0.3" gopd "^1.0.1" - has "^1.0.3" - has-property-descriptors "^1.0.0" - has-proto "^1.0.1" + has-property-descriptors "^1.0.2" + has-proto "^1.0.3" has-symbols "^1.0.3" - internal-slot "^1.0.5" - is-array-buffer "^3.0.2" + hasown "^2.0.2" + internal-slot "^1.0.7" + is-array-buffer "^3.0.4" is-callable "^1.2.7" - is-negative-zero "^2.0.2" + is-data-view "^1.0.1" + is-negative-zero "^2.0.3" is-regex "^1.1.4" - is-shared-array-buffer "^1.0.2" + is-shared-array-buffer "^1.0.3" is-string "^1.0.7" - is-typed-array "^1.1.10" + is-typed-array "^1.1.13" is-weakref "^1.0.2" - object-inspect "^1.12.3" + object-inspect "^1.13.1" object-keys "^1.1.1" - object.assign "^4.1.4" - regexp.prototype.flags "^1.5.0" - safe-array-concat "^1.0.0" - safe-regex-test "^1.0.0" - string.prototype.trim "^1.2.7" - string.prototype.trimend "^1.0.6" - string.prototype.trimstart "^1.0.6" - typed-array-buffer "^1.0.0" - typed-array-byte-length "^1.0.0" - typed-array-byte-offset "^1.0.0" - typed-array-length "^1.0.4" + object.assign "^4.1.5" + regexp.prototype.flags "^1.5.2" + safe-array-concat "^1.1.2" + safe-regex-test "^1.0.3" + string.prototype.trim "^1.2.9" + string.prototype.trimend "^1.0.8" + string.prototype.trimstart "^1.0.8" + typed-array-buffer "^1.0.2" + typed-array-byte-length "^1.0.1" + typed-array-byte-offset "^1.0.2" + typed-array-length "^1.0.6" unbox-primitive "^1.0.2" - which-typed-array "^1.1.10" + which-typed-array "^1.1.15" -es-get-iterator@^1.1.3: - version "1.1.3" - resolved "https://registry.yarnpkg.com/es-get-iterator/-/es-get-iterator-1.1.3.tgz#3ef87523c5d464d41084b2c3c9c214f1199763d6" - integrity sha512-sPZmqHBe6JIiTfN5q2pEi//TwxmAFHwj/XEuYjTuse78i8KxaqMTTzxPoFKuzRpDpTJ+0NAbpfenkmH2rePtuw== +es-define-property@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/es-define-property/-/es-define-property-1.0.0.tgz#c7faefbdff8b2696cf5f46921edfb77cc4ba3845" + integrity sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ== dependencies: - call-bind "^1.0.2" - get-intrinsic "^1.1.3" - has-symbols "^1.0.3" - is-arguments "^1.1.1" - is-map "^2.0.2" - is-set "^2.0.2" - is-string "^1.0.7" - isarray "^2.0.5" - stop-iteration-iterator "^1.0.0" + get-intrinsic "^1.2.4" -es-set-tostringtag@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/es-set-tostringtag/-/es-set-tostringtag-2.0.1.tgz#338d502f6f674301d710b80c8592de8a15f09cd8" - integrity sha512-g3OMbtlwY3QewlqAiMLI47KywjWZoEytKr8pf6iTC8uJq5bIAH52Z9pnQ8pVL6whrCto53JZDuUIsifGeLorTg== - dependencies: - get-intrinsic "^1.1.3" - has "^1.0.3" - has-tostringtag "^1.0.0" +es-errors@^1.2.1, es-errors@^1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/es-errors/-/es-errors-1.3.0.tgz#05f75a25dab98e4fb1dcd5e1472c0546d5057c8f" + integrity sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw== -es-shim-unscopables@^1.0.0: +es-object-atoms@^1.0.0: version "1.0.0" - resolved "https://registry.yarnpkg.com/es-shim-unscopables/-/es-shim-unscopables-1.0.0.tgz#702e632193201e3edf8713635d083d378e510241" - integrity sha512-Jm6GPcCdC30eMLbZ2x8z2WuRwAws3zTBBKuusffYVUrNj/GVSUAZ+xKMaUpfNDR5IbyNA5LJbaecoUVbmUcB1w== + resolved "https://registry.yarnpkg.com/es-object-atoms/-/es-object-atoms-1.0.0.tgz#ddb55cd47ac2e240701260bc2a8e31ecb643d941" + integrity sha512-MZ4iQ6JwHOBQjahnjwaC1ZtIBH+2ohjamzAO3oaHcXYup7qxjF2fixyH+Q71voWHeOkI2q/TnJao/KfXYIZWbw== + dependencies: + es-errors "^1.3.0" + +es-set-tostringtag@^2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/es-set-tostringtag/-/es-set-tostringtag-2.0.3.tgz#8bb60f0a440c2e4281962428438d58545af39777" + integrity sha512-3T8uNMC3OQTHkFUsFq8r/BwAXLHvU/9O9mE0fBc/MY5iq/8H7ncvO947LmYA6ldWw9Uh8Yhf25zu6n7nML5QWQ== + dependencies: + get-intrinsic "^1.2.4" + has-tostringtag "^1.0.2" + hasown "^2.0.1" + +es-shim-unscopables@^1.0.0, es-shim-unscopables@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/es-shim-unscopables/-/es-shim-unscopables-1.0.2.tgz#1f6942e71ecc7835ed1c8a83006d8771a63a3763" + integrity sha512-J3yBRXCzDu4ULnQwxyToo/OjdMx6akgVC7K6few0a7F/0wLtmKKN7I73AH5T2836UuXRqN7Qg+IIUw/+YJksRw== dependencies: - has "^1.0.3" + hasown "^2.0.0" es-to-primitive@^1.2.1: version "1.2.1" - resolved "https://registry.yarnpkg.com/es-to-primitive/-/es-to-primitive-1.2.1.tgz#e55cd4c9cdc188bcefb03b366c736323fc5c898a" + resolved "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz" integrity sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA== dependencies: is-callable "^1.1.4" @@ -1930,12 +2032,12 @@ es-to-primitive@^1.2.1: es6-error@^4.0.1: version "4.1.1" - resolved "https://registry.yarnpkg.com/es6-error/-/es6-error-4.1.1.tgz#9e3af407459deed47e9a91f9b885a84eb05c561d" + resolved "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz" integrity sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg== esbuild@0.16.12: version "0.16.12" - resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.16.12.tgz#60850b9ad2f103f1c4316be42c34d5023f27378d" + resolved "https://registry.npmjs.org/esbuild/-/esbuild-0.16.12.tgz" integrity sha512-eq5KcuXajf2OmivCl4e89AD3j8fbV+UTE9vczEzq5haA07U9oOTzBWlh3+6ZdjJR7Rz2QfWZ2uxZyhZxBgJ4+g== optionalDependencies: "@esbuild/android-arm" "0.16.12" @@ -1963,294 +2065,283 @@ esbuild@0.16.12: escalade@^3.1.1: version "3.1.1" - resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.1.tgz#d8cfdc7000965c5a0174b4a82eaa5c0552742e40" + resolved "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz" integrity sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw== escape-html@~1.0.3: version "1.0.3" - resolved "https://registry.yarnpkg.com/escape-html/-/escape-html-1.0.3.tgz#0258eae4d3d0c0974de1c169188ef0051d1d1988" + resolved "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz" integrity sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow== escape-string-regexp@4.0.0, escape-string-regexp@^4.0.0: version "4.0.0" - resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz#14ba83a5d373e3d311e5afca29cf5bfad965bf34" + resolved "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz" integrity sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA== escape-string-regexp@^1.0.5: version "1.0.5" - resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4" + resolved "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz" integrity sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg== escape-string-regexp@^2.0.0: version "2.0.0" - resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz#a30304e99daa32e23b2fd20f51babd07cffca344" + resolved "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz" integrity sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w== -eslint-config-standard@^11.0.0-beta.0: - version "11.0.0" - resolved "https://registry.yarnpkg.com/eslint-config-standard/-/eslint-config-standard-11.0.0.tgz#87ee0d3c9d95382dc761958cbb23da9eea31e0ba" - integrity sha512-oDdENzpViEe5fwuRCWla7AXQd++/oyIp8zP+iP9jiUPG6NBj3SHgdgtl/kTn00AjeN+1HNvavTKmYbMo+xMOlw== +eslint-compat-utils@^0.5.1: + version "0.5.1" + resolved "https://registry.yarnpkg.com/eslint-compat-utils/-/eslint-compat-utils-0.5.1.tgz#7fc92b776d185a70c4070d03fd26fde3d59652e4" + integrity sha512-3z3vFexKIEnjHE3zCMRo6fn/e44U7T1khUjg+Hp0ZQMCigh28rALD0nPFBcGZuiLC5rLZa2ubQHDRln09JfU2Q== + dependencies: + semver "^7.5.4" + +eslint-config-standard@^17.1.0: + version "17.1.0" + resolved "https://registry.yarnpkg.com/eslint-config-standard/-/eslint-config-standard-17.1.0.tgz#40ffb8595d47a6b242e07cbfd49dc211ed128975" + integrity sha512-IwHwmaBNtDK4zDHQukFDW5u/aTb8+meQWZvNFWkiGmbWjD6bqyuSSBxxXKkCftCUzc1zwCH2m/baCNDLGmuO5Q== -eslint-import-resolver-node@^0.3.6: - version "0.3.6" - resolved "https://registry.yarnpkg.com/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.6.tgz#4048b958395da89668252001dbd9eca6b83bacbd" - integrity sha512-0En0w03NRVMn9Uiyn8YRPDKvWjxCWkslUEhGNTdGx15RvPJYQ+lbOlqrlNI2vEAs4pDYK4f/HN2TbDmk5TP0iw== +eslint-import-resolver-node@^0.3.9: + version "0.3.9" + resolved "https://registry.yarnpkg.com/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.9.tgz#d4eaac52b8a2e7c3cd1903eb00f7e053356118ac" + integrity sha512-WFj2isz22JahUv+B788TlO3N6zL3nNJGU8CcZbPZvVEkBPaJdCV4vy5wyghty5ROFbCRnm132v8BScu5/1BQ8g== dependencies: debug "^3.2.7" - resolve "^1.20.0" + is-core-module "^2.13.0" + resolve "^1.22.4" -eslint-module-utils@^2.7.3: - version "2.7.4" - resolved "https://registry.yarnpkg.com/eslint-module-utils/-/eslint-module-utils-2.7.4.tgz#4f3e41116aaf13a20792261e61d3a2e7e0583974" - integrity sha512-j4GT+rqzCoRKHwURX7pddtIPGySnX9Si/cgMI5ztrcqOPtk5dDEeZ34CQVPphnqkJytlc97Vuk05Um2mJ3gEQA== +eslint-module-utils@^2.8.0: + version "2.8.1" + resolved "https://registry.yarnpkg.com/eslint-module-utils/-/eslint-module-utils-2.8.1.tgz#52f2404300c3bd33deece9d7372fb337cc1d7c34" + integrity sha512-rXDXR3h7cs7dy9RNpUlQf80nX31XWJEyGq1tRMo+6GsO5VmTe4UTwtmonAD4ZkAsrfMVDA2wlGJ3790Ys+D49Q== dependencies: debug "^3.2.7" -eslint-plugin-es@^4.1.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/eslint-plugin-es/-/eslint-plugin-es-4.1.0.tgz#f0822f0c18a535a97c3e714e89f88586a7641ec9" - integrity sha512-GILhQTnjYE2WorX5Jyi5i4dz5ALWxBIdQECVQavL6s7cI76IZTDWleTHkxz/QT3kvcs2QlGHvKLYsSlPOlPXnQ== +eslint-plugin-es-x@^7.5.0: + version "7.8.0" + resolved "https://registry.yarnpkg.com/eslint-plugin-es-x/-/eslint-plugin-es-x-7.8.0.tgz#a207aa08da37a7923f2a9599e6d3eb73f3f92b74" + integrity sha512-7Ds8+wAAoV3T+LAKeu39Y5BzXCrGKrcISfgKEqTS4BDN8SFEDQd0S43jiQ8vIa3wUKD07qitZdfzlenSi8/0qQ== dependencies: - eslint-utils "^2.0.0" - regexpp "^3.0.0" + "@eslint-community/eslint-utils" "^4.1.2" + "@eslint-community/regexpp" "^4.11.0" + eslint-compat-utils "^0.5.1" -eslint-plugin-import@^2.8.0: - version "2.26.0" - resolved "https://registry.yarnpkg.com/eslint-plugin-import/-/eslint-plugin-import-2.26.0.tgz#f812dc47be4f2b72b478a021605a59fc6fe8b88b" - integrity sha512-hYfi3FXaM8WPLf4S1cikh/r4IxnO6zrhZbEGz2b660EJRbuxgpDS5gkCuYgGWg2xxh2rBuIr4Pvhve/7c31koA== +eslint-plugin-import@^2.29.1: + version "2.29.1" + resolved "https://registry.yarnpkg.com/eslint-plugin-import/-/eslint-plugin-import-2.29.1.tgz#d45b37b5ef5901d639c15270d74d46d161150643" + integrity sha512-BbPC0cuExzhiMo4Ff1BTVwHpjjv28C5R+btTOGaCRC7UEz801up0JadwkeSk5Ued6TG34uaczuVuH6qyy5YUxw== dependencies: - array-includes "^3.1.4" - array.prototype.flat "^1.2.5" - debug "^2.6.9" + array-includes "^3.1.7" + array.prototype.findlastindex "^1.2.3" + array.prototype.flat "^1.3.2" + array.prototype.flatmap "^1.3.2" + debug "^3.2.7" doctrine "^2.1.0" - eslint-import-resolver-node "^0.3.6" - eslint-module-utils "^2.7.3" - has "^1.0.3" - is-core-module "^2.8.1" + eslint-import-resolver-node "^0.3.9" + eslint-module-utils "^2.8.0" + hasown "^2.0.0" + is-core-module "^2.13.1" is-glob "^4.0.3" minimatch "^3.1.2" - object.values "^1.1.5" - resolve "^1.22.0" - tsconfig-paths "^3.14.1" + object.fromentries "^2.0.7" + object.groupby "^1.0.1" + object.values "^1.1.7" + semver "^6.3.1" + tsconfig-paths "^3.15.0" -eslint-plugin-mocha@^10.1.0: - version "10.1.0" - resolved "https://registry.yarnpkg.com/eslint-plugin-mocha/-/eslint-plugin-mocha-10.1.0.tgz#69325414f875be87fb2cb00b2ef33168d4eb7c8d" - integrity sha512-xLqqWUF17llsogVOC+8C6/jvQ+4IoOREbN7ZCHuOHuD6cT5cDD4h7f2LgsZuzMAiwswWE21tO7ExaknHVDrSkw== +eslint-plugin-mocha@^10.4.3: + version "10.4.3" + resolved "https://registry.yarnpkg.com/eslint-plugin-mocha/-/eslint-plugin-mocha-10.4.3.tgz#bf641379d9f1c7d6a84646a3fc1a0baa50da8bfd" + integrity sha512-emc4TVjq5Ht0/upR+psftuz6IBG5q279p+1dSRDeHf+NS9aaerBi3lXKo1SEzwC29hFIW21gO89CEWSvRsi8IQ== dependencies: eslint-utils "^3.0.0" - rambda "^7.1.0" + globals "^13.24.0" + rambda "^7.4.0" -eslint-plugin-n@^15.7.0: - version "15.7.0" - resolved "https://registry.yarnpkg.com/eslint-plugin-n/-/eslint-plugin-n-15.7.0.tgz#e29221d8f5174f84d18f2eb94765f2eeea033b90" - integrity sha512-jDex9s7D/Qial8AGVIHq4W7NswpUD5DPDL2RH8Lzd9EloWUuvUkHfv4FRLMipH5q2UtyurorBkPeNi1wVWNh3Q== +eslint-plugin-n@^16.6.2: + version "16.6.2" + resolved "https://registry.yarnpkg.com/eslint-plugin-n/-/eslint-plugin-n-16.6.2.tgz#6a60a1a376870064c906742272074d5d0b412b0b" + integrity sha512-6TyDmZ1HXoFQXnhCTUjVFULReoBPOAjpuiKELMkeP40yffI/1ZRO+d9ug/VC6fqISo2WkuIBk3cvuRPALaWlOQ== dependencies: + "@eslint-community/eslint-utils" "^4.4.0" builtins "^5.0.1" - eslint-plugin-es "^4.1.0" - eslint-utils "^3.0.0" - ignore "^5.1.1" - is-core-module "^2.11.0" + eslint-plugin-es-x "^7.5.0" + get-tsconfig "^4.7.0" + globals "^13.24.0" + ignore "^5.2.4" + is-builtin-module "^3.2.1" + is-core-module "^2.12.1" minimatch "^3.1.2" - resolve "^1.22.1" - semver "^7.3.8" + resolve "^1.22.2" + semver "^7.5.3" -eslint-plugin-node@^5.2.1: - version "5.2.1" - resolved "https://registry.yarnpkg.com/eslint-plugin-node/-/eslint-plugin-node-5.2.1.tgz#80df3253c4d7901045ec87fa660a284e32bdca29" - integrity sha512-xhPXrh0Vl/b7870uEbaumb2Q+LxaEcOQ3kS1jtIXanBAwpMre1l5q/l2l/hESYJGEFKuI78bp6Uw50hlpr7B+g== - dependencies: - ignore "^3.3.6" - minimatch "^3.0.4" - resolve "^1.3.3" - semver "5.3.0" - -eslint-plugin-promise@^3.6.0: - version "3.8.0" - resolved "https://registry.yarnpkg.com/eslint-plugin-promise/-/eslint-plugin-promise-3.8.0.tgz#65ebf27a845e3c1e9d6f6a5622ddd3801694b621" - integrity sha512-JiFL9UFR15NKpHyGii1ZcvmtIqa3UTwiDAGb8atSffe43qJ3+1czVGN6UtkklpcJ2DVnqvTMzEKRaJdBkAL2aQ== - -eslint-plugin-standard@^3.0.1: - version "3.1.0" - resolved "https://registry.yarnpkg.com/eslint-plugin-standard/-/eslint-plugin-standard-3.1.0.tgz#2a9e21259ba4c47c02d53b2d0c9135d4b1022d47" - integrity sha512-fVcdyuKRr0EZ4fjWl3c+gp1BANFJD1+RaWa2UPYfMZ6jCtp5RG00kSaXnK/dE5sYzt4kaWJ9qdxqUfc0d9kX0w== +eslint-plugin-promise@^6.4.0: + version "6.4.0" + resolved "https://registry.yarnpkg.com/eslint-plugin-promise/-/eslint-plugin-promise-6.4.0.tgz#54926d53c79541efe9cea6ac1d823a58bbed1106" + integrity sha512-/KWWRaD3fGkVCZsdR0RU53PSthFmoHVhZl+y9+6DqeDLSikLdlUVpVEAmI6iCRR5QyOjBYBqHZV/bdv4DJ4Gtw== -eslint-scope@^7.1.1: - version "7.1.1" - resolved "https://registry.yarnpkg.com/eslint-scope/-/eslint-scope-7.1.1.tgz#fff34894c2f65e5226d3041ac480b4513a163642" - integrity sha512-QKQM/UXpIiHcLqJ5AOyIW7XZmzjkzQXYE54n1++wb0u9V/abW3l9uQnxX8Z5Xd18xyKIMTUAyQ0k1e8pz6LUrw== +eslint-scope@^7.2.2: + version "7.2.2" + resolved "https://registry.yarnpkg.com/eslint-scope/-/eslint-scope-7.2.2.tgz#deb4f92563390f32006894af62a22dba1c46423f" + integrity sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg== dependencies: esrecurse "^4.3.0" estraverse "^5.2.0" -eslint-utils@^2.0.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/eslint-utils/-/eslint-utils-2.1.0.tgz#d2de5e03424e707dc10c74068ddedae708741b27" - integrity sha512-w94dQYoauyvlDc43XnGB8lU3Zt713vNChgt4EWwhXAP2XkBvndfxF0AgIqKOOasjPIPzj9JqgwkwbCYD0/V3Zg== - dependencies: - eslint-visitor-keys "^1.1.0" - eslint-utils@^3.0.0: version "3.0.0" - resolved "https://registry.yarnpkg.com/eslint-utils/-/eslint-utils-3.0.0.tgz#8aebaface7345bb33559db0a1f13a1d2d48c3672" + resolved "https://registry.npmjs.org/eslint-utils/-/eslint-utils-3.0.0.tgz" integrity sha512-uuQC43IGctw68pJA1RgbQS8/NP7rch6Cwd4j3ZBtgo4/8Flj4eGE7ZYSZRN3iq5pVUv6GPdW5Z1RFleo84uLDA== dependencies: eslint-visitor-keys "^2.0.0" -eslint-visitor-keys@^1.1.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz#30ebd1ef7c2fdff01c3a4f151044af25fab0523e" - integrity sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ== - eslint-visitor-keys@^2.0.0: version "2.1.0" - resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-2.1.0.tgz#f65328259305927392c938ed44eb0a5c9b2bd303" + resolved "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.1.0.tgz" integrity sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw== -eslint-visitor-keys@^3.3.0: - version "3.3.0" - resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-3.3.0.tgz#f6480fa6b1f30efe2d1968aa8ac745b862469826" - integrity sha512-mQ+suqKJVyeuwGYHAdjMFqjCyfl8+Ldnxuyp3ldiMBFKkvytrXUZWaiPCEav8qDHKty44bD+qV1IP4T+w+xXRA== - -eslint@^8.23.0: - version "8.24.0" - resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.24.0.tgz#489516c927a5da11b3979dbfb2679394523383c8" - integrity sha512-dWFaPhGhTAiPcCgm3f6LI2MBWbogMnTJzFBbhXVRQDJPkr9pGZvVjlVfXd+vyDcWPA2Ic9L2AXPIQM0+vk/cSQ== - dependencies: - "@eslint/eslintrc" "^1.3.2" - "@humanwhocodes/config-array" "^0.10.5" - "@humanwhocodes/gitignore-to-minimatch" "^1.0.2" +eslint-visitor-keys@^3.3.0, eslint-visitor-keys@^3.4.1, eslint-visitor-keys@^3.4.3: + version "3.4.3" + resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz#0cd72fe8550e3c2eae156a96a4dddcd1c8ac5800" + integrity sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag== + +eslint@^8.57.0: + version "8.57.0" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.57.0.tgz#c786a6fd0e0b68941aaf624596fb987089195668" + integrity sha512-dZ6+mexnaTIbSBZWgou51U6OmzIhYM2VcNdtiTtI7qPNZm35Akpr0f6vtw3w1Kmn5PYo+tZVfh13WrhpS6oLqQ== + dependencies: + "@eslint-community/eslint-utils" "^4.2.0" + "@eslint-community/regexpp" "^4.6.1" + "@eslint/eslintrc" "^2.1.4" + "@eslint/js" "8.57.0" + "@humanwhocodes/config-array" "^0.11.14" "@humanwhocodes/module-importer" "^1.0.1" - ajv "^6.10.0" + "@nodelib/fs.walk" "^1.2.8" + "@ungap/structured-clone" "^1.2.0" + ajv "^6.12.4" chalk "^4.0.0" cross-spawn "^7.0.2" debug "^4.3.2" doctrine "^3.0.0" escape-string-regexp "^4.0.0" - eslint-scope "^7.1.1" - eslint-utils "^3.0.0" - eslint-visitor-keys "^3.3.0" - espree "^9.4.0" - esquery "^1.4.0" + eslint-scope "^7.2.2" + eslint-visitor-keys "^3.4.3" + espree "^9.6.1" + esquery "^1.4.2" esutils "^2.0.2" fast-deep-equal "^3.1.3" file-entry-cache "^6.0.1" find-up "^5.0.0" - glob-parent "^6.0.1" - globals "^13.15.0" - globby "^11.1.0" - grapheme-splitter "^1.0.4" + glob-parent "^6.0.2" + globals "^13.19.0" + graphemer "^1.4.0" ignore "^5.2.0" - import-fresh "^3.0.0" imurmurhash "^0.1.4" is-glob "^4.0.0" - js-sdsl "^4.1.4" + is-path-inside "^3.0.3" js-yaml "^4.1.0" json-stable-stringify-without-jsonify "^1.0.1" levn "^0.4.1" lodash.merge "^4.6.2" minimatch "^3.1.2" natural-compare "^1.4.0" - optionator "^0.9.1" - regexpp "^3.2.0" + optionator "^0.9.3" strip-ansi "^6.0.1" - strip-json-comments "^3.1.0" text-table "^0.2.0" esm@^3.2.25: version "3.2.25" - resolved "https://registry.yarnpkg.com/esm/-/esm-3.2.25.tgz#342c18c29d56157688ba5ce31f8431fbb795cc10" + resolved "https://registry.npmjs.org/esm/-/esm-3.2.25.tgz" integrity sha512-U1suiZ2oDVWv4zPO56S0NcR5QriEahGtdN2OR6FiOG4WJvcjBVFB0qI4+eKoWFH483PKGuLuu6V8Z4T5g63UVA== -espree@^9.4.0: - version "9.4.0" - resolved "https://registry.yarnpkg.com/espree/-/espree-9.4.0.tgz#cd4bc3d6e9336c433265fc0aa016fc1aaf182f8a" - integrity sha512-DQmnRpLj7f6TgN/NYb0MTzJXL+vJF9h3pHy4JhCIs3zwcgez8xmGg3sXHcEO97BrmO2OSvCwMdfdlyl+E9KjOw== +espree@^9.6.0, espree@^9.6.1: + version "9.6.1" + resolved "https://registry.yarnpkg.com/espree/-/espree-9.6.1.tgz#a2a17b8e434690a5432f2f8018ce71d331a48c6f" + integrity sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ== dependencies: - acorn "^8.8.0" + acorn "^8.9.0" acorn-jsx "^5.3.2" - eslint-visitor-keys "^3.3.0" + eslint-visitor-keys "^3.4.1" esprima@^4.0.0, esprima@~4.0.0: version "4.0.1" resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.1.tgz#13b04cdb3e6c5d19df91ab6987a8695619b0aa71" integrity sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A== -esquery@^1.4.0: - version "1.4.0" - resolved "https://registry.yarnpkg.com/esquery/-/esquery-1.4.0.tgz#2148ffc38b82e8c7057dfed48425b3e61f0f24a5" - integrity sha512-cCDispWt5vHHtwMY2YrAQ4ibFkAL8RbH5YGBnZBc90MolvvfkkQcJro/aZiAQUlQ3qgrYS6D6v8Gc5G5CQsc9w== +esquery@^1.4.2: + version "1.6.0" + resolved "https://registry.yarnpkg.com/esquery/-/esquery-1.6.0.tgz#91419234f804d852a82dceec3e16cdc22cf9dae7" + integrity sha512-ca9pw9fomFcKPvFLXhBKUK90ZvGibiGOvRJNbjljY7s7uq/5YO4BOzcYtJqExdx99rF6aAcnRxHmcUHcz6sQsg== dependencies: estraverse "^5.1.0" esrecurse@^4.3.0: version "4.3.0" - resolved "https://registry.yarnpkg.com/esrecurse/-/esrecurse-4.3.0.tgz#7ad7964d679abb28bee72cec63758b1c5d2c9921" + resolved "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz" integrity sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag== dependencies: estraverse "^5.2.0" estraverse@^5.1.0, estraverse@^5.2.0: version "5.3.0" - resolved "https://registry.yarnpkg.com/estraverse/-/estraverse-5.3.0.tgz#2eea5290702f26ab8fe5370370ff86c965d21123" + resolved "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz" integrity sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA== esutils@^2.0.2: version "2.0.3" - resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.3.tgz#74d2eb4de0b8da1293711910d50775b9b710ef64" + resolved "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz" integrity sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g== etag@~1.8.1: version "1.8.1" - resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887" + resolved "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz" integrity sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg== event-lite@^0.1.1: version "0.1.2" - resolved "https://registry.yarnpkg.com/event-lite/-/event-lite-0.1.2.tgz#838a3e0fdddef8cc90f128006c8e55a4e4e4c11b" + resolved "https://registry.npmjs.org/event-lite/-/event-lite-0.1.2.tgz" integrity sha512-HnSYx1BsJ87/p6swwzv+2v6B4X+uxUteoDfRxsAb1S1BePzQqOLevVmkdA15GHJVd9A9Ok6wygUR18Hu0YeV9g== events-to-array@^1.0.1: version "1.1.2" - resolved "https://registry.yarnpkg.com/events-to-array/-/events-to-array-1.1.2.tgz#2d41f563e1fe400ed4962fe1a4d5c6a7539df7f6" + resolved "https://registry.npmjs.org/events-to-array/-/events-to-array-1.1.2.tgz" integrity sha512-inRWzRY7nG+aXZxBzEqYKB3HPgwflZRopAjDCHv0whhRx+MTUr1ei0ICZUypdyE0HRm4L2d5VEcIqLD6yl+BFA== events@1.1.1: version "1.1.1" - resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924" - integrity sha512-kEcvvCBByWXGnZy6JUlgAp2gBIUjfCAV6P6TgT1/aaQKcmuAEC4OZTV1I4EWQLz2gxZw76atuVyvHhTxvi0Flw== + resolved "https://registry.npmjs.org/events/-/events-1.1.1.tgz" + integrity "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ= sha512-kEcvvCBByWXGnZy6JUlgAp2gBIUjfCAV6P6TgT1/aaQKcmuAEC4OZTV1I4EWQLz2gxZw76atuVyvHhTxvi0Flw==" -express@^4.18.2: - version "4.18.2" - resolved "https://registry.yarnpkg.com/express/-/express-4.18.2.tgz#3fabe08296e930c796c19e3c516979386ba9fd59" - integrity sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ== +express@^4.17.1, express@^4.18.2: + version "4.21.1" + resolved "https://registry.yarnpkg.com/express/-/express-4.21.1.tgz#9dae5dda832f16b4eec941a4e44aa89ec481b281" + integrity sha512-YSFlK1Ee0/GC8QaO91tHcDxJiE/X4FbpAyQWkxAvG6AXCuR65YzK8ua6D9hvi/TzUfZMpc+BwuM1IPw8fmQBiQ== dependencies: accepts "~1.3.8" array-flatten "1.1.1" - body-parser "1.20.1" + body-parser "1.20.3" content-disposition "0.5.4" content-type "~1.0.4" - cookie "0.5.0" + cookie "0.7.1" cookie-signature "1.0.6" debug "2.6.9" depd "2.0.0" - encodeurl "~1.0.2" + encodeurl "~2.0.0" escape-html "~1.0.3" etag "~1.8.1" - finalhandler "1.2.0" + finalhandler "1.3.1" fresh "0.5.2" http-errors "2.0.0" - merge-descriptors "1.0.1" + merge-descriptors "1.0.3" methods "~1.1.2" on-finished "2.4.1" parseurl "~1.3.3" - path-to-regexp "0.1.7" + path-to-regexp "0.1.10" proxy-addr "~2.0.7" - qs "6.11.0" + qs "6.13.0" range-parser "~1.2.1" safe-buffer "5.2.1" - send "0.18.0" - serve-static "1.15.0" + send "0.19.0" + serve-static "1.16.2" setprototypeof "1.2.0" statuses "2.0.1" type-is "~1.6.18" @@ -2259,23 +2350,12 @@ express@^4.18.2: fast-deep-equal@^3.1.1, fast-deep-equal@^3.1.3: version "3.1.3" - resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz#3a7d56b559d6cbc3eb512325244e619a65c6c525" + resolved "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz" integrity sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q== -fast-glob@^3.2.9: - version "3.2.12" - resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.2.12.tgz#7f39ec99c2e6ab030337142da9e0c18f37afae80" - integrity sha512-DVj4CQIYYow0BlaelwK1pHl5n5cRSJfM60UA0zK891sVInoPri2Ekj7+e1CT3/3qxXenpI+nBBmQAcJPJgaj4w== - dependencies: - "@nodelib/fs.stat" "^2.0.2" - "@nodelib/fs.walk" "^1.2.3" - glob-parent "^5.1.2" - merge2 "^1.3.0" - micromatch "^4.0.4" - fast-json-stable-stringify@^2.0.0: version "2.1.0" - resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633" + resolved "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz" integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw== fast-levenshtein@^2.0.6: @@ -2285,40 +2365,40 @@ fast-levenshtein@^2.0.6: fastq@^1.6.0: version "1.13.0" - resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.13.0.tgz#616760f88a7526bdfc596b7cab8c18938c36b98c" + resolved "https://registry.npmjs.org/fastq/-/fastq-1.13.0.tgz" integrity sha512-YpkpUnK8od0o1hmeSc7UUs/eB/vIPWJYjKck2QKIzAf71Vm1AAQ3EbuZB3g2JIy+pg+ERD0vqI79KyZiB2e2Nw== dependencies: reusify "^1.0.4" file-entry-cache@^6.0.1: version "6.0.1" - resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-6.0.1.tgz#211b2dd9659cb0394b073e7323ac3c933d522027" + resolved "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz" integrity sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg== dependencies: flat-cache "^3.0.4" fill-keys@^1.0.2: version "1.0.2" - resolved "https://registry.yarnpkg.com/fill-keys/-/fill-keys-1.0.2.tgz#9a8fa36f4e8ad634e3bf6b4f3c8882551452eb20" + resolved "https://registry.npmjs.org/fill-keys/-/fill-keys-1.0.2.tgz" integrity sha512-tcgI872xXjwFF4xgQmLxi76GnwJG3g/3isB1l4/G5Z4zrbddGpBjqZCO9oEAcB5wX0Hj/5iQB3toxfO7in1hHA== dependencies: is-object "~1.0.1" merge-descriptors "~1.0.0" -fill-range@^7.0.1: - version "7.0.1" - resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.0.1.tgz#1919a6a7c75fe38b2c7c77e5198535da9acdda40" - integrity sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ== +fill-range@^7.1.1: + version "7.1.1" + resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.1.1.tgz#44265d3cac07e3ea7dc247516380643754a05292" + integrity sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg== dependencies: to-regex-range "^5.0.1" -finalhandler@1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.2.0.tgz#7d23fe5731b207b4640e4fcd00aec1f9207a7b32" - integrity sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg== +finalhandler@1.3.1: + version "1.3.1" + resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.3.1.tgz#0c575f1d1d324ddd1da35ad7ece3df7d19088019" + integrity sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ== dependencies: debug "2.6.9" - encodeurl "~1.0.2" + encodeurl "~2.0.0" escape-html "~1.0.3" on-finished "2.4.1" parseurl "~1.3.3" @@ -2327,7 +2407,7 @@ finalhandler@1.2.0: find-cache-dir@^3.2.0: version "3.3.2" - resolved "https://registry.yarnpkg.com/find-cache-dir/-/find-cache-dir-3.3.2.tgz#b30c5b6eff0730731aea9bbd9dbecbd80256d64b" + resolved "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-3.3.2.tgz" integrity sha512-wXZV5emFEjrridIgED11OoUKLxiYjAcqot/NJdAkOhlJ+vGzwhOAfcG5OX1jP+S0PcjEn8bdMJv+g2jwQ3Onig== dependencies: commondir "^1.0.1" @@ -2336,7 +2416,7 @@ find-cache-dir@^3.2.0: find-up@5.0.0, find-up@^5.0.0: version "5.0.0" - resolved "https://registry.yarnpkg.com/find-up/-/find-up-5.0.0.tgz#4c92819ecb7083561e4f4a240a86be5198f536fc" + resolved "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz" integrity sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng== dependencies: locate-path "^6.0.0" @@ -2344,7 +2424,7 @@ find-up@5.0.0, find-up@^5.0.0: find-up@^4.0.0, find-up@^4.1.0: version "4.1.0" - resolved "https://registry.yarnpkg.com/find-up/-/find-up-4.1.0.tgz#97afe7d6cdc0bc5928584b7c8d7b16e8a9aa5d19" + resolved "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz" integrity sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw== dependencies: locate-path "^5.0.0" @@ -2352,12 +2432,12 @@ find-up@^4.0.0, find-up@^4.1.0: findit@^2.0.0: version "2.0.0" - resolved "https://registry.yarnpkg.com/findit/-/findit-2.0.0.tgz#6509f0126af4c178551cfa99394e032e13a4d56e" + resolved "https://registry.npmjs.org/findit/-/findit-2.0.0.tgz" integrity sha512-ENZS237/Hr8bjczn5eKuBohLgaD0JyUd0arxretR1f9RO46vZHA1b2y0VorgGV3WaOT3c+78P8h7v4JGJ1i/rg== flat-cache@^3.0.4: version "3.0.4" - resolved "https://registry.yarnpkg.com/flat-cache/-/flat-cache-3.0.4.tgz#61b0338302b2fe9f957dcc32fc2a87f1c3048b11" + resolved "https://registry.npmjs.org/flat-cache/-/flat-cache-3.0.4.tgz" integrity sha512-dm9s5Pw7Jc0GvMYbshN6zchCA9RgQlzzEZX3vylR9IqFfS8XciblUXOKfW6SiuJ0e13eDYZoZV5wdrev7P3Nwg== dependencies: flatted "^3.1.0" @@ -2365,29 +2445,29 @@ flat-cache@^3.0.4: flat@^5.0.2: version "5.0.2" - resolved "https://registry.yarnpkg.com/flat/-/flat-5.0.2.tgz#8ca6fe332069ffa9d324c327198c598259ceb241" + resolved "https://registry.npmjs.org/flat/-/flat-5.0.2.tgz" integrity sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ== flatted@^3.1.0: version "3.2.7" - resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.2.7.tgz#609f39207cb614b89d0765b477cb2d437fbf9787" + resolved "https://registry.npmjs.org/flatted/-/flatted-3.2.7.tgz" integrity sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ== -follow-redirects@^1.14.0: - version "1.15.2" - resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.2.tgz#b460864144ba63f2681096f274c4e57026da2c13" - integrity sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA== +follow-redirects@^1.15.6: + version "1.15.6" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.6.tgz#7f815c0cda4249c74ff09e95ef97c23b5fd0399b" + integrity sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA== for-each@^0.3.3: version "0.3.3" - resolved "https://registry.yarnpkg.com/for-each/-/for-each-0.3.3.tgz#69b447e88a0a5d32c3e7084f3f1710034b21376e" + resolved "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz" integrity sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw== dependencies: is-callable "^1.1.3" foreground-child@^2.0.0: version "2.0.0" - resolved "https://registry.yarnpkg.com/foreground-child/-/foreground-child-2.0.0.tgz#71b32800c9f15aa8f2f83f4a6bd9bff35d861a53" + resolved "https://registry.npmjs.org/foreground-child/-/foreground-child-2.0.0.tgz" integrity sha512-dCIq9FpEcyQyXKCkyzmlPTFNgrCzPudOe+mhvJU5zAtlBnGVy2yKxtfsxK2tQBThwq225jcvBjpw1Gr40uzZCA== dependencies: cross-spawn "^7.0.0" @@ -2395,148 +2475,152 @@ foreground-child@^2.0.0: form-data@^2.5.1: version "2.5.1" - resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.5.1.tgz#f2cbec57b5e59e23716e128fe44d4e5dd23895f4" + resolved "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz" integrity sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA== dependencies: asynckit "^0.4.0" combined-stream "^1.0.6" mime-types "^2.1.12" +form-data@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452" + integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.8" + mime-types "^2.1.12" + forwarded@0.2.0: version "0.2.0" - resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.2.0.tgz#2269936428aad4c15c7ebe9779a84bf0b2a81811" + resolved "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz" integrity sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow== fresh@0.5.2: version "0.5.2" - resolved "https://registry.yarnpkg.com/fresh/-/fresh-0.5.2.tgz#3d8cadd90d976569fa835ab1f8e4b23a105605a7" + resolved "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz" integrity sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q== fromentries@^1.2.0: version "1.3.2" - resolved "https://registry.yarnpkg.com/fromentries/-/fromentries-1.3.2.tgz#e4bca6808816bf8f93b52750f1127f5a6fd86e3a" + resolved "https://registry.npmjs.org/fromentries/-/fromentries-1.3.2.tgz" integrity sha512-cHEpEQHUg0f8XdtZCc2ZAhrHzKzT0MrFUTcvx+hfxYu7rGMDc5SKoXFh+n4YigxsHXRzc6OrCshdR1bWH6HHyg== fs-exists-cached@^1.0.0: version "1.0.0" - resolved "https://registry.yarnpkg.com/fs-exists-cached/-/fs-exists-cached-1.0.0.tgz#cf25554ca050dc49ae6656b41de42258989dcbce" + resolved "https://registry.npmjs.org/fs-exists-cached/-/fs-exists-cached-1.0.0.tgz" integrity sha512-kSxoARUDn4F2RPXX48UXnaFKwVU7Ivd/6qpzZL29MCDmr9sTvybv4gFCp+qaI4fM9m0z9fgz/yJvi56GAz+BZg== fs.realpath@^1.0.0: version "1.0.0" - resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" + resolved "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz" integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw== -fsevents@~2.3.1, fsevents@~2.3.2: +fsevents@~2.3.2: version "2.3.2" - resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a" + resolved "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz" integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA== -function-bind@^1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d" - integrity sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A== - function-bind@^1.1.2: version "1.1.2" - resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.2.tgz#2c02d864d97f3ea6c8830c464cbd11ab6eab7a1c" - integrity sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA== + resolved "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz" + integrity "sha1-LALYZNl/PqbIgwxGTL0Rq26rehw= sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==" function-loop@^2.0.1: version "2.0.1" - resolved "https://registry.yarnpkg.com/function-loop/-/function-loop-2.0.1.tgz#799c56ced01698cf12a1b80e4802e9dafc2ebada" + resolved "https://registry.npmjs.org/function-loop/-/function-loop-2.0.1.tgz" integrity sha512-ktIR+O6i/4h+j/ZhZJNdzeI4i9lEPeEK6UPR2EVyTVBqOwcU3Za9xYKLH64ZR9HmcROyRrOkizNyjjtWJzDDkQ== -function.prototype.name@^1.1.5: - version "1.1.5" - resolved "https://registry.yarnpkg.com/function.prototype.name/-/function.prototype.name-1.1.5.tgz#cce0505fe1ffb80503e6f9e46cc64e46a12a9621" - integrity sha512-uN7m/BzVKQnCUF/iW8jYea67v++2u7m5UgENbHRtdDVclOUP+FMPlCNdmk0h/ysGyo2tavMJEDqJAkJdRa1vMA== +function.prototype.name@^1.1.6: + version "1.1.6" + resolved "https://registry.yarnpkg.com/function.prototype.name/-/function.prototype.name-1.1.6.tgz#cdf315b7d90ee77a4c6ee216c3c3362da07533fd" + integrity sha512-Z5kx79swU5P27WEayXM1tBi5Ze/lbIyiNgU3qyXUOf9b2rgXYyF9Dy9Cx+IQv/Lc8WCG6L82zwUPpSS9hGehIg== dependencies: call-bind "^1.0.2" - define-properties "^1.1.3" - es-abstract "^1.19.0" - functions-have-names "^1.2.2" + define-properties "^1.2.0" + es-abstract "^1.22.1" + functions-have-names "^1.2.3" -functions-have-names@^1.2.2, functions-have-names@^1.2.3: +functions-have-names@^1.2.3: version "1.2.3" resolved "https://registry.yarnpkg.com/functions-have-names/-/functions-have-names-1.2.3.tgz#0404fe4ee2ba2f607f0e0ec3c80bae994133b834" integrity sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ== gensync@^1.0.0-beta.2: version "1.0.0-beta.2" - resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0" + resolved "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz" integrity sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg== get-caller-file@^2.0.1, get-caller-file@^2.0.5: version "2.0.5" - resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e" + resolved "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz" integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg== get-func-name@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/get-func-name/-/get-func-name-2.0.0.tgz#ead774abee72e20409433a066366023dd6887a41" - integrity sha512-Hm0ixYtaSZ/V7C8FJrtZIuBBI+iSgL+1Aq82zSu8VQNB4S3Gk8e7Qs3VwBDJAhmRZcFqkl3tQu36g/Foh5I5ig== - -get-intrinsic@^1.0.2, get-intrinsic@^1.1.0, get-intrinsic@^1.1.1, get-intrinsic@^1.1.3: - version "1.1.3" - resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.1.3.tgz#063c84329ad93e83893c7f4f243ef63ffa351385" - integrity sha512-QJVz1Tj7MS099PevUG5jvnt9tSkXN8K14dxQlikJuPt4uD9hHAHjLyLBiLR5zELelBdD9QNRAXZzsJx0WaDL9A== - dependencies: - function-bind "^1.1.1" - has "^1.0.3" - has-symbols "^1.0.3" + version "2.0.2" + resolved "https://registry.yarnpkg.com/get-func-name/-/get-func-name-2.0.2.tgz#0d7cf20cd13fda808669ffa88f4ffc7a3943fc41" + integrity "sha1-DXzyDNE/2oCGaf+oj0/8ejlD/EE= sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==" -get-intrinsic@^1.2.0, get-intrinsic@^1.2.1: - version "1.2.1" - resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.2.1.tgz#d295644fed4505fc9cde952c37ee12b477a83d82" - integrity sha512-2DcsyfABl+gVHEfCOaTrWgyt+tb6MSEGmKq+kI5HwLbIYgjgmMcV8KQ41uaKz1xxUcn9tJtgFbQUEVcEbd0FYw== +get-intrinsic@^1.0.2, get-intrinsic@^1.1.3, get-intrinsic@^1.2.1, get-intrinsic@^1.2.3, get-intrinsic@^1.2.4: + version "1.2.4" + resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.2.4.tgz#e385f5a4b5227d449c3eabbad05494ef0abbeadd" + integrity sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ== dependencies: - function-bind "^1.1.1" - has "^1.0.3" + es-errors "^1.3.0" + function-bind "^1.1.2" has-proto "^1.0.1" has-symbols "^1.0.3" + hasown "^2.0.0" get-package-type@^0.1.0: version "0.1.0" - resolved "https://registry.yarnpkg.com/get-package-type/-/get-package-type-0.1.0.tgz#8de2d803cff44df3bc6c456e6668b36c3926e11a" + resolved "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz" integrity sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q== get-port@^3.2.0: version "3.2.0" - resolved "https://registry.yarnpkg.com/get-port/-/get-port-3.2.0.tgz#dd7ce7de187c06c8bf353796ac71e099f0980ebc" + resolved "https://registry.npmjs.org/get-port/-/get-port-3.2.0.tgz" integrity sha512-x5UJKlgeUiNT8nyo/AcnwLnZuZNcSjSw0kogRB+Whd1fjjFq4B1hySFxSFWWSn4mIBzg3sRNUDFYc4g5gjPoLg== -get-symbol-description@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/get-symbol-description/-/get-symbol-description-1.0.0.tgz#7fdb81c900101fbd564dd5f1a30af5aadc1e58d6" - integrity sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw== +get-symbol-description@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/get-symbol-description/-/get-symbol-description-1.0.2.tgz#533744d5aa20aca4e079c8e5daf7fd44202821f5" + integrity sha512-g0QYk1dZBxGwk+Ngc+ltRH2IBp2f7zBkBMBJZCDerh6EhlhSR6+9irMCuT/09zD6qkarHUSn529sK/yL4S27mg== dependencies: - call-bind "^1.0.2" - get-intrinsic "^1.1.1" + call-bind "^1.0.5" + es-errors "^1.3.0" + get-intrinsic "^1.2.4" + +get-tsconfig@^4.7.0: + version "4.7.5" + resolved "https://registry.yarnpkg.com/get-tsconfig/-/get-tsconfig-4.7.5.tgz#5e012498579e9a6947511ed0cd403272c7acbbaf" + integrity sha512-ZCuZCnlqNzjb4QprAzXKdpp/gh6KTxSJuw3IBsPnV/7fV4NxC9ckB+vPTt8w7fJA0TaSD7c55BR47JD6MEDyDw== + dependencies: + resolve-pkg-maps "^1.0.0" getopts@2.3.0: version "2.3.0" - resolved "https://registry.yarnpkg.com/getopts/-/getopts-2.3.0.tgz#71e5593284807e03e2427449d4f6712a268666f4" + resolved "https://registry.npmjs.org/getopts/-/getopts-2.3.0.tgz" integrity sha512-5eDf9fuSXwxBL6q5HX+dhDj+dslFGWzU5thZ9kNKUkcPtaPdatmUFKwHFrLb/uf/WpA4BHET+AX3Scl56cAjpA== -glob-parent@^5.1.2, glob-parent@~5.1.0, glob-parent@~5.1.2: - version "5.1.2" - resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.2.tgz#869832c58034fe68a4093c17dc15e8340d8401c4" - integrity sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow== - dependencies: - is-glob "^4.0.1" - -glob-parent@^6.0.1: +glob-parent@^6.0.2: version "6.0.2" resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-6.0.2.tgz#6d237d99083950c79290f24c7642a3de9a28f9e3" integrity sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A== dependencies: is-glob "^4.0.3" -glob@7.1.6: - version "7.1.6" - resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.6.tgz#141f33b81a7c2492e125594307480c46679278a6" - integrity sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA== +glob-parent@~5.1.2: + version "5.1.2" + resolved "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz" + integrity sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow== + dependencies: + is-glob "^4.0.1" + +glob@7.2.0: + version "7.2.0" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.0.tgz#d15535af7732e02e948f4c41628bd910293f6023" + integrity sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q== dependencies: fs.realpath "^1.0.0" inflight "^1.0.4" @@ -2547,7 +2631,7 @@ glob@7.1.6: glob@^7.0.5, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6, glob@^7.2.3: version "7.2.3" - resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.3.tgz#b8df0fb802bbfa8e89bd1d938b4e16578ed44f2b" + resolved "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz" integrity sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q== dependencies: fs.realpath "^1.0.0" @@ -2559,134 +2643,115 @@ glob@^7.0.5, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6, glob@^7.2.3: globals@^11.1.0: version "11.12.0" - resolved "https://registry.yarnpkg.com/globals/-/globals-11.12.0.tgz#ab8795338868a0babd8525758018c2a7eb95c42e" + resolved "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz" integrity sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA== -globals@^13.15.0: - version "13.17.0" - resolved "https://registry.yarnpkg.com/globals/-/globals-13.17.0.tgz#902eb1e680a41da93945adbdcb5a9f361ba69bd4" - integrity sha512-1C+6nQRb1GwGMKm2dH/E7enFAMxGTmGI7/dEdhy/DNelv85w9B72t3uc5frtMNXIbzrarJJ/lTCjcaZwbLJmyw== +globals@^13.19.0, globals@^13.24.0: + version "13.24.0" + resolved "https://registry.yarnpkg.com/globals/-/globals-13.24.0.tgz#8432a19d78ce0c1e833949c36adb345400bb1171" + integrity sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ== dependencies: type-fest "^0.20.2" globalthis@^1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/globalthis/-/globalthis-1.0.3.tgz#5852882a52b80dc301b0660273e1ed082f0b6ccf" - integrity sha512-sFdI5LyBiNTHjRd7cGPWapiHWMOXKyuBNX/cWJ3NfzrZQVa8GI/8cofCl74AOVqq9W5kNmguTIzJ/1s2gyI9wA== - dependencies: - define-properties "^1.1.3" - -globby@^11.1.0: - version "11.1.0" - resolved "https://registry.yarnpkg.com/globby/-/globby-11.1.0.tgz#bd4be98bb042f83d796f7e3811991fbe82a0d34b" - integrity sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g== + version "1.0.4" + resolved "https://registry.yarnpkg.com/globalthis/-/globalthis-1.0.4.tgz#7430ed3a975d97bfb59bcce41f5cabbafa651236" + integrity sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ== dependencies: - array-union "^2.1.0" - dir-glob "^3.0.1" - fast-glob "^3.2.9" - ignore "^5.2.0" - merge2 "^1.4.1" - slash "^3.0.0" + define-properties "^1.2.1" + gopd "^1.0.1" gopd@^1.0.1: version "1.0.1" - resolved "https://registry.yarnpkg.com/gopd/-/gopd-1.0.1.tgz#29ff76de69dac7489b7c0918a5788e56477c332c" - integrity sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA== + resolved "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz" + integrity "sha1-Kf923mnax0ibfAkYpXiOVkd8Myw= sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==" dependencies: get-intrinsic "^1.1.3" graceful-fs@^4.1.15: version "4.2.10" - resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.10.tgz#147d3a006da4ca3ce14728c7aefc287c367d7a6c" + resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz" integrity sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA== -grapheme-splitter@^1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/grapheme-splitter/-/grapheme-splitter-1.0.4.tgz#9cf3a665c6247479896834af35cf1dbb4400767e" - integrity sha512-bzh50DW9kTPM00T8y4o8vQg89Di9oLJVLW/KaOGIXJWP/iqCN6WKYkbNOF04vFLJhwcpYUh9ydh/+5vpOqV4YQ== +graphemer@^1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/graphemer/-/graphemer-1.4.0.tgz#fb2f1d55e0e3a1849aeffc90c4fa0dd53a0e66c6" + integrity sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag== graphql@0.13.2: version "0.13.2" - resolved "https://registry.yarnpkg.com/graphql/-/graphql-0.13.2.tgz#4c740ae3c222823e7004096f832e7b93b2108270" + resolved "https://registry.npmjs.org/graphql/-/graphql-0.13.2.tgz" integrity sha512-QZ5BL8ZO/B20VA8APauGBg3GyEgZ19eduvpLWoq5x7gMmWnHoy8rlQWPLmWgFvo1yNgjSEFMesmS4R6pPr7xog== dependencies: iterall "^1.2.1" growl@1.10.5: version "1.10.5" - resolved "https://registry.yarnpkg.com/growl/-/growl-1.10.5.tgz#f2735dc2283674fa67478b10181059355c369e5e" + resolved "https://registry.npmjs.org/growl/-/growl-1.10.5.tgz" integrity sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA== has-async-hooks@^1.0.0: version "1.0.0" - resolved "https://registry.yarnpkg.com/has-async-hooks/-/has-async-hooks-1.0.0.tgz#3df965ade8cd2d9dbfdacfbca3e0a5152baaf204" + resolved "https://registry.npmjs.org/has-async-hooks/-/has-async-hooks-1.0.0.tgz" integrity sha512-YF0VPGjkxr7AyyQQNykX8zK4PvtEDsUJAPqwu06UFz1lb6EvI53sPh5H1kWxg8NXI5LsfRCZ8uX9NkYDZBb/mw== has-bigints@^1.0.1, has-bigints@^1.0.2: version "1.0.2" - resolved "https://registry.yarnpkg.com/has-bigints/-/has-bigints-1.0.2.tgz#0871bd3e3d51626f6ca0966668ba35d5602d6eaa" + resolved "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.2.tgz" integrity sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ== -has-dynamic-import@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/has-dynamic-import/-/has-dynamic-import-2.0.1.tgz#9bca87846aa264f2ad224fcd014946f5e5182f52" - integrity sha512-X3fbtsZmwb6W7fJGR9o7x65fZoodygCrZ3TVycvghP62yYQfS0t4RS0Qcz+j5tQYUKeSWS09tHkWW6WhFV3XhQ== - dependencies: - call-bind "^1.0.2" - get-intrinsic "^1.1.1" - has-flag@^3.0.0: version "3.0.0" - resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd" + resolved "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz" integrity sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw== has-flag@^4.0.0: version "4.0.0" - resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-4.0.0.tgz#944771fd9c81c81265c4d6941860da06bb59479b" + resolved "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz" integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ== -has-property-descriptors@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/has-property-descriptors/-/has-property-descriptors-1.0.0.tgz#610708600606d36961ed04c196193b6a607fa861" - integrity sha512-62DVLZGoiEBDHQyqG4w9xCuZ7eJEwNmJRWw2VY84Oedb7WFcA27fiEVe8oUQx9hAUJ4ekurquucTGwsyO1XGdQ== +has-property-descriptors@^1.0.0, has-property-descriptors@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz#963ed7d071dc7bf5f084c5bfbe0d1b6222586854" + integrity sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg== dependencies: - get-intrinsic "^1.1.1" + es-define-property "^1.0.0" -has-proto@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/has-proto/-/has-proto-1.0.1.tgz#1885c1305538958aff469fef37937c22795408e0" - integrity sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg== +has-proto@^1.0.1, has-proto@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/has-proto/-/has-proto-1.0.3.tgz#b31ddfe9b0e6e9914536a6ab286426d0214f77fd" + integrity sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q== has-symbols@^1.0.2, has-symbols@^1.0.3: version "1.0.3" - resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.0.3.tgz#bb7b2c4349251dce87b125f7bdf874aa7c8b39f8" + resolved "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz" integrity sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A== -has-tostringtag@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/has-tostringtag/-/has-tostringtag-1.0.0.tgz#7e133818a7d394734f941e73c3d3f9291e658b25" - integrity sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ== - dependencies: - has-symbols "^1.0.2" - -has@^1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/has/-/has-1.0.3.tgz#722d7cbfc1f6aa8241f16dd814e011e1f41e8796" - integrity sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw== +has-tostringtag@^1.0.0, has-tostringtag@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/has-tostringtag/-/has-tostringtag-1.0.2.tgz#2cdc42d40bef2e5b4eeab7c01a73c54ce7ab5abc" + integrity sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw== dependencies: - function-bind "^1.1.1" + has-symbols "^1.0.3" hasha@^5.0.0: version "5.2.2" - resolved "https://registry.yarnpkg.com/hasha/-/hasha-5.2.2.tgz#a48477989b3b327aea3c04f53096d816d97522a1" + resolved "https://registry.npmjs.org/hasha/-/hasha-5.2.2.tgz" integrity sha512-Hrp5vIK/xr5SkeN2onO32H0MgNZ0f17HRNH39WfL0SYUNOTZ5Lz1TJ8Pajo/87dYGEFlLMm7mIc/k/s6Bvz9HQ== dependencies: is-stream "^2.0.0" type-fest "^0.8.0" +hasown@^2.0.0, hasown@^2.0.1, hasown@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/hasown/-/hasown-2.0.2.tgz#003eaf91be7adc372e84ec59dc37252cedb80003" + integrity sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ== + dependencies: + function-bind "^1.1.2" + hdr-histogram-js@^1.0.0, hdr-histogram-js@^1.1.4: version "1.2.0" - resolved "https://registry.yarnpkg.com/hdr-histogram-js/-/hdr-histogram-js-1.2.0.tgz#1213c0b317f39b9c05bc4f208cb7931dbbc192ae" + resolved "https://registry.npmjs.org/hdr-histogram-js/-/hdr-histogram-js-1.2.0.tgz" integrity sha512-h0YToJ3ewqsaZ3nFTTa6dLOD7sqx+EgdC4+OcJ9Ou7zZDlT0sXSPHHr3cyenQsPqqbVHGn/oFY6zjfEKXGvzmQ== dependencies: base64-js "^1.2.0" @@ -2694,24 +2759,24 @@ hdr-histogram-js@^1.0.0, hdr-histogram-js@^1.1.4: hdr-histogram-percentiles-obj@^2.0.0: version "2.0.1" - resolved "https://registry.yarnpkg.com/hdr-histogram-percentiles-obj/-/hdr-histogram-percentiles-obj-2.0.1.tgz#7a4d52fa02087118c66469e6b66b74f9fbb44d82" + resolved "https://registry.npmjs.org/hdr-histogram-percentiles-obj/-/hdr-histogram-percentiles-obj-2.0.1.tgz" integrity sha512-QBvbTxPlGwHj36IRF16XLoYEbUv5YEyO385kiS0IS3831fcSTNXTR785VtFFZ2ahY733z0ky8Jv4d6In+Ss+wQ== dependencies: hdr-histogram-js "^1.0.0" he@1.2.0: version "1.2.0" - resolved "https://registry.yarnpkg.com/he/-/he-1.2.0.tgz#84ae65fa7eafb165fddb61566ae14baf05664f0f" + resolved "https://registry.npmjs.org/he/-/he-1.2.0.tgz" integrity sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw== html-escaper@^2.0.0: version "2.0.2" - resolved "https://registry.yarnpkg.com/html-escaper/-/html-escaper-2.0.2.tgz#dfd60027da36a36dfcbe236262c00a5822681453" + resolved "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz" integrity sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg== http-errors@2.0.0: version "2.0.0" - resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-2.0.0.tgz#b7774a1486ef73cf7667ac9ae0858c012c57b9d3" + resolved "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz" integrity sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ== dependencies: depd "2.0.0" @@ -2722,12 +2787,12 @@ http-errors@2.0.0: http-parser-js@^0.5.2: version "0.5.8" - resolved "https://registry.yarnpkg.com/http-parser-js/-/http-parser-js-0.5.8.tgz#af23090d9ac4e24573de6f6aecc9d84a48bf20e3" + resolved "https://registry.npmjs.org/http-parser-js/-/http-parser-js-0.5.8.tgz" integrity sha512-SGeBX54F94Wgu5RH3X5jsDtf4eHyRogWX1XGT3b4HuW3tQPM4AaBzoUji/4AAJNXCEOWZ5O0DgZmJw1947gD5Q== hyperid@^2.0.3: version "2.3.1" - resolved "https://registry.yarnpkg.com/hyperid/-/hyperid-2.3.1.tgz#70cc2c917b6367c9f7307718be243bc28b258353" + resolved "https://registry.npmjs.org/hyperid/-/hyperid-2.3.1.tgz" integrity sha512-mIbI7Ymn6MCdODaW1/6wdf5lvvXzmPsARN4zTLakMmcziBOuP4PxCBJvHF6kbAIHX6H4vAELx/pDmt0j6Th5RQ== dependencies: uuid "^8.3.2" @@ -2735,85 +2800,75 @@ hyperid@^2.0.3: iconv-lite@0.4.24: version "0.4.24" - resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b" + resolved "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz" integrity sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA== dependencies: safer-buffer ">= 2.1.2 < 3" ieee754@1.1.13: version "1.1.13" - resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84" - integrity sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg== + resolved "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz" + integrity "sha1-7BaFWOlaoYH9h9N/VcMrvLZwi4Q= sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" ieee754@^1.1.4, ieee754@^1.1.8: version "1.2.1" - resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" + resolved "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz" integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA== -ignore@^3.3.6: - version "3.3.10" - resolved "https://registry.yarnpkg.com/ignore/-/ignore-3.3.10.tgz#0a97fb876986e8081c631160f8f9f389157f0043" - integrity sha512-Pgs951kaMm5GXP7MOvxERINe3gsaVjUWFm+UZPSq9xYriQAksyhg0csnS0KXSNRD5NmNdapXEpjxG49+AKh/ug== - -ignore@^5.1.1, ignore@^5.2.4: - version "5.2.4" - resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.2.4.tgz#a291c0c6178ff1b960befe47fcdec301674a6324" - integrity sha512-MAb38BcSbH0eHNBxn7ql2NH/kX33OkB3lZ1BNdh7ENeRChHTYsTvWrMubiIAMNS2llXEEgZ1MUOBtXChP3kaFQ== - -ignore@^5.2.0: - version "5.2.0" - resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.2.0.tgz#6d3bac8fa7fe0d45d9f9be7bac2fc279577e345a" - integrity sha512-CmxgYGiEPCLhfLnpPp1MoRmifwEIOgjcHXxOBjv7mY96c+eWScsOP9c112ZyLdWHi0FxHjI+4uVhKYp/gcdRmQ== +ignore@^5.2.0, ignore@^5.2.4: + version "5.3.1" + resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.3.1.tgz#5073e554cd42c5b33b394375f538b8593e34d4ef" + integrity sha512-5Fytz/IraMjqpwfd34ke28PTVMjZjJG2MPn5t7OE4eUCUNf8BAa7b5WUS9/Qvr6mwOQS7Mk6vdsMno5he+T8Xw== immediate@~3.0.5: version "3.0.6" - resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.0.6.tgz#9db1dbd0faf8de6fbe0f5dd5e56bb606280de69b" + resolved "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz" integrity sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ== -import-fresh@^3.0.0, import-fresh@^3.2.1: +import-fresh@^3.2.1: version "3.3.0" - resolved "https://registry.yarnpkg.com/import-fresh/-/import-fresh-3.3.0.tgz#37162c25fcb9ebaa2e6e53d5b4d88ce17d9e0c2b" + resolved "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz" integrity sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw== dependencies: parent-module "^1.0.0" resolve-from "^4.0.0" -import-in-the-middle@^1.4.2: - version "1.4.2" - resolved "https://registry.yarnpkg.com/import-in-the-middle/-/import-in-the-middle-1.4.2.tgz#2a266676e3495e72c04bbaa5ec14756ba168391b" - integrity sha512-9WOz1Yh/cvO/p69sxRmhyQwrIGGSp7EIdcb+fFNVi7CzQGQB8U1/1XrKVSbEd/GNOAeM0peJtmi7+qphe7NvAw== +import-in-the-middle@1.11.2: + version "1.11.2" + resolved "https://registry.yarnpkg.com/import-in-the-middle/-/import-in-the-middle-1.11.2.tgz#dd848e72b63ca6cd7c34df8b8d97fc9baee6174f" + integrity sha512-gK6Rr6EykBcc6cVWRSBR5TWf8nn6hZMYSRYqCcHa0l0d1fPK7JSYo6+Mlmck76jIX9aL/IZ71c06U2VpFwl1zA== dependencies: acorn "^8.8.2" - acorn-import-assertions "^1.9.0" + acorn-import-attributes "^1.9.5" cjs-module-lexer "^1.2.2" module-details-from-path "^1.0.3" imurmurhash@^0.1.4: version "0.1.4" - resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea" + resolved "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz" integrity sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA== indent-string@^4.0.0: version "4.0.0" - resolved "https://registry.yarnpkg.com/indent-string/-/indent-string-4.0.0.tgz#624f8f4497d619b2d9768531d58f4122854d7251" + resolved "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz" integrity sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg== inflight@^1.0.4: version "1.0.6" - resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9" + resolved "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz" integrity sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA== dependencies: once "^1.3.0" wrappy "1" -inherits@2, inherits@2.0.4, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.3: +inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@~2.0.3: version "2.0.4" - resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" + resolved "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== ink@^3.2.0: version "3.2.0" - resolved "https://registry.yarnpkg.com/ink/-/ink-3.2.0.tgz#434793630dc57d611c8fe8fffa1db6b56f1a16bb" + resolved "https://registry.npmjs.org/ink/-/ink-3.2.0.tgz" integrity sha512-firNp1q3xxTzoItj/eOOSZQnYSlyrWks5llCTVX37nJ59K3eXbQ8PtzCguqo8YI19EELo5QxaKnJd4VxzhU8tg== dependencies: ansi-escapes "^4.2.1" @@ -2842,270 +2897,233 @@ ink@^3.2.0: int64-buffer@^0.1.9: version "0.1.10" - resolved "https://registry.yarnpkg.com/int64-buffer/-/int64-buffer-0.1.10.tgz#277b228a87d95ad777d07c13832022406a473423" + resolved "https://registry.npmjs.org/int64-buffer/-/int64-buffer-0.1.10.tgz" integrity sha512-v7cSY1J8ydZ0GyjUHqF+1bshJ6cnEVLo9EnjB8p+4HDRPZc9N5jjmvUV7NvEsqQOKyH0pmIBFWXVQbiS0+OBbA== -internal-slot@^1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/internal-slot/-/internal-slot-1.0.3.tgz#7347e307deeea2faac2ac6205d4bc7d34967f59c" - integrity sha512-O0DB1JC/sPyZl7cIo78n5dR7eUSwwpYPiXRhTzNxZVAMUuB8vlnRFyLxdrVToks6XPLVnFfbzaVd5WLjhgg+vA== - dependencies: - get-intrinsic "^1.1.0" - has "^1.0.3" - side-channel "^1.0.4" - -internal-slot@^1.0.4, internal-slot@^1.0.5: - version "1.0.5" - resolved "https://registry.yarnpkg.com/internal-slot/-/internal-slot-1.0.5.tgz#f2a2ee21f668f8627a4667f309dc0f4fb6674986" - integrity sha512-Y+R5hJrzs52QCG2laLn4udYVnxsfny9CpOhNhUvk/SSSVyF6T27FzRbF0sroPidSu3X8oEAkOn2K804mjpt6UQ== +internal-slot@^1.0.7: + version "1.0.7" + resolved "https://registry.yarnpkg.com/internal-slot/-/internal-slot-1.0.7.tgz#c06dcca3ed874249881007b0a5523b172a190802" + integrity sha512-NGnrKwXzSms2qUUih/ILZ5JBqNTSa1+ZmP6flaIp6KmSElgE9qdndzS3cqjrDovwFdmwsGsLdeFgB6suw+1e9g== dependencies: - get-intrinsic "^1.2.0" - has "^1.0.3" + es-errors "^1.3.0" + hasown "^2.0.0" side-channel "^1.0.4" interpret@^2.2.0: version "2.2.0" - resolved "https://registry.yarnpkg.com/interpret/-/interpret-2.2.0.tgz#1a78a0b5965c40a5416d007ad6f50ad27c417df9" + resolved "https://registry.npmjs.org/interpret/-/interpret-2.2.0.tgz" integrity sha512-Ju0Bz/cEia55xDwUWEa8+olFpCiQoypjnQySseKtmjNrnps3P+xfpUmGr90T7yjlVJmOtybRvPXhKMbHr+fWnw== ipaddr.js@1.9.1: version "1.9.1" - resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.9.1.tgz#bff38543eeb8984825079ff3a2a8e6cbd46781b3" + resolved "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz" integrity sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g== -ipaddr.js@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-2.1.0.tgz#2119bc447ff8c257753b196fc5f1ce08a4cdf39f" - integrity sha512-LlbxQ7xKzfBusov6UMi4MFpEg0m+mAm9xyNGEduwXMEDuf4WfzB/RZwMVYEd7IKGvh4IUkEXYxtAVu9T3OelJQ== - -is-arguments@^1.0.4, is-arguments@^1.1.1: +is-arguments@^1.0.4: version "1.1.1" - resolved "https://registry.yarnpkg.com/is-arguments/-/is-arguments-1.1.1.tgz#15b3f88fda01f2a97fec84ca761a560f123efa9b" + resolved "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz" integrity sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA== dependencies: call-bind "^1.0.2" has-tostringtag "^1.0.0" -is-array-buffer@^3.0.1, is-array-buffer@^3.0.2: - version "3.0.2" - resolved "https://registry.yarnpkg.com/is-array-buffer/-/is-array-buffer-3.0.2.tgz#f2653ced8412081638ecb0ebbd0c41c6e0aecbbe" - integrity sha512-y+FyyR/w8vfIRq4eQcM1EYgSTnmHXPqaF+IgzgraytCFq5Xh8lllDVmAZolPJiZttZLeFSINPYMaEJ7/vWUa1w== +is-array-buffer@^3.0.4: + version "3.0.4" + resolved "https://registry.yarnpkg.com/is-array-buffer/-/is-array-buffer-3.0.4.tgz#7a1f92b3d61edd2bc65d24f130530ea93d7fae98" + integrity sha512-wcjaerHw0ydZwfhiKbXJWLDY8A7yV7KhjQOpb83hGgGfId/aQa4TOvwyzn2PuswW2gPCYEL/nEAiSVpdOj1lXw== dependencies: call-bind "^1.0.2" - get-intrinsic "^1.2.0" - is-typed-array "^1.1.10" + get-intrinsic "^1.2.1" is-bigint@^1.0.1: version "1.0.4" - resolved "https://registry.yarnpkg.com/is-bigint/-/is-bigint-1.0.4.tgz#08147a1875bc2b32005d41ccd8291dffc6691df3" + resolved "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.4.tgz" integrity sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg== dependencies: has-bigints "^1.0.1" is-binary-path@~2.1.0: version "2.1.0" - resolved "https://registry.yarnpkg.com/is-binary-path/-/is-binary-path-2.1.0.tgz#ea1f7f3b80f064236e83470f86c09c254fb45b09" + resolved "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz" integrity sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw== dependencies: binary-extensions "^2.0.0" is-boolean-object@^1.1.0: version "1.1.2" - resolved "https://registry.yarnpkg.com/is-boolean-object/-/is-boolean-object-1.1.2.tgz#5c6dc200246dd9321ae4b885a114bb1f75f63719" + resolved "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.2.tgz" integrity sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA== dependencies: call-bind "^1.0.2" has-tostringtag "^1.0.0" -is-callable@^1.1.3, is-callable@^1.1.4, is-callable@^1.2.6, is-callable@^1.2.7: +is-builtin-module@^3.2.1: + version "3.2.1" + resolved "https://registry.yarnpkg.com/is-builtin-module/-/is-builtin-module-3.2.1.tgz#f03271717d8654cfcaf07ab0463faa3571581169" + integrity sha512-BSLE3HnV2syZ0FK0iMA/yUGplUeMmNz4AW5fnTunbCIqZi4vG3WjJT9FHMy5D69xmAYBHXQhJdALdpwVxV501A== + dependencies: + builtin-modules "^3.3.0" + +is-callable@^1.1.3, is-callable@^1.1.4, is-callable@^1.2.7: version "1.2.7" - resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.2.7.tgz#3bc2a85ea742d9e36205dcacdd72ca1fdc51b055" + resolved "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz" integrity sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA== is-ci@^2.0.0: version "2.0.0" - resolved "https://registry.yarnpkg.com/is-ci/-/is-ci-2.0.0.tgz#6bc6334181810e04b5c22b3d589fdca55026404c" + resolved "https://registry.npmjs.org/is-ci/-/is-ci-2.0.0.tgz" integrity sha512-YfJT7rkpQB0updsdHLGWrvhBJfcfzNNawYDNIyQXJz0IViGf75O8EBPKSdvw2rF+LGCsX4FZ8tcr3b19LcZq4w== dependencies: ci-info "^2.0.0" -is-core-module@^2.11.0: - version "2.12.0" - resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.12.0.tgz#36ad62f6f73c8253fd6472517a12483cf03e7ec4" - integrity sha512-RECHCBCd/viahWmwj6enj19sKbHfJrddi/6cBDsNTKbNq0f7VeaUkBo60BqzvPqo/W54ChS62Z5qyun7cfOMqQ== +is-core-module@^2.12.1, is-core-module@^2.13.0, is-core-module@^2.13.1: + version "2.15.0" + resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.15.0.tgz#71c72ec5442ace7e76b306e9d48db361f22699ea" + integrity sha512-Dd+Lb2/zvk9SKy1TGCt1wFJFo/MWBPMX5x7KcvLajWTGuomczdQX61PvY5yK6SVACwpoexWo81IfFyoKY2QnTA== dependencies: - has "^1.0.3" + hasown "^2.0.2" -is-core-module@^2.8.1, is-core-module@^2.9.0: - version "2.10.0" - resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.10.0.tgz#9012ede0a91c69587e647514e1d5277019e728ed" - integrity sha512-Erxj2n/LDAZ7H8WNJXd9tw38GYM3dv8rk8Zcs+jJuxYTW7sozH+SS8NtrSjVL1/vpLvWi1hxy96IzjJ3EHTJJg== +is-data-view@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/is-data-view/-/is-data-view-1.0.1.tgz#4b4d3a511b70f3dc26d42c03ca9ca515d847759f" + integrity sha512-AHkaJrsUVW6wq6JS8y3JnM/GJF/9cf+k20+iDzlSaJrinEo5+7vRiteOSwBhHRiAyQATN1AmY4hwzxJKPmYf+w== dependencies: - has "^1.0.3" + is-typed-array "^1.1.13" -is-date-object@^1.0.1, is-date-object@^1.0.5: +is-date-object@^1.0.1: version "1.0.5" - resolved "https://registry.yarnpkg.com/is-date-object/-/is-date-object-1.0.5.tgz#0841d5536e724c25597bf6ea62e1bd38298df31f" + resolved "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.5.tgz" integrity sha512-9YQaSxsAiSwcvS33MBk3wTCVnWK+HhF8VZR2jRxehM16QcVOdHqPn4VPHmRK4lSr38n9JriurInLcP90xsYNfQ== dependencies: has-tostringtag "^1.0.0" is-extglob@^2.1.1: version "2.1.1" - resolved "https://registry.yarnpkg.com/is-extglob/-/is-extglob-2.1.1.tgz#a88c02535791f02ed37c76a1b9ea9773c833f8c2" + resolved "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz" integrity sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ== is-fullwidth-code-point@^2.0.0: version "2.0.0" - resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz#a3b30a5c4f199183167aaab93beefae3ddfb654f" + resolved "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz" integrity sha512-VHskAKYM8RfSFXwee5t5cbN5PZeq1Wrh6qd5bkyiXIf6UQcN6w/A0eXM9r6t8d+GYOh+o6ZhiEnb88LN/Y8m2w== is-fullwidth-code-point@^3.0.0: version "3.0.0" - resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz#f116f8064fe90b3f7844a38997c0b75051269f1d" + resolved "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz" integrity sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg== is-generator-function@^1.0.7: version "1.0.10" - resolved "https://registry.yarnpkg.com/is-generator-function/-/is-generator-function-1.0.10.tgz#f1558baf1ac17e0deea7c0415c438351ff2b3c72" - integrity sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A== + resolved "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.10.tgz" + integrity "sha1-8VWLrxrBfg3up8BBXEODUf8rPHI= sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==" dependencies: has-tostringtag "^1.0.0" is-glob@^4.0.0, is-glob@^4.0.1, is-glob@^4.0.3, is-glob@~4.0.1: version "4.0.3" - resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-4.0.3.tgz#64f61e42cbbb2eec2071a9dac0b28ba1e65d5084" + resolved "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz" integrity sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg== dependencies: is-extglob "^2.1.1" -is-map@^2.0.1, is-map@^2.0.2: - version "2.0.2" - resolved "https://registry.yarnpkg.com/is-map/-/is-map-2.0.2.tgz#00922db8c9bf73e81b7a335827bc2a43f2b91127" - integrity sha512-cOZFQQozTha1f4MxLFzlgKYPTyj26picdZTx82hbc/Xf4K/tZOOXSCkMvU4pKioRXGDLJRn0GM7Upe7kR721yg== - -is-negative-zero@^2.0.2: - version "2.0.2" - resolved "https://registry.yarnpkg.com/is-negative-zero/-/is-negative-zero-2.0.2.tgz#7bf6f03a28003b8b3965de3ac26f664d765f3150" - integrity sha512-dqJvarLawXsFbNDeJW7zAz8ItJ9cd28YufuuFzh0G8pNHjJMnY08Dv7sYX2uF5UpQOwieAeOExEYAWWfu7ZZUA== +is-negative-zero@^2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/is-negative-zero/-/is-negative-zero-2.0.3.tgz#ced903a027aca6381b777a5743069d7376a49747" + integrity sha512-5KoIu2Ngpyek75jXodFvnafB6DJgr3u8uuK0LEZJjrU19DrMD3EVERaR8sjz8CCGgpZvxPl9SuE1GMVPFHx1mw== is-number-object@^1.0.4: version "1.0.7" - resolved "https://registry.yarnpkg.com/is-number-object/-/is-number-object-1.0.7.tgz#59d50ada4c45251784e9904f5246c742f07a42fc" + resolved "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.7.tgz" integrity sha512-k1U0IRzLMo7ZlYIfzRu23Oh6MiIFasgpb9X76eqfFZAqwH44UI4KTBvBYIZ1dSL9ZzChTB9ShHfLkR4pdW5krQ== dependencies: has-tostringtag "^1.0.0" is-number@^7.0.0: version "7.0.0" - resolved "https://registry.yarnpkg.com/is-number/-/is-number-7.0.0.tgz#7535345b896734d5f80c4d06c50955527a14f12b" + resolved "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz" integrity sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng== is-object@~1.0.1: version "1.0.2" - resolved "https://registry.yarnpkg.com/is-object/-/is-object-1.0.2.tgz#a56552e1c665c9e950b4a025461da87e72f86fcf" + resolved "https://registry.npmjs.org/is-object/-/is-object-1.0.2.tgz" integrity sha512-2rRIahhZr2UWb45fIOuvZGpFtz0TyOZLf32KxBbSoUCeZR495zCKlWUKKUByk3geS2eAs7ZAABt0Y/Rx0GiQGA== +is-path-inside@^3.0.3: + version "3.0.3" + resolved "https://registry.yarnpkg.com/is-path-inside/-/is-path-inside-3.0.3.tgz#d231362e53a07ff2b0e0ea7fed049161ffd16283" + integrity sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ== + is-plain-obj@^2.1.0: version "2.1.0" - resolved "https://registry.yarnpkg.com/is-plain-obj/-/is-plain-obj-2.1.0.tgz#45e42e37fccf1f40da8e5f76ee21515840c09287" + resolved "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-2.1.0.tgz" integrity sha512-YWnfyRwxL/+SsrWYfOpUtz5b3YD+nyfkHvjbcanzk8zgyO4ASD67uVMRt8k5bM4lLMDnXfriRhOpemw+NfT1eA== is-regex@^1.1.4: version "1.1.4" - resolved "https://registry.yarnpkg.com/is-regex/-/is-regex-1.1.4.tgz#eef5663cd59fa4c0ae339505323df6854bb15958" + resolved "https://registry.npmjs.org/is-regex/-/is-regex-1.1.4.tgz" integrity sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg== dependencies: call-bind "^1.0.2" has-tostringtag "^1.0.0" -is-set@^2.0.1, is-set@^2.0.2: - version "2.0.2" - resolved "https://registry.yarnpkg.com/is-set/-/is-set-2.0.2.tgz#90755fa4c2562dc1c5d4024760d6119b94ca18ec" - integrity sha512-+2cnTEZeY5z/iXGbLhPrOAaK/Mau5k5eXq9j14CpRTftq0pAJu2MwVRSZhyZWBzx3o6X795Lz6Bpb6R0GKf37g== - -is-shared-array-buffer@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/is-shared-array-buffer/-/is-shared-array-buffer-1.0.2.tgz#8f259c573b60b6a32d4058a1a07430c0a7344c79" - integrity sha512-sqN2UDu1/0y6uvXyStCOzyhAjCSlHceFoMKJW8W9EU9cvic/QdsZ0kEU93HEy3IUEFZIiH/3w+AH/UQbPHNdhA== +is-shared-array-buffer@^1.0.2, is-shared-array-buffer@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/is-shared-array-buffer/-/is-shared-array-buffer-1.0.3.tgz#1237f1cba059cdb62431d378dcc37d9680181688" + integrity sha512-nA2hv5XIhLR3uVzDDfCIknerhx8XUKnstuOERPNNIinXG7v9u+ohXF67vxm4TPTEPU6lm61ZkwP3c9PCB97rhg== dependencies: - call-bind "^1.0.2" + call-bind "^1.0.7" is-stream@^2.0.0: version "2.0.1" - resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-2.0.1.tgz#fac1e3d53b97ad5a9d0ae9cef2389f5810a5c077" + resolved "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz" integrity sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg== is-string@^1.0.5, is-string@^1.0.7: version "1.0.7" - resolved "https://registry.yarnpkg.com/is-string/-/is-string-1.0.7.tgz#0dd12bf2006f255bb58f695110eff7491eebc0fd" + resolved "https://registry.npmjs.org/is-string/-/is-string-1.0.7.tgz" integrity sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg== dependencies: has-tostringtag "^1.0.0" is-symbol@^1.0.2, is-symbol@^1.0.3: version "1.0.4" - resolved "https://registry.yarnpkg.com/is-symbol/-/is-symbol-1.0.4.tgz#a6dac93b635b063ca6872236de88910a57af139c" + resolved "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.4.tgz" integrity sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg== dependencies: has-symbols "^1.0.2" -is-typed-array@^1.1.10, is-typed-array@^1.1.9: - version "1.1.11" - resolved "https://registry.yarnpkg.com/is-typed-array/-/is-typed-array-1.1.11.tgz#893621188e6919d4e6a488b9f6557d8c4b051953" - integrity sha512-l2SCJk9RflSWHQjOJJgNsV5FnE1pq/RpHnYW6ckSjTCYypv07SMbiRSCmLQD63WOv2eXaEwNsn+7kcn3csvYSw== - dependencies: - available-typed-arrays "^1.0.5" - call-bind "^1.0.2" - for-each "^0.3.3" - gopd "^1.0.1" - has-tostringtag "^1.0.0" - -is-typed-array@^1.1.3: - version "1.1.12" - resolved "https://registry.yarnpkg.com/is-typed-array/-/is-typed-array-1.1.12.tgz#d0bab5686ef4a76f7a73097b95470ab199c57d4a" - integrity sha512-Z14TF2JNG8Lss5/HMqt0//T9JeHXttXy5pH/DBU4vi98ozO2btxzq9MwYDZYnKwU8nRsz/+GVFVRDq3DkVuSPg== +is-typed-array@^1.1.13, is-typed-array@^1.1.3: + version "1.1.13" + resolved "https://registry.yarnpkg.com/is-typed-array/-/is-typed-array-1.1.13.tgz#d6c5ca56df62334959322d7d7dd1cca50debe229" + integrity sha512-uZ25/bUAlUY5fR4OKT4rZQEBrzQWYV9ZJYGGsUmEJ6thodVJ1HX64ePQ6Z0qPWP+m+Uq6e9UugrE38jeYsDSMw== dependencies: - which-typed-array "^1.1.11" + which-typed-array "^1.1.14" is-typedarray@^1.0.0: version "1.0.0" - resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" + resolved "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz" integrity sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA== -is-weakmap@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/is-weakmap/-/is-weakmap-2.0.1.tgz#5008b59bdc43b698201d18f62b37b2ca243e8cf2" - integrity sha512-NSBR4kH5oVj1Uwvv970ruUkCV7O1mzgVFO4/rev2cLRda9Tm9HrL70ZPut4rOHgY0FNrUu9BCbXA2sdQ+x0chA== +is-unicode-supported@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz#3f26c76a809593b52bfa2ecb5710ed2779b522a7" + integrity sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw== is-weakref@^1.0.2: version "1.0.2" - resolved "https://registry.yarnpkg.com/is-weakref/-/is-weakref-1.0.2.tgz#9529f383a9338205e89765e0392efc2f100f06f2" + resolved "https://registry.npmjs.org/is-weakref/-/is-weakref-1.0.2.tgz" integrity sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ== dependencies: call-bind "^1.0.2" -is-weakset@^2.0.1: - version "2.0.2" - resolved "https://registry.yarnpkg.com/is-weakset/-/is-weakset-2.0.2.tgz#4569d67a747a1ce5a994dfd4ef6dcea76e7c0a1d" - integrity sha512-t2yVvttHkQktwnNNmBQ98AhENLdPUTDTE21uPqAQ0ARwQfGeQKRVS0NNurH7bTf7RrvcVn1OOge45CnBeHCSmg== - dependencies: - call-bind "^1.0.2" - get-intrinsic "^1.1.1" - is-windows@^1.0.2: version "1.0.2" - resolved "https://registry.yarnpkg.com/is-windows/-/is-windows-1.0.2.tgz#d1850eb9791ecd18e6182ce12a30f396634bb19d" + resolved "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz" integrity sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA== -isarray@0.0.1: - version "0.0.1" - resolved "https://registry.yarnpkg.com/isarray/-/isarray-0.0.1.tgz#8a18acfca9a8f4177e09abfc6038939b05d1eedf" - integrity sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ== - isarray@^1.0.0, isarray@~1.0.0: version "1.0.0" - resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" + resolved "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz" integrity sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ== isarray@^2.0.5: @@ -3115,24 +3133,24 @@ isarray@^2.0.5: isexe@^2.0.0: version "2.0.0" - resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" + resolved "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz" integrity sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw== istanbul-lib-coverage@3.2.0, istanbul-lib-coverage@^3.0.0, istanbul-lib-coverage@^3.2.0: version "3.2.0" - resolved "https://registry.yarnpkg.com/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.0.tgz#189e7909d0a39fa5a3dfad5b03f71947770191d3" + resolved "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.0.tgz" integrity sha512-eOeJ5BHCmHYvQK7xt9GkdHuzuCGS1Y6g9Gvnx3Ym33fz/HpLRYxiS0wHNr+m/MBC8B647Xt608vCDEvhl9c6Mw== istanbul-lib-hook@^3.0.0: version "3.0.0" - resolved "https://registry.yarnpkg.com/istanbul-lib-hook/-/istanbul-lib-hook-3.0.0.tgz#8f84c9434888cc6b1d0a9d7092a76d239ebf0cc6" + resolved "https://registry.npmjs.org/istanbul-lib-hook/-/istanbul-lib-hook-3.0.0.tgz" integrity sha512-Pt/uge1Q9s+5VAZ+pCo16TYMWPBIl+oaNIjgLQxcX0itS6ueeaA+pEfThZpH8WxhFgCiEb8sAJY6MdUKgiIWaQ== dependencies: append-transform "^2.0.0" istanbul-lib-instrument@^4.0.0: version "4.0.3" - resolved "https://registry.yarnpkg.com/istanbul-lib-instrument/-/istanbul-lib-instrument-4.0.3.tgz#873c6fff897450118222774696a3f28902d77c1d" + resolved "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-4.0.3.tgz" integrity sha512-BXgQl9kf4WTCPCCpmFGoJkz/+uhvm7h7PFKUYxh7qarQd3ER33vHG//qaE8eN25l07YqZPpHXU9I09l/RD5aGQ== dependencies: "@babel/core" "^7.7.5" @@ -3142,7 +3160,7 @@ istanbul-lib-instrument@^4.0.0: istanbul-lib-processinfo@^2.0.2, istanbul-lib-processinfo@^2.0.3: version "2.0.3" - resolved "https://registry.yarnpkg.com/istanbul-lib-processinfo/-/istanbul-lib-processinfo-2.0.3.tgz#366d454cd0dcb7eb6e0e419378e60072c8626169" + resolved "https://registry.npmjs.org/istanbul-lib-processinfo/-/istanbul-lib-processinfo-2.0.3.tgz" integrity sha512-NkwHbo3E00oybX6NGJi6ar0B29vxyvNwoC7eJ4G4Yq28UfY758Hgn/heV8VRFhevPED4LXfFz0DQ8z/0kw9zMg== dependencies: archy "^1.0.0" @@ -3154,7 +3172,7 @@ istanbul-lib-processinfo@^2.0.2, istanbul-lib-processinfo@^2.0.3: istanbul-lib-report@^3.0.0: version "3.0.0" - resolved "https://registry.yarnpkg.com/istanbul-lib-report/-/istanbul-lib-report-3.0.0.tgz#7518fe52ea44de372f460a76b5ecda9ffb73d8a6" + resolved "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.0.tgz" integrity sha512-wcdi+uAKzfiGT2abPpKZ0hSU1rGQjUQnLvtY5MpQ7QCTahD3VODhcu4wcfY1YtkGaDD5yuydOLINXsfbus9ROw== dependencies: istanbul-lib-coverage "^3.0.0" @@ -3163,7 +3181,7 @@ istanbul-lib-report@^3.0.0: istanbul-lib-source-maps@^4.0.0: version "4.0.1" - resolved "https://registry.yarnpkg.com/istanbul-lib-source-maps/-/istanbul-lib-source-maps-4.0.1.tgz#895f3a709fcfba34c6de5a42939022f3e4358551" + resolved "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-4.0.1.tgz" integrity sha512-n3s8EwkdFIJCG3BPKBYvskgXGoy88ARzvegkitk60NxRdwltLOTaH7CUiMRXvwYorl0Q712iEjcWB+fK/MrWVw== dependencies: debug "^4.1.1" @@ -3172,7 +3190,7 @@ istanbul-lib-source-maps@^4.0.0: istanbul-reports@^3.0.2: version "3.1.5" - resolved "https://registry.yarnpkg.com/istanbul-reports/-/istanbul-reports-3.1.5.tgz#cc9a6ab25cb25659810e4785ed9d9fb742578bae" + resolved "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.1.5.tgz" integrity sha512-nUsEMa9pBt/NOHqbcbeJEgqIlY/K7rVWUX6Lql2orY5e9roQOthbR3vtY4zzf2orPELg80fnxxk9zUyPlgwD1w== dependencies: html-escaper "^2.0.0" @@ -3180,100 +3198,83 @@ istanbul-reports@^3.0.2: iterall@^1.2.1: version "1.3.0" - resolved "https://registry.yarnpkg.com/iterall/-/iterall-1.3.0.tgz#afcb08492e2915cbd8a0884eb93a8c94d0d72fea" + resolved "https://registry.npmjs.org/iterall/-/iterall-1.3.0.tgz" integrity sha512-QZ9qOMdF+QLHxy1QIpUHUU1D5pS2CG2P69LF6L6CPjPYA/XMOmKV3PZpawHoAjHNyB0swdVTRxdYT4tbBbxqwg== jackspeak@^1.4.2: version "1.4.2" - resolved "https://registry.yarnpkg.com/jackspeak/-/jackspeak-1.4.2.tgz#30ad5e4b7b36f9f3ae580e23272b1a386b4f6b93" + resolved "https://registry.npmjs.org/jackspeak/-/jackspeak-1.4.2.tgz" integrity sha512-GHeGTmnuaHnvS+ZctRB01bfxARuu9wW83ENbuiweu07SFcVlZrJpcshSre/keGT7YGBhLHg/+rXCNSrsEHKU4Q== dependencies: cliui "^7.0.4" jest-docblock@^29.7.0: version "29.7.0" - resolved "https://registry.yarnpkg.com/jest-docblock/-/jest-docblock-29.7.0.tgz#8fddb6adc3cdc955c93e2a87f61cfd350d5d119a" - integrity sha512-q617Auw3A612guyaFgsbFeYpNP5t2aoUNLwBUbc/0kD1R4t9ixDbyFTHd1nok4epoVFpr7PmeWHrhvuV3XaJ4g== + resolved "https://registry.npmjs.org/jest-docblock/-/jest-docblock-29.7.0.tgz" + integrity "sha1-j922rcPNyVXJPiqH9hz9NQ1dEZo= sha512-q617Auw3A612guyaFgsbFeYpNP5t2aoUNLwBUbc/0kD1R4t9ixDbyFTHd1nok4epoVFpr7PmeWHrhvuV3XaJ4g==" dependencies: detect-newline "^3.0.0" jmespath@0.16.0: version "0.16.0" - resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.16.0.tgz#b15b0a85dfd4d930d43e69ed605943c802785076" - integrity sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw== - -js-sdsl@^4.1.4: - version "4.1.4" - resolved "https://registry.yarnpkg.com/js-sdsl/-/js-sdsl-4.1.4.tgz#78793c90f80e8430b7d8dc94515b6c77d98a26a6" - integrity sha512-Y2/yD55y5jteOAmY50JbUZYwk3CP3wnLPEZnlR1w9oKhITrBEtAxwuWKebFf8hMrPMgbYwFoWK/lH2sBkErELw== + resolved "https://registry.npmjs.org/jmespath/-/jmespath-0.16.0.tgz" + integrity "sha1-sVsKhd/U2TDUPmntYFlDyAJ4UHY= sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw==" "js-tokens@^3.0.0 || ^4.0.0", js-tokens@^4.0.0: version "4.0.0" - resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499" + resolved "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz" integrity sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ== -js-yaml@4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-4.0.0.tgz#f426bc0ff4b4051926cd588c71113183409a121f" - integrity sha512-pqon0s+4ScYUvX30wxQi3PogGFAlUyH0awepWvwkj4jD4v+ova3RiYw8bmA6x2rDrEaj8i/oWKoRxpVNW+Re8Q== +js-yaml@4.1.0, js-yaml@^4.1.0: + version "4.1.0" + resolved "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz" + integrity sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA== dependencies: argparse "^2.0.1" js-yaml@^3.13.1: version "3.14.1" - resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.14.1.tgz#dae812fdb3825fa306609a8717383c50c36a0537" + resolved "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz" integrity sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g== dependencies: argparse "^1.0.7" esprima "^4.0.0" -js-yaml@^4.1.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-4.1.0.tgz#c1fb65f8f5017901cdd2c951864ba18458a10602" - integrity sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA== - dependencies: - argparse "^2.0.1" - jsesc@^2.5.1: version "2.5.2" - resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-2.5.2.tgz#80564d2e483dacf6e8ef209650a67df3f0c283a4" + resolved "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz" integrity sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA== json-schema-traverse@^0.4.1: version "0.4.1" - resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660" + resolved "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz" integrity sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg== json-stable-stringify-without-jsonify@^1.0.1: version "1.0.1" - resolved "https://registry.yarnpkg.com/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz#9db7b59496ad3f3cfef30a75142d2d930ad72651" + resolved "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz" integrity sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw== json-stringify-safe@^5.0.1: version "5.0.1" - resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb" + resolved "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz" integrity sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA== -json5@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/json5/-/json5-1.0.1.tgz#779fb0018604fa854eacbf6252180d83543e3dbe" - integrity sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow== +json5@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/json5/-/json5-1.0.2.tgz#63d98d60f21b313b77c4d6da18bfa69d80e1d593" + integrity sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA== dependencies: minimist "^1.2.0" -json5@^2.2.1: - version "2.2.1" - resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.1.tgz#655d50ed1e6f95ad1a3caababd2b0efda10b395c" - integrity sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA== - -json5@^2.2.2: +json5@^2.2.3: version "2.2.3" resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.3.tgz#78cd6f1a19bdc12b73db5ad0c61efd66c1e29283" integrity sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg== jszip@^3.5.0: version "3.10.1" - resolved "https://registry.yarnpkg.com/jszip/-/jszip-3.10.1.tgz#34aee70eb18ea1faec2f589208a157d1feb091c2" + resolved "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz" integrity sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g== dependencies: lie "~3.3.0" @@ -3281,14 +3282,14 @@ jszip@^3.5.0: readable-stream "~2.3.6" setimmediate "^1.0.5" -just-extend@^4.0.2: - version "4.2.1" - resolved "https://registry.yarnpkg.com/just-extend/-/just-extend-4.2.1.tgz#ef5e589afb61e5d66b24eca749409a8939a8c744" - integrity sha512-g3UB796vUFIY90VIv/WX3L2c8CS2MdWUww3CNrYmqza1Fg0DURc2K/O4YrnklBdQarSJ/y8JnJYDGc+1iumQjg== +just-extend@^6.2.0: + version "6.2.0" + resolved "https://registry.yarnpkg.com/just-extend/-/just-extend-6.2.0.tgz#b816abfb3d67ee860482e7401564672558163947" + integrity sha512-cYofQu2Xpom82S6qD778jBDpwvvy39s1l/hrYij2u9AMdQcGRpaBu6kY4mVhuno5kJVi1DAz4aiphA2WI1/OAw== knex@^2.4.2: version "2.4.2" - resolved "https://registry.yarnpkg.com/knex/-/knex-2.4.2.tgz#a34a289d38406dc19a0447a78eeaf2d16ebedd61" + resolved "https://registry.npmjs.org/knex/-/knex-2.4.2.tgz" integrity sha512-tMI1M7a+xwHhPxjbl/H9K1kHX+VncEYcvCx5K00M16bWvpYPKAZd6QrCu68PtHAdIZNQPWZn0GVhqVBEthGWCg== dependencies: colorette "2.0.19" @@ -3308,12 +3309,12 @@ knex@^2.4.2: koalas@^1.0.2: version "1.0.2" - resolved "https://registry.yarnpkg.com/koalas/-/koalas-1.0.2.tgz#318433f074235db78fae5661a02a8ca53ee295cd" + resolved "https://registry.npmjs.org/koalas/-/koalas-1.0.2.tgz" integrity sha512-RYhBbYaTTTHId3l6fnMZc3eGQNW6FVCqMG6AMwA5I1Mafr6AflaXeoi6x3xQuATRotGYRLk6+1ELZH4dstFNOA== levn@^0.4.1: version "0.4.1" - resolved "https://registry.yarnpkg.com/levn/-/levn-0.4.1.tgz#ae4562c007473b932a6200d403268dd2fffc6ade" + resolved "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz" integrity sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ== dependencies: prelude-ls "^1.2.1" @@ -3321,7 +3322,7 @@ levn@^0.4.1: libtap@^1.4.0: version "1.4.0" - resolved "https://registry.yarnpkg.com/libtap/-/libtap-1.4.0.tgz#5c6dea65d2d95f2c855d819a457e1fa7d2af5bf0" + resolved "https://registry.npmjs.org/libtap/-/libtap-1.4.0.tgz" integrity sha512-STLFynswQ2A6W14JkabgGetBNk6INL1REgJ9UeNKw5llXroC2cGLgKTqavv0sl8OLVztLLipVKMcQ7yeUcqpmg== dependencies: async-hook-domain "^2.0.4" @@ -3340,33 +3341,33 @@ libtap@^1.4.0: lie@~3.3.0: version "3.3.0" - resolved "https://registry.yarnpkg.com/lie/-/lie-3.3.0.tgz#dcf82dee545f46074daf200c7c1c5a08e0f40f6a" + resolved "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz" integrity sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ== dependencies: immediate "~3.0.5" -limiter@^1.1.4: +limiter@1.1.5: version "1.1.5" resolved "https://registry.yarnpkg.com/limiter/-/limiter-1.1.5.tgz#8f92a25b3b16c6131293a0cc834b4a838a2aa7c2" integrity sha512-FWWMIEOxz3GwUI4Ts/IvgVy6LPvoMPgjMdQ185nN6psJyBJ4yOpzqm695/h5umdLJg2vW3GR5iG11MAkR2AzJA== locate-path@^5.0.0: version "5.0.0" - resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-5.0.0.tgz#1afba396afd676a6d42504d0a67a3a7eb9f62aa0" + resolved "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz" integrity sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g== dependencies: p-locate "^4.1.0" locate-path@^6.0.0: version "6.0.0" - resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-6.0.0.tgz#55321eb309febbc59c4801d931a72452a681d286" + resolved "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz" integrity sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw== dependencies: p-locate "^5.0.0" lodash.flattendeep@^4.4.0: version "4.4.0" - resolved "https://registry.yarnpkg.com/lodash.flattendeep/-/lodash.flattendeep-4.4.0.tgz#fb030917f86a3134e5bc9bec0d69e0013ddfedb2" + resolved "https://registry.npmjs.org/lodash.flattendeep/-/lodash.flattendeep-4.4.0.tgz" integrity sha512-uHaJFihxmJcEX3kT4I23ABqKKalJ/zDrDg0lsFtc1h+3uw49SIJ5beyhx5ExVRti3AvKoOJngIj7xz3oylPdWQ== lodash.get@^4.4.2: @@ -3374,121 +3375,102 @@ lodash.get@^4.4.2: resolved "https://registry.yarnpkg.com/lodash.get/-/lodash.get-4.4.2.tgz#2d177f652fa31e939b4438d5341499dfa3825e99" integrity sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ== -lodash.kebabcase@^4.1.1: - version "4.1.1" - resolved "https://registry.yarnpkg.com/lodash.kebabcase/-/lodash.kebabcase-4.1.1.tgz#8489b1cb0d29ff88195cceca448ff6d6cc295c36" - integrity sha512-N8XRTIMMqqDgSy4VLKPnJ/+hpGZN+PHQiJnSenYqPaVV/NCqEogTnAdZLQiGKhxX+JCs8waWq2t1XHWKOmlY8g== - lodash.merge@^4.6.2: version "4.6.2" - resolved "https://registry.yarnpkg.com/lodash.merge/-/lodash.merge-4.6.2.tgz#558aa53b43b661e1925a0afdfa36a9a1085fe57a" + resolved "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz" integrity sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ== -lodash.pick@^4.4.0: - version "4.4.0" - resolved "https://registry.yarnpkg.com/lodash.pick/-/lodash.pick-4.4.0.tgz#52f05610fff9ded422611441ed1fc123a03001b3" - integrity sha512-hXt6Ul/5yWjfklSGvLQl8vM//l3FtyHZeuelpzK6mm99pNvN9yTDruNZPEJZD1oWrqo+izBmB7oUfWgcCX7s4Q== - lodash.sortby@^4.7.0: version "4.7.0" - resolved "https://registry.yarnpkg.com/lodash.sortby/-/lodash.sortby-4.7.0.tgz#edd14c824e2cc9c1e0b0a1b42bb5210516a42438" + resolved "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz" integrity sha512-HDWXG8isMntAyRF5vZ7xKuEvOhT4AhlRt/3czTSjvGUxjYCBVRQY48ViDHyfYz9VIoBkW4TMGQNapx+l3RUwdA== -lodash.uniq@^4.5.0: - version "4.5.0" - resolved "https://registry.yarnpkg.com/lodash.uniq/-/lodash.uniq-4.5.0.tgz#d0225373aeb652adc1bc82e4945339a842754773" - integrity sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ== - lodash@^4.17.13, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.17.4: version "4.17.21" - resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" + resolved "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== -log-symbols@4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/log-symbols/-/log-symbols-4.0.0.tgz#69b3cc46d20f448eccdb75ea1fa733d9e821c920" - integrity sha512-FN8JBzLx6CzeMrB0tg6pqlGU1wCrXW+ZXGH481kfsBqer0hToTIiHdjH4Mq8xJUbvATujKCvaREGWpGUionraA== +log-symbols@4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/log-symbols/-/log-symbols-4.1.0.tgz#3fbdbb95b4683ac9fc785111e792e558d4abd503" + integrity sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg== dependencies: - chalk "^4.0.0" + chalk "^4.1.0" + is-unicode-supported "^0.1.0" + +loglevel@^1.6.8: + version "1.9.2" + resolved "https://registry.yarnpkg.com/loglevel/-/loglevel-1.9.2.tgz#c2e028d6c757720107df4e64508530db6621ba08" + integrity sha512-HgMmCqIJSAKqo68l0rS2AanEWfkxaZ5wNiEFb5ggm08lDs9Xl2KxBlX3PTcaD2chBM1gXAYf491/M2Rv8Jwayg== + +long@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/long/-/long-4.0.0.tgz#9a7b71cfb7d361a194ea555241c92f7468d5bf28" + integrity sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA== long@^5.0.0: version "5.2.0" - resolved "https://registry.yarnpkg.com/long/-/long-5.2.0.tgz#2696dadf4b4da2ce3f6f6b89186085d94d52fd61" + resolved "https://registry.npmjs.org/long/-/long-5.2.0.tgz" integrity sha512-9RTUNjK60eJbx3uz+TEGF7fUr29ZDxR5QzXcyDpeSfeH28S9ycINflOgOlppit5U+4kNTe83KQnMEerw7GmE8w== loose-envify@^1.1.0: version "1.4.0" - resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.4.0.tgz#71ee51fa7be4caec1a63839f7e682d8132d30caf" + resolved "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz" integrity sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q== dependencies: js-tokens "^3.0.0 || ^4.0.0" loupe@^2.3.1: version "2.3.4" - resolved "https://registry.yarnpkg.com/loupe/-/loupe-2.3.4.tgz#7e0b9bffc76f148f9be769cb1321d3dcf3cb25f3" + resolved "https://registry.npmjs.org/loupe/-/loupe-2.3.4.tgz" integrity sha512-OvKfgCC2Ndby6aSTREl5aCCPTNIzlDfQZvZxNUrBrihDhL3xcrYegTblhmEiCrg2kKQz4XsFIaemE5BF4ybSaQ== dependencies: get-func-name "^2.0.0" lru-cache@^5.1.1: version "5.1.1" - resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-5.1.1.tgz#1da27e6710271947695daf6848e847f01d84b920" + resolved "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz" integrity sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w== dependencies: yallist "^3.0.2" -lru-cache@^6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-6.0.0.tgz#6d6fe6570ebd96aaf90fcad1dafa3b2566db3a94" - integrity sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA== - dependencies: - yallist "^4.0.0" - -lru-cache@^7.14.0: - version "7.14.0" - resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-7.14.0.tgz#21be64954a4680e303a09e9468f880b98a0b3c7f" - integrity sha512-EIRtP1GrSJny0dqb50QXRUNBxHJhcpxHC++M5tD7RYbvLLn5KVWKsbyswSSqDuU15UFi3bgTQIY8nhDMeF6aDQ== +lru-cache@^7.10.1, lru-cache@^7.14.0, lru-cache@^7.14.1: + version "7.18.3" + resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-7.18.3.tgz#f793896e0fd0e954a59dfdd82f0773808df6aa89" + integrity sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA== make-dir@^3.0.0, make-dir@^3.0.2: version "3.1.0" - resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-3.1.0.tgz#415e967046b3a7f1d185277d84aa58203726a13f" + resolved "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz" integrity sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw== dependencies: semver "^6.0.0" manage-path@^2.0.0: version "2.0.0" - resolved "https://registry.yarnpkg.com/manage-path/-/manage-path-2.0.0.tgz#f4cf8457b926eeee2a83b173501414bc76eb9597" + resolved "https://registry.npmjs.org/manage-path/-/manage-path-2.0.0.tgz" integrity sha512-NJhyB+PJYTpxhxZJ3lecIGgh4kwIY2RAh44XvAz9UlqthlQwtPBf62uBVR8XaD8CRuSjQ6TnZH2lNJkbLPZM2A== media-typer@0.3.0: version "0.3.0" - resolved "https://registry.yarnpkg.com/media-typer/-/media-typer-0.3.0.tgz#8710d7af0aa626f8fffa1ce00168545263255748" + resolved "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz" integrity sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ== -merge-descriptors@1.0.1, merge-descriptors@~1.0.0: +merge-descriptors@1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.3.tgz#d80319a65f3c7935351e5cfdac8f9318504dbed5" + integrity sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ== + +merge-descriptors@~1.0.0: version "1.0.1" - resolved "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.1.tgz#b00aaa556dd8b44568150ec9d1b953f3f90cbb61" + resolved "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz" integrity sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w== -merge2@^1.3.0, merge2@^1.4.1: - version "1.4.1" - resolved "https://registry.yarnpkg.com/merge2/-/merge2-1.4.1.tgz#4368892f885e907455a6fd7dc55c0c9d404990ae" - integrity sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg== - -methods@^1.1.2, methods@~1.1.2: +methods@~1.1.2: version "1.1.2" - resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee" + resolved "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz" integrity sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w== -micromatch@^4.0.4: - version "4.0.5" - resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.5.tgz#bc8999a7cbbf77cdc89f132f6e467051b49090c6" - integrity sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA== - dependencies: - braces "^3.0.2" - picomatch "^2.3.1" - mime-db@1.52.0: version "1.52.0" resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70" @@ -3503,126 +3485,115 @@ mime-types@^2.1.12, mime-types@~2.1.24, mime-types@~2.1.34: mime@1.6.0: version "1.6.0" - resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1" + resolved "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz" integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg== mimic-fn@^2.1.0: version "2.1.0" - resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b" + resolved "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz" integrity sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg== -minimatch@3.0.4: - version "3.0.4" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" - integrity sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA== +minimatch@4.2.1: + version "4.2.1" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-4.2.1.tgz#40d9d511a46bdc4e563c22c3080cde9c0d8299b4" + integrity sha512-9Uq1ChtSZO+Mxa/CL1eGizn2vRn3MlLgzhT0Iz8zaY8NdvxvB0d5QdPFmCKf7JKA9Lerx5vRrnwO03jsSfGG9g== dependencies: brace-expansion "^1.1.7" -minimatch@^3.0.4, minimatch@^3.1.1, minimatch@^3.1.2: +minimatch@^3.0.4, minimatch@^3.0.5, minimatch@^3.1.1, minimatch@^3.1.2: version "3.1.2" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b" + resolved "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz" integrity sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw== dependencies: brace-expansion "^1.1.7" -minimist@^1.2.0: - version "1.2.6" - resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.6.tgz#8637a5b759ea0d6e98702cfb3a9283323c93af44" - integrity sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q== - -minimist@^1.2.6: +minimist@^1.2.0, minimist@^1.2.6: version "1.2.7" - resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.7.tgz#daa1c4d91f507390437c6a8bc01078e7000c4d18" + resolved "https://registry.npmjs.org/minimist/-/minimist-1.2.7.tgz" integrity sha512-bzfL1YUZsP41gmu/qjrEk0Q6i2ix/cVeAhbCbqH9u3zYutS1cLg00qhrD0M2MVdCcx4Sc0UpP2eBWo9rotpq6g== -minimist@^1.2.8: - version "1.2.8" - resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c" - integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA== - minipass@^3.1.5, minipass@^3.1.6, minipass@^3.3.4: version "3.3.6" - resolved "https://registry.yarnpkg.com/minipass/-/minipass-3.3.6.tgz#7bba384db3a1520d18c9c0e5251c3444e95dd94a" + resolved "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz" integrity sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw== dependencies: yallist "^4.0.0" mkdirp@^0.5.0, mkdirp@^0.5.4: version "0.5.6" - resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.6.tgz#7def03d2432dcae4ba1d611445c48396062255f6" + resolved "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz" integrity sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw== dependencies: minimist "^1.2.6" mkdirp@^1.0.4: version "1.0.4" - resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" + resolved "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz" integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw== mkdirp@^3.0.1: version "3.0.1" - resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-3.0.1.tgz#e44e4c5607fb279c168241713cc6e0fea9adcb50" - integrity sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg== + resolved "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz" + integrity "sha1-5E5MVgf7J5wWgkFxPMbg/qmty1A= sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==" -mocha@8: - version "8.4.0" - resolved "https://registry.yarnpkg.com/mocha/-/mocha-8.4.0.tgz#677be88bf15980a3cae03a73e10a0fc3997f0cff" - integrity sha512-hJaO0mwDXmZS4ghXsvPVriOhsxQ7ofcpQdm8dE+jISUOKopitvnXFQmpRR7jd2K6VBG6E26gU3IAbXXGIbu4sQ== +mocha@^9: + version "9.2.2" + resolved "https://registry.yarnpkg.com/mocha/-/mocha-9.2.2.tgz#d70db46bdb93ca57402c809333e5a84977a88fb9" + integrity sha512-L6XC3EdwT6YrIk0yXpavvLkn8h+EU+Y5UcCHKECyMbdUIxyMuZj4bX4U9e1nvnvUUvQVsV2VHQr5zLdcUkhW/g== dependencies: "@ungap/promise-all-settled" "1.1.2" ansi-colors "4.1.1" browser-stdout "1.3.1" - chokidar "3.5.1" - debug "4.3.1" + chokidar "3.5.3" + debug "4.3.3" diff "5.0.0" escape-string-regexp "4.0.0" find-up "5.0.0" - glob "7.1.6" + glob "7.2.0" growl "1.10.5" he "1.2.0" - js-yaml "4.0.0" - log-symbols "4.0.0" - minimatch "3.0.4" + js-yaml "4.1.0" + log-symbols "4.1.0" + minimatch "4.2.1" ms "2.1.3" - nanoid "3.1.20" - serialize-javascript "5.0.1" + nanoid "3.3.1" + serialize-javascript "6.0.0" strip-json-comments "3.1.1" supports-color "8.1.1" which "2.0.2" - wide-align "1.1.3" - workerpool "6.1.0" + workerpool "6.2.0" yargs "16.2.0" yargs-parser "20.2.4" yargs-unparser "2.0.0" module-details-from-path@^1.0.3: version "1.0.3" - resolved "https://registry.yarnpkg.com/module-details-from-path/-/module-details-from-path-1.0.3.tgz#114c949673e2a8a35e9d35788527aa37b679da2b" + resolved "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.3.tgz" integrity sha512-ySViT69/76t8VhE1xXHK6Ch4NcDd26gx0MzKXLO+F7NOtnqH68d9zF94nT8ZWSxXh8ELOERsnJO/sWt1xZYw5A== module-not-found-error@^1.0.0: version "1.0.1" - resolved "https://registry.yarnpkg.com/module-not-found-error/-/module-not-found-error-1.0.1.tgz#cf8b4ff4f29640674d6cdd02b0e3bc523c2bbdc0" + resolved "https://registry.npmjs.org/module-not-found-error/-/module-not-found-error-1.0.1.tgz" integrity sha512-pEk4ECWQXV6z2zjhRZUongnLJNUeGQJ3w6OQ5ctGwD+i5o93qjRQUk2Rt6VdNeu3sEP0AB4LcfvdebpxBRVr4g== ms@2.0.0: version "2.0.0" - resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" + resolved "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz" integrity sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A== ms@2.1.2: version "2.1.2" - resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" + resolved "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== ms@2.1.3, ms@^2.1.1, ms@^2.1.2: version "2.1.3" - resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" + resolved "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== msgpack-lite@^0.1.26: version "0.1.26" - resolved "https://registry.yarnpkg.com/msgpack-lite/-/msgpack-lite-0.1.26.tgz#dd3c50b26f059f25e7edee3644418358e2a9ad89" + resolved "https://registry.npmjs.org/msgpack-lite/-/msgpack-lite-0.1.26.tgz" integrity sha512-SZ2IxeqZ1oRFGo0xFGbvBJWMp3yLIY9rlIJyxy8CGrwZn1f0ZK4r6jV/AM1r0FZMDUkWkglOk/eeKIL9g77Nxw== dependencies: event-lite "^0.1.1" @@ -3632,7 +3603,7 @@ msgpack-lite@^0.1.26: multer@^1.4.5-lts.1: version "1.4.5-lts.1" - resolved "https://registry.yarnpkg.com/multer/-/multer-1.4.5-lts.1.tgz#803e24ad1984f58edffbc79f56e305aec5cfd1ac" + resolved "https://registry.npmjs.org/multer/-/multer-1.4.5-lts.1.tgz" integrity sha512-ywPWvcDMeH+z9gQq5qYHCCy+ethsk4goepZ45GLD63fOu0YcNecQxi64nDs3qluZB+murG3/D4dJ7+dGctcCQQ== dependencies: append-field "^1.0.0" @@ -3643,35 +3614,40 @@ multer@^1.4.5-lts.1: type-is "^1.6.4" xtend "^4.0.0" -nanoid@3.1.20: - version "3.1.20" - resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.1.20.tgz#badc263c6b1dcf14b71efaa85f6ab4c1d6cfc788" - integrity sha512-a1cQNyczgKbLX9jwbS/+d7W8fX/RfgYR7lVWwWOGIPNgK2m0MWvrGF6/m4kk6U3QcFMnZf3RIhL0v2Jgh/0Uxw== +nanoid@3.3.1: + version "3.3.1" + resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.1.tgz#6347a18cac88af88f58af0b3594b723d5e99bb35" + integrity sha512-n6Vs/3KGyxPQd6uO0eH4Bv0ojGSUvuLlIHtC3Y0kEO23YRge8H9x1GCzLn28YX0H66pMkxuaeESFq4tKISKwdw== natural-compare@^1.4.0: version "1.4.0" - resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7" + resolved "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz" integrity sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw== negotiator@0.6.3: version "0.6.3" - resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.3.tgz#58e323a72fedc0d6f9cd4d31fe49f51479590ccd" + resolved "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz" integrity sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg== +negotiator@^0.6.3: + version "0.6.4" + resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.4.tgz#777948e2452651c570b712dd01c23e262713fff7" + integrity sha512-myRT3DiWPHqho5PrJaIRyaMv2kgYf0mUVgBNOYMuCH5Ki1yEiQaf/ZJuQ62nvpc44wL5WDbTX7yGJi1Neevw8w== + nise@^5.1.4: - version "5.1.4" - resolved "https://registry.yarnpkg.com/nise/-/nise-5.1.4.tgz#491ce7e7307d4ec546f5a659b2efe94a18b4bbc0" - integrity sha512-8+Ib8rRJ4L0o3kfmyVCL7gzrohyDe0cMFTBa2d364yIrEGMEoetznKJx899YxjybU6bL9SQkYPSBBs1gyYs8Xg== + version "5.1.9" + resolved "https://registry.yarnpkg.com/nise/-/nise-5.1.9.tgz#0cb73b5e4499d738231a473cd89bd8afbb618139" + integrity sha512-qOnoujW4SV6e40dYxJOb3uvuoPHtmLzIk4TFo+j0jPJoC+5Z9xja5qH5JZobEPsa8+YYphMrOSwnrshEhG2qww== dependencies: - "@sinonjs/commons" "^2.0.0" - "@sinonjs/fake-timers" "^10.0.2" - "@sinonjs/text-encoding" "^0.7.1" - just-extend "^4.0.2" - path-to-regexp "^1.7.0" + "@sinonjs/commons" "^3.0.0" + "@sinonjs/fake-timers" "^11.2.2" + "@sinonjs/text-encoding" "^0.7.2" + just-extend "^6.2.0" + path-to-regexp "^6.2.1" nock@^11.3.3: version "11.9.1" - resolved "https://registry.yarnpkg.com/nock/-/nock-11.9.1.tgz#2b026c5beb6d0dbcb41e7e4cefa671bc36db9c61" + resolved "https://registry.npmjs.org/nock/-/nock-11.9.1.tgz" integrity sha512-U5wPctaY4/ar2JJ5Jg4wJxlbBfayxgKbiAeGh+a1kk6Pwnc2ZEuKviLyDSG6t0uXl56q7AALIxoM6FJrBSsVXA== dependencies: debug "^4.1.0" @@ -3687,39 +3663,46 @@ node-abort-controller@^3.1.1: node-addon-api@^6.1.0: version "6.1.0" - resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-6.1.0.tgz#ac8470034e58e67d0c6f1204a18ae6995d9c0d76" + resolved "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz" integrity sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA== +node-fetch@^2.6.7: + version "2.7.0" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" + integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== + dependencies: + whatwg-url "^5.0.0" + node-gyp-build@<4.0, node-gyp-build@^3.9.0: version "3.9.0" - resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-3.9.0.tgz#53a350187dd4d5276750da21605d1cb681d09e25" + resolved "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-3.9.0.tgz" integrity sha512-zLcTg6P4AbcHPq465ZMFNXx7XpKKJh+7kkN699NiQWisR2uWYOWNWqRHAmbnmKiL4e9aLSlmy5U7rEMUXV59+A== node-gyp-build@^4.5.0: version "4.5.0" - resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.5.0.tgz#7a64eefa0b21112f89f58379da128ac177f20e40" + resolved "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.5.0.tgz" integrity sha512-2iGbaQBV+ITgCz76ZEjmhUKAKVf7xfY1sRl4UiKQspfZMH2h06SyhNsnSVy50cwkFQDGLyif6m/6uFXHkOZ6rg== node-preload@^0.2.1: version "0.2.1" - resolved "https://registry.yarnpkg.com/node-preload/-/node-preload-0.2.1.tgz#c03043bb327f417a18fee7ab7ee57b408a144301" + resolved "https://registry.npmjs.org/node-preload/-/node-preload-0.2.1.tgz" integrity sha512-RM5oyBy45cLEoHqCeh+MNuFAxO0vTFBLskvQbOKnEE7YTTSN4tbN8QWDIPQ6L+WvKsB/qLEGpYe2ZZ9d4W9OIQ== dependencies: process-on-spawn "^1.0.0" -node-releases@^2.0.6: - version "2.0.6" - resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.6.tgz#8a7088c63a55e493845683ebf3c828d8c51c5503" - integrity sha512-PiVXnNuFm5+iYkLBNeq5211hvO38y63T0i2KKh2KnUs3RpzJ+JtODFjkD8yjLwnDkTYF1eKXheUwdssR+NRZdg== +node-releases@^2.0.14: + version "2.0.14" + resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.14.tgz#2ffb053bceb8b2be8495ece1ab6ce600c4461b0b" + integrity "sha1-L/sFO864sr6Elezhq2zmAMRGGws= sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw==" normalize-path@^3.0.0, normalize-path@~3.0.0: version "3.0.0" - resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-3.0.0.tgz#0dcd69ff23a1c9b11fd0978316644a0388216a65" + resolved "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz" integrity sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA== nyc@^15.1.0: version "15.1.0" - resolved "https://registry.yarnpkg.com/nyc/-/nyc-15.1.0.tgz#1335dae12ddc87b6e249d5a1994ca4bdaea75f02" + resolved "https://registry.npmjs.org/nyc/-/nyc-15.1.0.tgz" integrity sha512-jMW04n9SxKdKi1ZMGhvUTHBN0EICCRkHemEoE5jm6mTYcqcdas0ATzgUgejlQUHMvpnOZqGB5Xxsv9KxJW1j8A== dependencies: "@istanbuljs/load-nyc-config" "^1.0.0" @@ -3750,163 +3733,169 @@ nyc@^15.1.0: test-exclude "^6.0.0" yargs "^15.0.2" -object-assign@^4.1.0, object-assign@^4.1.1: +object-assign@^4, object-assign@^4.1.0, object-assign@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" integrity sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg== -object-inspect@^1.12.2, object-inspect@^1.9.0: - version "1.12.2" - resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.12.2.tgz#c0641f26394532f28ab8d796ab954e43c009a8ea" - integrity sha512-z+cPxW0QGUp0mcqcsgQyLVRDoXFQbXOwBaqyF7VIgI4TWNQsDHrBpUQslRmIfAoYWdYzs6UlKJtB2XJpTaNSpQ== - -object-inspect@^1.12.3: - version "1.12.3" - resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.12.3.tgz#ba62dffd67ee256c8c086dfae69e016cd1f198b9" - integrity sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g== - -object-is@^1.1.5: - version "1.1.5" - resolved "https://registry.yarnpkg.com/object-is/-/object-is-1.1.5.tgz#b9deeaa5fc7f1846a0faecdceec138e5778f53ac" - integrity sha512-3cyDsyHgtmi7I7DfSSI2LDp6SK2lwvtbg0p0R1e0RvTqF5ceGx+K2dfSjm1bKDMVCFEDAQvy+o8c6a7VujOddw== - dependencies: - call-bind "^1.0.2" - define-properties "^1.1.3" +object-inspect@^1.13.1, object-inspect@^1.9.0: + version "1.13.2" + resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.13.2.tgz#dea0088467fb991e67af4058147a24824a3043ff" + integrity sha512-IRZSRuzJiynemAXPYtPe5BoI/RESNYR7TYm50MC5Mqbd3Jmw5y790sErYw3V6SryFJD64b74qQQs9wn5Bg/k3g== object-keys@^1.1.1: version "1.1.1" - resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.1.1.tgz#1c47f272df277f3b1daf061677d9c82e2322c60e" + resolved "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz" integrity sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA== -object.assign@^4.1.4: - version "4.1.4" - resolved "https://registry.yarnpkg.com/object.assign/-/object.assign-4.1.4.tgz#9673c7c7c351ab8c4d0b516f4343ebf4dfb7799f" - integrity sha512-1mxKf0e58bvyjSCtKYY4sRe9itRk3PJpquJOjeIkz885CczcI4IvJJDLPS72oowuSh+pBxUFROpX+TU++hxhZQ== +object.assign@^4.1.5: + version "4.1.5" + resolved "https://registry.yarnpkg.com/object.assign/-/object.assign-4.1.5.tgz#3a833f9ab7fdb80fc9e8d2300c803d216d8fdbb0" + integrity sha512-byy+U7gp+FVwmyzKPYhW2h5l3crpmGsxl7X2s8y43IgxvG4g3QZ6CffDtsNQy1WsmZpQbO+ybo0AlW7TY6DcBQ== dependencies: - call-bind "^1.0.2" - define-properties "^1.1.4" + call-bind "^1.0.5" + define-properties "^1.2.1" has-symbols "^1.0.3" object-keys "^1.1.1" -object.values@^1.1.5: - version "1.1.5" - resolved "https://registry.yarnpkg.com/object.values/-/object.values-1.1.5.tgz#959f63e3ce9ef108720333082131e4a459b716ac" - integrity sha512-QUZRW0ilQ3PnPpbNtgdNV1PDbEqLIiSFB3l+EnGtBQ/8SUTLj1PZwtQHABZtLgwpJZTSZhuGLOGk57Drx2IvYg== +object.fromentries@^2.0.7: + version "2.0.8" + resolved "https://registry.yarnpkg.com/object.fromentries/-/object.fromentries-2.0.8.tgz#f7195d8a9b97bd95cbc1999ea939ecd1a2b00c65" + integrity sha512-k6E21FzySsSK5a21KRADBd/NGneRegFO5pLHfdQLpRDETUNJueLXs3WCzyQ3tFRDYgbq3KHGXfTbi2bs8WQ6rQ== dependencies: - call-bind "^1.0.2" - define-properties "^1.1.3" - es-abstract "^1.19.1" + call-bind "^1.0.7" + define-properties "^1.2.1" + es-abstract "^1.23.2" + es-object-atoms "^1.0.0" + +object.groupby@^1.0.1: + version "1.0.3" + resolved "https://registry.yarnpkg.com/object.groupby/-/object.groupby-1.0.3.tgz#9b125c36238129f6f7b61954a1e7176148d5002e" + integrity sha512-+Lhy3TQTuzXI5hevh8sBGqbmurHbbIjAi0Z4S63nthVLmLxfbj4T54a4CfZrXIrt9iP4mVAPYMo/v99taj3wjQ== + dependencies: + call-bind "^1.0.7" + define-properties "^1.2.1" + es-abstract "^1.23.2" + +object.values@^1.1.7: + version "1.2.0" + resolved "https://registry.yarnpkg.com/object.values/-/object.values-1.2.0.tgz#65405a9d92cee68ac2d303002e0b8470a4d9ab1b" + integrity sha512-yBYjY9QX2hnRmZHAjG/f13MzmBzxzYgQhFrke06TTyKY5zSTEqkOeukBzIdVA3j3ulu8Qa3MbVFShV7T2RmGtQ== + dependencies: + call-bind "^1.0.7" + define-properties "^1.2.1" + es-object-atoms "^1.0.0" on-finished@2.4.1: version "2.4.1" - resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.4.1.tgz#58c8c44116e54845ad57f14ab10b03533184ac3f" + resolved "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz" integrity sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg== dependencies: ee-first "1.1.1" on-net-listen@^1.1.1: version "1.1.2" - resolved "https://registry.yarnpkg.com/on-net-listen/-/on-net-listen-1.1.2.tgz#671e55a81c910fa7e5b1e4d506545e9ea0f2e11c" + resolved "https://registry.npmjs.org/on-net-listen/-/on-net-listen-1.1.2.tgz" integrity sha512-y1HRYy8s/RlcBvDUwKXSmkODMdx4KSuIvloCnQYJ2LdBBC1asY4HtfhXwe3UWknLakATZDnbzht2Ijw3M1EqFg== once@^1.3.0: version "1.4.0" - resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" + resolved "https://registry.npmjs.org/once/-/once-1.4.0.tgz" integrity sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w== dependencies: wrappy "1" onetime@^5.1.0: version "5.1.2" - resolved "https://registry.yarnpkg.com/onetime/-/onetime-5.1.2.tgz#d0e96ebb56b07476df1dd9c4806e5237985ca45e" + resolved "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz" integrity sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg== dependencies: mimic-fn "^2.1.0" opener@^1.5.1: version "1.5.2" - resolved "https://registry.yarnpkg.com/opener/-/opener-1.5.2.tgz#5d37e1f35077b9dcac4301372271afdeb2a13598" + resolved "https://registry.npmjs.org/opener/-/opener-1.5.2.tgz" integrity sha512-ur5UIdyw5Y7yEj9wLzhqXiy6GZ3Mwx0yGI+5sMn2r0N0v3cKJvUmFH5yPP+WXh9e0xfyzyJX95D8l088DNFj7A== opentracing@>=0.12.1: version "0.14.7" - resolved "https://registry.yarnpkg.com/opentracing/-/opentracing-0.14.7.tgz#25d472bd0296dc0b64d7b94cbc995219031428f5" + resolved "https://registry.npmjs.org/opentracing/-/opentracing-0.14.7.tgz" integrity sha512-vz9iS7MJ5+Bp1URw8Khvdyw1H/hGvzHWlKQ7eRrQojSCDL1/SrWfrY9QebLw97n2deyRtzHRC3MkQfVNUCo91Q== optimist@~0.3.5: version "0.3.7" - resolved "https://registry.yarnpkg.com/optimist/-/optimist-0.3.7.tgz#c90941ad59e4273328923074d2cf2e7cbc6ec0d9" + resolved "https://registry.npmjs.org/optimist/-/optimist-0.3.7.tgz" integrity sha512-TCx0dXQzVtSCg2OgY/bO9hjM9cV4XYx09TVK+s3+FhkjT6LovsLe+pPMzpWf+6yXK/hUizs2gUoTw3jHM0VaTQ== dependencies: wordwrap "~0.0.2" -optionator@^0.9.1: - version "0.9.1" - resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.9.1.tgz#4f236a6373dae0566a6d43e1326674f50c291499" - integrity sha512-74RlY5FCnhq4jRxVUPKDaRwrVNXMqsGsiW6AJw4XK8hmtm10wC0ypZBLw5IIp85NZMr91+qd1RvvENwg7jjRFw== +optionator@^0.9.3: + version "0.9.4" + resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.9.4.tgz#7ea1c1a5d91d764fb282139c88fe11e182a3a734" + integrity sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g== dependencies: deep-is "^0.1.3" fast-levenshtein "^2.0.6" levn "^0.4.1" prelude-ls "^1.2.1" type-check "^0.4.0" - word-wrap "^1.2.3" + word-wrap "^1.2.5" own-or-env@^1.0.2: version "1.0.2" - resolved "https://registry.yarnpkg.com/own-or-env/-/own-or-env-1.0.2.tgz#84e78d2d5128f7ee8a59f741ad5aafb4256a7c89" + resolved "https://registry.npmjs.org/own-or-env/-/own-or-env-1.0.2.tgz" integrity sha512-NQ7v0fliWtK7Lkb+WdFqe6ky9XAzYmlkXthQrBbzlYbmFKoAYbDDcwmOm6q8kOuwSRXW8bdL5ORksploUJmWgw== dependencies: own-or "^1.0.0" own-or@^1.0.0: version "1.0.0" - resolved "https://registry.yarnpkg.com/own-or/-/own-or-1.0.0.tgz#4e877fbeda9a2ec8000fbc0bcae39645ee8bf8dc" + resolved "https://registry.npmjs.org/own-or/-/own-or-1.0.0.tgz" integrity sha512-NfZr5+Tdf6MB8UI9GLvKRs4cXY8/yB0w3xtt84xFdWy8hkGjn+JFc60VhzS/hFRfbyxFcGYMTjnF4Me+RbbqrA== p-limit@^2.2.0: version "2.3.0" - resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-2.3.0.tgz#3dd33c647a214fdfffd835933eb086da0dc21db1" + resolved "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz" integrity sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w== dependencies: p-try "^2.0.0" p-limit@^3.0.2, p-limit@^3.1.0: version "3.1.0" - resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-3.1.0.tgz#e1daccbe78d0d1388ca18c64fea38e3e57e3706b" + resolved "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz" integrity sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ== dependencies: yocto-queue "^0.1.0" p-locate@^4.1.0: version "4.1.0" - resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-4.1.0.tgz#a3428bb7088b3a60292f66919278b7c297ad4f07" + resolved "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz" integrity sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A== dependencies: p-limit "^2.2.0" p-locate@^5.0.0: version "5.0.0" - resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-5.0.0.tgz#83c8315c6785005e3bd021839411c9e110e6d834" + resolved "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz" integrity sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw== dependencies: p-limit "^3.0.2" p-map@^3.0.0: version "3.0.0" - resolved "https://registry.yarnpkg.com/p-map/-/p-map-3.0.0.tgz#d704d9af8a2ba684e2600d9a215983d4141a979d" + resolved "https://registry.npmjs.org/p-map/-/p-map-3.0.0.tgz" integrity sha512-d3qXVTF/s+W+CdJ5A29wywV2n8CQQYahlgz2bFiA+4eVNJbHJodPZ+/gXwPGh0bOqA+j8S+6+ckmvLGPk1QpxQ== dependencies: aggregate-error "^3.0.0" p-try@^2.0.0: version "2.2.0" - resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.2.0.tgz#cb2868540e313d61de58fafbe35ce9004d5540e6" + resolved "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz" integrity sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ== package-hash@^4.0.0: version "4.0.0" - resolved "https://registry.yarnpkg.com/package-hash/-/package-hash-4.0.0.tgz#3537f654665ec3cc38827387fc904c163c54f506" + resolved "https://registry.npmjs.org/package-hash/-/package-hash-4.0.0.tgz" integrity sha512-whdkPIooSu/bASggZ96BWVvZTRMOFxnyUG5PnTSGKoJE2gd5mbVNmR2Nj20QFzxYYgAXpoqC+AiXzl+UMRh7zQ== dependencies: graceful-fs "^4.1.15" @@ -3916,136 +3905,134 @@ package-hash@^4.0.0: pako@^1.0.3, pako@~1.0.2: version "1.0.11" - resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.11.tgz#6c9599d340d54dfd3946380252a35705a6b992bf" + resolved "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz" integrity sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw== parent-module@^1.0.0: version "1.0.1" - resolved "https://registry.yarnpkg.com/parent-module/-/parent-module-1.0.1.tgz#691d2709e78c79fae3a156622452d00762caaaa2" + resolved "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz" integrity sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g== dependencies: callsites "^3.0.0" parseurl@~1.3.3: version "1.3.3" - resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.3.tgz#9da19e7bee8d12dff0513ed5b76957793bc2e8d4" + resolved "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz" integrity sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ== patch-console@^1.0.0: version "1.0.0" - resolved "https://registry.yarnpkg.com/patch-console/-/patch-console-1.0.0.tgz#19b9f028713feb8a3c023702a8cc8cb9f7466f9d" + resolved "https://registry.npmjs.org/patch-console/-/patch-console-1.0.0.tgz" integrity sha512-nxl9nrnLQmh64iTzMfyylSlRozL7kAXIaxw1fVcLYdyhNkJCRUzirRZTikXGJsg+hc4fqpneTK6iU2H1Q8THSA== path-exists@^4.0.0: version "4.0.0" - resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-4.0.0.tgz#513bdbe2d3b95d7762e8c1137efa195c6c61b5b3" + resolved "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz" integrity sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w== path-is-absolute@^1.0.0: version "1.0.1" - resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" + resolved "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz" integrity sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg== path-key@^3.1.0: version "3.1.1" - resolved "https://registry.yarnpkg.com/path-key/-/path-key-3.1.1.tgz#581f6ade658cbba65a0d3380de7753295054f375" + resolved "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz" integrity sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q== path-parse@^1.0.7: version "1.0.7" - resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.7.tgz#fbc114b60ca42b30d9daf5858e4bd68bbedb6735" + resolved "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz" integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw== -path-to-regexp@0.1.7, path-to-regexp@^0.1.2: - version "0.1.7" - resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.7.tgz#df604178005f522f15eb4490e7247a1bfaa67f8c" - integrity sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ== - -path-to-regexp@^1.7.0: - version "1.8.0" - resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-1.8.0.tgz#887b3ba9d84393e87a0a0b9f4cb756198b53548a" - integrity sha512-n43JRhlUKUAlibEJhPeir1ncUID16QnEjNpwzNdO3Lm4ywrBpBZ5oLD0I6br9evr1Y9JTqwRtAh7JLoOzAQdVA== - dependencies: - isarray "0.0.1" +path-to-regexp@0.1.10, path-to-regexp@^0.1.10: + version "0.1.10" + resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.10.tgz#67e9108c5c0551b9e5326064387de4763c4d5f8b" + integrity sha512-7lf7qcQidTku0Gu3YDPc8DJ1q7OOucfa/BSsIwjuh56VU7katFvuM8hULfkwB3Fns/rsVF7PwPKVw1sl5KQS9w== -path-type@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/path-type/-/path-type-4.0.0.tgz#84ed01c0a7ba380afe09d90a8c180dcd9d03043b" - integrity sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw== +path-to-regexp@^6.2.1: + version "6.3.0" + resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-6.3.0.tgz#2b6a26a337737a8e1416f9272ed0766b1c0389f4" + integrity sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ== pathval@^1.1.1: version "1.1.1" - resolved "https://registry.yarnpkg.com/pathval/-/pathval-1.1.1.tgz#8534e77a77ce7ac5a2512ea21e0fdb8fcf6c3d8d" + resolved "https://registry.npmjs.org/pathval/-/pathval-1.1.1.tgz" integrity sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ== pg-connection-string@2.5.0: version "2.5.0" - resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.5.0.tgz#538cadd0f7e603fc09a12590f3b8a452c2c0cf34" + resolved "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.5.0.tgz" integrity sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ== picocolors@^1.0.0: version "1.0.0" - resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.0.tgz#cb5bdc74ff3f51892236eaf79d68bc44564ab81c" + resolved "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz" integrity sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ== -picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.3.1: +picomatch@^2.0.4, picomatch@^2.2.1: version "2.3.1" - resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42" + resolved "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz" integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA== pkg-dir@^4.1.0: version "4.2.0" - resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-4.2.0.tgz#f099133df7ede422e81d1d8448270eeb3e4261f3" + resolved "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz" integrity sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ== dependencies: find-up "^4.0.0" platform@^1.3.3: version "1.3.6" - resolved "https://registry.yarnpkg.com/platform/-/platform-1.3.6.tgz#48b4ce983164b209c2d45a107adb31f473a6e7a7" + resolved "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz" integrity sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg== -pprof-format@^2.0.7: - version "2.0.7" - resolved "https://registry.yarnpkg.com/pprof-format/-/pprof-format-2.0.7.tgz#526e4361f8b37d16b2ec4bb0696b5292de5046a4" - integrity sha512-1qWaGAzwMpaXJP9opRa23nPnt2Egi7RMNoNBptEE/XwHbcn4fC2b/4U4bKc5arkGkIh2ZabpF2bEb+c5GNHEKA== +possible-typed-array-names@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/possible-typed-array-names/-/possible-typed-array-names-1.0.0.tgz#89bb63c6fada2c3e90adc4a647beeeb39cc7bf8f" + integrity sha512-d7Uw+eZoloe0EHDIYoe+bQ5WXnGMOpmiZFTuMWCwpjzzkL2nTjcKiAk4hh8TjnGye2TwWOk3UXucZ+3rbmBa8Q== + +pprof-format@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/pprof-format/-/pprof-format-2.1.0.tgz#acc8d7773bcf4faf0a3d3df11bceefba7ac06664" + integrity sha512-0+G5bHH0RNr8E5hoZo/zJYsL92MhkZjwrHp3O2IxmY8RJL9ooKeuZ8Tm0ZNBw5sGZ9TiM71sthTjWoR2Vf5/xw== prelude-ls@^1.2.1: version "1.2.1" - resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.2.1.tgz#debc6489d7a6e6b0e7611888cec880337d316396" + resolved "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz" integrity sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g== pretty-bytes@^5.3.0: version "5.6.0" - resolved "https://registry.yarnpkg.com/pretty-bytes/-/pretty-bytes-5.6.0.tgz#356256f643804773c82f64723fe78c92c62beaeb" + resolved "https://registry.npmjs.org/pretty-bytes/-/pretty-bytes-5.6.0.tgz" integrity sha512-FFw039TmrBqFK8ma/7OL3sDz/VytdtJr044/QUJtH0wK9lb9jLq9tJyIxUwtQJHwar2BqtiA4iCWSwo9JLkzFg== process-nextick-args@~2.0.0: version "2.0.1" - resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2" + resolved "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz" integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag== process-on-spawn@^1.0.0: version "1.0.0" - resolved "https://registry.yarnpkg.com/process-on-spawn/-/process-on-spawn-1.0.0.tgz#95b05a23073d30a17acfdc92a440efd2baefdc93" + resolved "https://registry.npmjs.org/process-on-spawn/-/process-on-spawn-1.0.0.tgz" integrity sha512-1WsPDsUSMmZH5LeMLegqkPDrsGgsWwk1Exipy2hvB0o/F0ASzbpIctSCcZIK1ykJvtTJULEH+20WOFjMvGnCTg== dependencies: fromentries "^1.2.0" progress@^2.0.3: version "2.0.3" - resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8" + resolved "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz" integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA== propagate@^2.0.0: version "2.0.1" - resolved "https://registry.yarnpkg.com/propagate/-/propagate-2.0.1.tgz#40cdedab18085c792334e64f0ac17256d38f9a45" + resolved "https://registry.npmjs.org/propagate/-/propagate-2.0.1.tgz" integrity sha512-vGrhOavPSTz4QVNuBNdcNXePNdNMaO1xj9yBeH1ScQPjk/rhg9sSlCXPhMkFuaNNW/syTvYqsnbIJxMBfRbbag== -protobufjs@^7.2.4: - version "7.2.4" - resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-7.2.4.tgz#3fc1ec0cdc89dd91aef9ba6037ba07408485c3ae" - integrity sha512-AT+RJgD2sH8phPmCf7OUZR8xGdcJRga4+1cOaXJ64hvcSkVhNcRHOwIxUatPH15+nj59WAGTDv3LSGZPEQbJaQ== +protobufjs@^7.2.5: + version "7.2.5" + resolved "https://registry.npmjs.org/protobufjs/-/protobufjs-7.2.5.tgz" + integrity "sha1-RdXFc4em0poXqraEbcwoP5uOfy0= sha512-gGXRSXvxQ7UiPgfw8gevrfRWcTlSbOFg+p/N+JVJEK5VhueL2miT6qTymqAmjr1Q5WbOCyJbyrk6JfWKwlFn6A==" dependencies: "@protobufjs/aspromise" "^1.1.2" "@protobufjs/base64" "^1.1.2" @@ -4062,15 +4049,20 @@ protobufjs@^7.2.4: proxy-addr@~2.0.7: version "2.0.7" - resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.7.tgz#f19fe69ceab311eeb94b42e70e8c2070f9ba1025" + resolved "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz" integrity sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg== dependencies: forwarded "0.2.0" ipaddr.js "1.9.1" +proxy-from-env@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2" + integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg== + proxyquire@^1.8.0: version "1.8.0" - resolved "https://registry.yarnpkg.com/proxyquire/-/proxyquire-1.8.0.tgz#02d514a5bed986f04cbb2093af16741535f79edc" + resolved "https://registry.npmjs.org/proxyquire/-/proxyquire-1.8.0.tgz" integrity sha512-mZZq4F50qaBkngvlf9paNfaSb5gtJ0mFPnBjda4NxCpXpMAaVfSLguRr9y2KXF6koOSBf4AanD2inuEQw3aCcA== dependencies: fill-keys "^1.0.2" @@ -4079,67 +4071,52 @@ proxyquire@^1.8.0: punycode@1.3.2: version "1.3.2" - resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.3.2.tgz#9653a036fb7c1ee42342f2325cceefea3926c48d" - integrity sha512-RofWgt/7fL5wP1Y7fxE7/EmTLzQVnB0ycyibJ0OOHIlJqTNzglYFxVwETOcIoJqJmpDXJ9xImDv+Fq34F/d4Dw== + resolved "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz" + integrity "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0= sha512-RofWgt/7fL5wP1Y7fxE7/EmTLzQVnB0ycyibJ0OOHIlJqTNzglYFxVwETOcIoJqJmpDXJ9xImDv+Fq34F/d4Dw==" -punycode@^2.0.0: +punycode@^2.0.0, punycode@^2.1.0: version "2.3.0" - resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.3.0.tgz#f67fa67c94da8f4d0cfff981aee4118064199b8f" + resolved "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz" integrity sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA== -punycode@^2.1.0: - version "2.1.1" - resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" - integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A== - -qs@6.11.0: - version "6.11.0" - resolved "https://registry.yarnpkg.com/qs/-/qs-6.11.0.tgz#fd0d963446f7a65e1367e01abd85429453f0c37a" - integrity sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q== +qs@6.13.0: + version "6.13.0" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.13.0.tgz#6ca3bd58439f7e245655798997787b0d88a51906" + integrity sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg== dependencies: - side-channel "^1.0.4" + side-channel "^1.0.6" querystring@0.2.0: version "0.2.0" - resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620" - integrity sha512-X/xY82scca2tau62i9mDyU9K+I+djTMUsvwf7xnUX5GLvVzgJybOJf4Y6o9Zx3oJK/LSXg5tTZBjwzqVPaPO2g== + resolved "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz" + integrity "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA= sha512-X/xY82scca2tau62i9mDyU9K+I+djTMUsvwf7xnUX5GLvVzgJybOJf4Y6o9Zx3oJK/LSXg5tTZBjwzqVPaPO2g==" queue-microtask@^1.2.2: version "1.2.3" - resolved "https://registry.yarnpkg.com/queue-microtask/-/queue-microtask-1.2.3.tgz#4929228bbc724dfac43e0efb058caf7b6cfb6243" + resolved "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz" integrity sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A== -rambda@^7.1.0: - version "7.4.0" - resolved "https://registry.yarnpkg.com/rambda/-/rambda-7.4.0.tgz#61ec9de31d3dd6affe804de3bae04a5b818781e5" - integrity sha512-A9hihu7dUTLOUCM+I8E61V4kRXnN4DwYeK0DwCBydC1MqNI1PidyAtbtpsJlBBzK4icSctEcCQ1bGcLpBuETUQ== +rambda@^7.4.0: + version "7.5.0" + resolved "https://registry.yarnpkg.com/rambda/-/rambda-7.5.0.tgz#1865044c59bc0b16f63026c6e5a97e4b1bbe98fe" + integrity sha512-y/M9weqWAH4iopRd7EHDEQQvpFPHj1AA3oHozE9tfITHUtTR7Z9PSlIRRG2l1GuW7sefC1cXFfIcF+cgnShdBA== randombytes@^2.1.0: version "2.1.0" - resolved "https://registry.yarnpkg.com/randombytes/-/randombytes-2.1.0.tgz#df6f84372f0270dc65cdf6291349ab7a473d4f2a" + resolved "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz" integrity sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ== dependencies: safe-buffer "^5.1.0" range-parser@~1.2.1: version "1.2.1" - resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.1.tgz#3cf37023d199e1c24d1a55b84800c2f3e6468031" + resolved "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz" integrity sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg== -raw-body@2.5.1: - version "2.5.1" - resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.5.1.tgz#fe1b1628b181b700215e5fd42389f98b71392857" - integrity sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig== - dependencies: - bytes "3.1.2" - http-errors "2.0.0" - iconv-lite "0.4.24" - unpipe "1.0.0" - raw-body@2.5.2: version "2.5.2" - resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.5.2.tgz#99febd83b90e08975087e8f1f9419a149366b68a" - integrity sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA== + resolved "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz" + integrity "sha1-mf69g7kOCJdQh+jx+UGaFJNmtoo= sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==" dependencies: bytes "3.1.2" http-errors "2.0.0" @@ -4147,16 +4124,16 @@ raw-body@2.5.2: unpipe "1.0.0" react-devtools-core@^4.19.1: - version "4.27.5" - resolved "https://registry.yarnpkg.com/react-devtools-core/-/react-devtools-core-4.27.5.tgz#35e41c09e7662ea29948d3caaeeea82f068cbbac" - integrity sha512-QJTriF1V4oyIenViCvM6qQuvcevQsp0sbKkHBZIQOij+AwY9DdOBY+dOeuymUqO5zV61CbmGxWsAIjeWlFS++w== + version "4.28.5" + resolved "https://registry.yarnpkg.com/react-devtools-core/-/react-devtools-core-4.28.5.tgz#c8442b91f068cdf0c899c543907f7f27d79c2508" + integrity "sha1-yEQrkfBozfDImcVDkH9/J9ecJQg= sha512-cq/o30z9W2Wb4rzBefjv5fBalHU0rJGZCHAkf/RHSBWSSYwh8PlQTqqOJmgIIbBtpj27T6FIPXeomIjZtCNVqA==" dependencies: shell-quote "^1.6.1" ws "^7" react-reconciler@^0.26.2: version "0.26.2" - resolved "https://registry.yarnpkg.com/react-reconciler/-/react-reconciler-0.26.2.tgz#bbad0e2d1309423f76cf3c3309ac6c96e05e9d91" + resolved "https://registry.npmjs.org/react-reconciler/-/react-reconciler-0.26.2.tgz" integrity sha512-nK6kgY28HwrMNwDnMui3dvm3rCFjZrcGiuwLc5COUipBK5hWHLOxMJhSnSomirqWwjPBJKV1QcbkI0VJr7Gl1Q== dependencies: loose-envify "^1.1.0" @@ -4165,7 +4142,7 @@ react-reconciler@^0.26.2: react@^17.0.2: version "17.0.2" - resolved "https://registry.yarnpkg.com/react/-/react-17.0.2.tgz#d0b5cc516d29eb3eee383f75b62864cfb6800037" + resolved "https://registry.npmjs.org/react/-/react-17.0.2.tgz" integrity sha512-gnhPt75i/dq/z3/6q/0asP78D0u592D5L1pd7M8P+dck6Fu/jJeL6iVVK23fptSUZj8Vjf++7wXA8UNclGQcbA== dependencies: loose-envify "^1.1.0" @@ -4173,7 +4150,7 @@ react@^17.0.2: readable-stream@^2.2.2, readable-stream@~2.3.6: version "2.3.7" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57" + resolved "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz" integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw== dependencies: core-util-is "~1.0.0" @@ -4184,257 +4161,201 @@ readable-stream@^2.2.2, readable-stream@~2.3.6: string_decoder "~1.1.1" util-deprecate "~1.0.1" -readdirp@~3.5.0: - version "3.5.0" - resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-3.5.0.tgz#9ba74c019b15d365278d2e91bb8c48d7b4d42c9e" - integrity sha512-cMhu7c/8rdhkHXWsY+osBhfSy0JikwpHK/5+imo+LpeasTF8ouErHrlYkwT0++njiyuDvc7OFY5T3ukvZ8qmFQ== - dependencies: - picomatch "^2.2.1" - readdirp@~3.6.0: version "3.6.0" - resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-3.6.0.tgz#74a370bd857116e245b29cc97340cd431a02a6c7" + resolved "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz" integrity sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA== dependencies: picomatch "^2.2.1" rechoir@^0.8.0: version "0.8.0" - resolved "https://registry.yarnpkg.com/rechoir/-/rechoir-0.8.0.tgz#49f866e0d32146142da3ad8f0eff352b3215ff22" + resolved "https://registry.npmjs.org/rechoir/-/rechoir-0.8.0.tgz" integrity sha512-/vxpCXddiX8NGfGO/mTafwjq4aFa/71pvamip0++IQk3zG8cbCj0fifNPrjjF1XMXUne91jL9OoxmdykoEtifQ== dependencies: resolve "^1.20.0" redeyed@~2.1.0: version "2.1.1" - resolved "https://registry.yarnpkg.com/redeyed/-/redeyed-2.1.1.tgz#8984b5815d99cb220469c99eeeffe38913e6cc0b" + resolved "https://registry.npmjs.org/redeyed/-/redeyed-2.1.1.tgz" integrity sha512-FNpGGo1DycYAdnrKFxCMmKYgo/mILAqtRYbkdQD8Ep/Hk2PQ5+aEAEx+IU713RTDmuBaH0c8P5ZozurNu5ObRQ== dependencies: esprima "~4.0.0" -regexp.prototype.flags@^1.4.3: - version "1.4.3" - resolved "https://registry.yarnpkg.com/regexp.prototype.flags/-/regexp.prototype.flags-1.4.3.tgz#87cab30f80f66660181a3bb7bf5981a872b367ac" - integrity sha512-fjggEOO3slI6Wvgjwflkc4NFRCTZAu5CnNfBd5qOMYhWdn67nJBBu34/TkD++eeFmd8C9r9jfXJ27+nSiRkSUA== - dependencies: - call-bind "^1.0.2" - define-properties "^1.1.3" - functions-have-names "^1.2.2" - -regexp.prototype.flags@^1.5.0: - version "1.5.0" - resolved "https://registry.yarnpkg.com/regexp.prototype.flags/-/regexp.prototype.flags-1.5.0.tgz#fe7ce25e7e4cca8db37b6634c8a2c7009199b9cb" - integrity sha512-0SutC3pNudRKgquxGoRGIz946MZVHqbNfPjBdxeOhBrdgDKlRoXmYLQN9xRbrR09ZXWeGAdPuif7egofn6v5LA== +regexp.prototype.flags@^1.5.2: + version "1.5.2" + resolved "https://registry.yarnpkg.com/regexp.prototype.flags/-/regexp.prototype.flags-1.5.2.tgz#138f644a3350f981a858c44f6bb1a61ff59be334" + integrity sha512-NcDiDkTLuPR+++OCKB0nWafEmhg/Da8aUPLPMQbK+bxKKCm1/S5he+AqYa4PlMCVBalb4/yxIRub6qkEx5yJbw== dependencies: - call-bind "^1.0.2" - define-properties "^1.2.0" - functions-have-names "^1.2.3" - -regexpp@^3.0.0, regexpp@^3.2.0: - version "3.2.0" - resolved "https://registry.yarnpkg.com/regexpp/-/regexpp-3.2.0.tgz#0425a2768d8f23bad70ca4b90461fa2f1213e1b2" - integrity sha512-pq2bWo9mVD43nbts2wGv17XLiNLya+GklZ8kaDLV2Z08gDCsGpnKn9BFMepvWuHCbyVvY7J5o5+BVvoQbmlJLg== + call-bind "^1.0.6" + define-properties "^1.2.1" + es-errors "^1.3.0" + set-function-name "^2.0.1" reinterval@^1.1.0: version "1.1.0" - resolved "https://registry.yarnpkg.com/reinterval/-/reinterval-1.1.0.tgz#3361ecfa3ca6c18283380dd0bb9546f390f5ece7" + resolved "https://registry.npmjs.org/reinterval/-/reinterval-1.1.0.tgz" integrity sha512-QIRet3SYrGp0HUHO88jVskiG6seqUGC5iAG7AwI/BV4ypGcuqk9Du6YQBUOUqm9c8pw1eyLoIaONifRua1lsEQ== release-zalgo@^1.0.0: version "1.0.0" - resolved "https://registry.yarnpkg.com/release-zalgo/-/release-zalgo-1.0.0.tgz#09700b7e5074329739330e535c5a90fb67851730" + resolved "https://registry.npmjs.org/release-zalgo/-/release-zalgo-1.0.0.tgz" integrity sha512-gUAyHVHPPC5wdqX/LG4LWtRYtgjxyX78oanFNTMMyFEfOqdC54s3eE82imuWKbOeqYht2CrNf64Qb8vgmmtZGA== dependencies: es6-error "^4.0.1" require-directory@^2.1.1: version "2.1.1" - resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" + resolved "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz" integrity sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q== require-main-filename@^2.0.0: version "2.0.0" - resolved "https://registry.yarnpkg.com/require-main-filename/-/require-main-filename-2.0.0.tgz#d0b329ecc7cc0f61649f62215be69af54aa8989b" + resolved "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz" integrity sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg== resolve-from@^3.0.0: version "3.0.0" - resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-3.0.0.tgz#b22c7af7d9d6881bc8b6e653335eebcb0a188748" + resolved "https://registry.npmjs.org/resolve-from/-/resolve-from-3.0.0.tgz" integrity sha512-GnlH6vxLymXJNMBo7XP1fJIzBFbdYt49CuTwmB/6N53t+kMPRMFKz783LlQ4tv28XoQfMWinAJX6WCGf2IlaIw== resolve-from@^4.0.0: version "4.0.0" - resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-4.0.0.tgz#4abcd852ad32dd7baabfe9b40e00a36db5f392e6" + resolved "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz" integrity sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g== resolve-from@^5.0.0: version "5.0.0" - resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-5.0.0.tgz#c35225843df8f776df21c57557bc087e9dfdfc69" + resolved "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz" integrity sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw== -resolve@^1.20.0, resolve@^1.22.0: - version "1.22.1" - resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.1.tgz#27cb2ebb53f91abb49470a928bba7558066ac177" - integrity sha512-nBpuuYuY5jFsli/JIs1oldw6fOQCBioohqWZg/2hiaOybXOft4lonv85uDOKXdf8rhyK159cxU5cDcK/NKk8zw== - dependencies: - is-core-module "^2.9.0" - path-parse "^1.0.7" - supports-preserve-symlinks-flag "^1.0.0" - -resolve@^1.22.1, resolve@^1.3.3: - version "1.22.2" - resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.2.tgz#0ed0943d4e301867955766c9f3e1ae6d01c6845f" - integrity sha512-Sb+mjNHOULsBv818T40qSPeRiuWLyaGMa5ewydRLFimneixmVy2zdivRl+AF6jaYPC8ERxGDmFSiqui6SfPd+g== - dependencies: - is-core-module "^2.11.0" - path-parse "^1.0.7" - supports-preserve-symlinks-flag "^1.0.0" +resolve-pkg-maps@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz#616b3dc2c57056b5588c31cdf4b3d64db133720f" + integrity sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw== -resolve@^2.0.0-next.4: - version "2.0.0-next.4" - resolved "https://registry.yarnpkg.com/resolve/-/resolve-2.0.0-next.4.tgz#3d37a113d6429f496ec4752d2a2e58efb1fd4660" - integrity sha512-iMDbmAWtfU+MHpxt/I5iWI7cY6YVEZUQ3MBgPQ++XD1PELuJHIl82xBmObyP2KyQmkNB2dsqF7seoQQiAn5yDQ== +resolve@^1.20.0, resolve@^1.22.2, resolve@^1.22.4: + version "1.22.8" + resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.8.tgz#b6c87a9f2aa06dfab52e3d70ac8cde321fa5a48d" + integrity sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw== dependencies: - is-core-module "^2.9.0" + is-core-module "^2.13.0" path-parse "^1.0.7" supports-preserve-symlinks-flag "^1.0.0" resolve@~1.1.7: version "1.1.7" - resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.1.7.tgz#203114d82ad2c5ed9e8e0411b3932875e889e97b" + resolved "https://registry.npmjs.org/resolve/-/resolve-1.1.7.tgz" integrity sha512-9znBF0vBcaSN3W2j7wKvdERPwqTxSpCq+if5C0WoTCyV9n24rua28jeuQ2pL/HOf+yUe/Mef+H/5p60K0Id3bg== restore-cursor@^3.1.0: version "3.1.0" - resolved "https://registry.yarnpkg.com/restore-cursor/-/restore-cursor-3.1.0.tgz#39f67c54b3a7a58cea5236d95cf0034239631f7e" + resolved "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz" integrity sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA== dependencies: onetime "^5.1.0" signal-exit "^3.0.2" -resumer@^0.0.0: - version "0.0.0" - resolved "https://registry.yarnpkg.com/resumer/-/resumer-0.0.0.tgz#f1e8f461e4064ba39e82af3cdc2a8c893d076759" - integrity sha512-Fn9X8rX8yYF4m81rZCK/5VmrmsSbqS/i3rDLl6ZZHAXgC2nTAx3dhwG8q8odP/RmdLa2YrybDJaAMg+X1ajY3w== - dependencies: - through "~2.3.4" - retimer@^2.0.0: version "2.0.0" - resolved "https://registry.yarnpkg.com/retimer/-/retimer-2.0.0.tgz#e8bd68c5e5a8ec2f49ccb5c636db84c04063bbca" + resolved "https://registry.npmjs.org/retimer/-/retimer-2.0.0.tgz" integrity sha512-KLXY85WkEq2V2bKex/LOO1ViXVn2KGYe4PYysAdYdjmraYIUsVkXu8O4am+8+5UbaaGl1qho4aqAAPHNQ4GSbg== -retry@^0.13.1: +retry@0.13.1, retry@^0.13.1: version "0.13.1" - resolved "https://registry.yarnpkg.com/retry/-/retry-0.13.1.tgz#185b1587acf67919d63b357349e03537b2484658" - integrity sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg== + resolved "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz" + integrity "sha1-GFsVh6z2eRnWOzVzSeA1N7JIRlg= sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==" reusify@^1.0.4: version "1.0.4" - resolved "https://registry.yarnpkg.com/reusify/-/reusify-1.0.4.tgz#90da382b1e126efc02146e90845a88db12925d76" + resolved "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz" integrity sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw== +rfdc@^1.3.1: + version "1.3.1" + resolved "https://registry.yarnpkg.com/rfdc/-/rfdc-1.3.1.tgz#2b6d4df52dffe8bb346992a10ea9451f24373a8f" + integrity sha512-r5a3l5HzYlIC68TpmYKlxWjmOP6wiPJ1vWv2HeLhNsRZMrCkxeqxiHlQ21oXmQ4F3SiryXBHhAD7JZqvOJjFmg== + rimraf@^3.0.0, rimraf@^3.0.2: version "3.0.2" - resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-3.0.2.tgz#f1a5402ba6220ad52cc1282bac1ae3aa49fd061a" + resolved "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz" integrity sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA== dependencies: glob "^7.1.3" run-parallel@^1.1.9: version "1.2.0" - resolved "https://registry.yarnpkg.com/run-parallel/-/run-parallel-1.2.0.tgz#66d1368da7bdf921eb9d95bd1a9229e7f21a43ee" + resolved "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz" integrity sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA== dependencies: queue-microtask "^1.2.2" -safe-array-concat@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/safe-array-concat/-/safe-array-concat-1.0.0.tgz#2064223cba3c08d2ee05148eedbc563cd6d84060" - integrity sha512-9dVEFruWIsnie89yym+xWTAYASdpw3CJV7Li/6zBewGf9z2i1j31rP6jnY0pHEO4QZh6N0K11bFjWmdR8UGdPQ== +safe-array-concat@^1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/safe-array-concat/-/safe-array-concat-1.1.2.tgz#81d77ee0c4e8b863635227c721278dd524c20edb" + integrity sha512-vj6RsCsWBCf19jIeHEfkRMw8DPiBb+DMXklQ/1SGDHOMlHdPUkZXFQ2YdplS23zESTijAcurb1aSgJA3AgMu1Q== dependencies: - call-bind "^1.0.2" - get-intrinsic "^1.2.0" + call-bind "^1.0.7" + get-intrinsic "^1.2.4" has-symbols "^1.0.3" isarray "^2.0.5" -safe-buffer@5.2.1, safe-buffer@^5.1.0: +safe-buffer@5.2.1, safe-buffer@^5.0.1, safe-buffer@^5.1.0: version "5.2.1" - resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" + resolved "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz" integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== safe-buffer@~5.1.0, safe-buffer@~5.1.1: version "5.1.2" - resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" + resolved "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz" integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== -safe-regex-test@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/safe-regex-test/-/safe-regex-test-1.0.0.tgz#793b874d524eb3640d1873aad03596db2d4f2295" - integrity sha512-JBUUzyOgEwXQY1NuPtvcj/qcBDbDmEvWufhlnXZIm75DEHp+afM1r1ujJpJsV/gSM4t59tpDyPi1sd6ZaPFfsA== +safe-regex-test@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/safe-regex-test/-/safe-regex-test-1.0.3.tgz#a5b4c0f06e0ab50ea2c395c14d8371232924c377" + integrity sha512-CdASjNJPvRa7roO6Ra/gLYBTzYzzPyyBXxIMdGW3USQLyjWEls2RgW5UBTXaQVp+OrpeCK3bLem8smtmheoRuw== dependencies: - call-bind "^1.0.2" - get-intrinsic "^1.1.3" + call-bind "^1.0.6" + es-errors "^1.3.0" is-regex "^1.1.4" "safer-buffer@>= 2.1.2 < 3": version "2.1.2" - resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" + resolved "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== sax@1.2.1: version "1.2.1" - resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" - integrity sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA== + resolved "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz" + integrity "sha1-e45lYZCyKOgaZq6nSEgNgozS03o= sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA==" sax@>=0.6.0: version "1.3.0" - resolved "https://registry.yarnpkg.com/sax/-/sax-1.3.0.tgz#a5dbe77db3be05c9d1ee7785dbd3ea9de51593d0" - integrity sha512-0s+oAmw9zLl1V1cS9BtZN7JAd0cW5e0QH4W3LWEK6a4LaLEA2OTpGYWDY+6XasBLtz6wkm3u1xRw95mRuJ59WA== + resolved "https://registry.npmjs.org/sax/-/sax-1.3.0.tgz" + integrity "sha1-pdvnfbO+BcnR7neF29PqneUVk9A= sha512-0s+oAmw9zLl1V1cS9BtZN7JAd0cW5e0QH4W3LWEK6a4LaLEA2OTpGYWDY+6XasBLtz6wkm3u1xRw95mRuJ59WA==" scheduler@^0.20.2: version "0.20.2" - resolved "https://registry.yarnpkg.com/scheduler/-/scheduler-0.20.2.tgz#4baee39436e34aa93b4874bddcbf0fe8b8b50e91" + resolved "https://registry.npmjs.org/scheduler/-/scheduler-0.20.2.tgz" integrity sha512-2eWfGgAqqWFGqtdMmcL5zCMK1U8KlXv8SQFGglL3CEtd0aDVDWgeF/YoCmvln55m5zSk3J/20hTaSBeSObsQDQ== dependencies: loose-envify "^1.1.0" object-assign "^4.1.1" -semver@5.3.0: - version "5.3.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" - integrity sha512-mfmm3/H9+67MCVix1h+IXTpDwL6710LyHuk7+cWC9T1mE0qz4iHhh6r4hU2wrIT9iTsAAC2XQRvfblL028cpLw== - -semver@^6.0.0, semver@^6.3.0: - version "6.3.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d" - integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== - -semver@^7.0.0: - version "7.4.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-7.4.0.tgz#8481c92feffc531ab1e012a8ffc15bdd3a0f4318" - integrity sha512-RgOxM8Mw+7Zus0+zcLEUn8+JfoLpj/huFTItQy2hsM4khuC1HYRDp0cU482Ewn/Fcy6bCjufD8vAj7voC66KQw== - dependencies: - lru-cache "^6.0.0" - -semver@^7.3.8: - version "7.5.3" - resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.3.tgz#161ce8c2c6b4b3bdca6caadc9fa3317a4c4fe88e" - integrity sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ== - dependencies: - lru-cache "^6.0.0" +semver@^6.0.0, semver@^6.3.0, semver@^6.3.1: + version "6.3.1" + resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" + integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== -semver@^7.5.4: - version "7.5.4" - resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.4.tgz#483986ec4ed38e1c6c48c34894a9182dbff68a6e" - integrity sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA== - dependencies: - lru-cache "^6.0.0" +semver@^7.0.0, semver@^7.5.3, semver@^7.5.4: + version "7.6.3" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143" + integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A== -send@0.18.0: - version "0.18.0" - resolved "https://registry.yarnpkg.com/send/-/send-0.18.0.tgz#670167cc654b05f5aa4a767f9113bb371bc706be" - integrity sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg== +send@0.19.0: + version "0.19.0" + resolved "https://registry.yarnpkg.com/send/-/send-0.19.0.tgz#bbc5a388c8ea6c048967049dbeac0e4a3f09d7f8" + integrity sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw== dependencies: debug "2.6.9" depd "2.0.0" @@ -4450,88 +4371,118 @@ send@0.18.0: range-parser "~1.2.1" statuses "2.0.1" -serialize-javascript@5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/serialize-javascript/-/serialize-javascript-5.0.1.tgz#7886ec848049a462467a97d3d918ebb2aaf934f4" - integrity sha512-SaaNal9imEO737H2c05Og0/8LUXG7EnsZyMa8MzkmuHoELfT6txuj0cMqRj6zfPKnmQ1yasR4PCJc8x+M4JSPA== +serialize-javascript@6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/serialize-javascript/-/serialize-javascript-6.0.0.tgz#efae5d88f45d7924141da8b5c3a7a7e663fefeb8" + integrity sha512-Qr3TosvguFt8ePWqsvRfrKyQXIiW+nGbYpy8XK24NQHE83caxWt+mIymTT19DGFbNWNLfEwsrkSmN64lVWB9ag== dependencies: randombytes "^2.1.0" -serve-static@1.15.0: - version "1.15.0" - resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.15.0.tgz#faaef08cffe0a1a62f60cad0c4e513cff0ac9540" - integrity sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g== +serve-static@1.16.2: + version "1.16.2" + resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.16.2.tgz#b6a5343da47f6bdd2673848bf45754941e803296" + integrity sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw== dependencies: - encodeurl "~1.0.2" + encodeurl "~2.0.0" escape-html "~1.0.3" parseurl "~1.3.3" - send "0.18.0" + send "0.19.0" set-blocking@^2.0.0: version "2.0.0" - resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7" + resolved "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz" integrity sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw== -set-function-length@^1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/set-function-length/-/set-function-length-1.1.1.tgz#4bc39fafb0307224a33e106a7d35ca1218d659ed" - integrity sha512-VoaqjbBJKiWtg4yRcKBQ7g7wnGnLV3M8oLvVWwOk2PdYY6PEFegR1vezXR0tw6fZGF9csVakIRjrJiy2veSBFQ== +set-function-length@^1.2.1: + version "1.2.2" + resolved "https://registry.yarnpkg.com/set-function-length/-/set-function-length-1.2.2.tgz#aac72314198eaed975cf77b2c3b6b880695e5449" + integrity sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg== dependencies: - define-data-property "^1.1.1" - get-intrinsic "^1.2.1" + define-data-property "^1.1.4" + es-errors "^1.3.0" + function-bind "^1.1.2" + get-intrinsic "^1.2.4" gopd "^1.0.1" - has-property-descriptors "^1.0.0" + has-property-descriptors "^1.0.2" + +set-function-name@^2.0.1: + version "2.0.2" + resolved "https://registry.yarnpkg.com/set-function-name/-/set-function-name-2.0.2.tgz#16a705c5a0dc2f5e638ca96d8a8cd4e1c2b90985" + integrity sha512-7PGFlmtwsEADb0WYyvCMa1t+yke6daIG4Wirafur5kcf+MhUnPms1UeR0CKQdTZD81yESwMHbtn+TR+dMviakQ== + dependencies: + define-data-property "^1.1.4" + es-errors "^1.3.0" + functions-have-names "^1.2.3" + has-property-descriptors "^1.0.2" setimmediate@^1.0.5: version "1.0.5" - resolved "https://registry.yarnpkg.com/setimmediate/-/setimmediate-1.0.5.tgz#290cbb232e306942d7d7ea9b83732ab7856f8285" + resolved "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz" integrity sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA== setprototypeof@1.2.0: version "1.2.0" - resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.2.0.tgz#66c9a24a73f9fc28cbe66b09fed3d33dcaf1b424" + resolved "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz" integrity sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw== +sha.js@^2.4.11: + version "2.4.11" + resolved "https://registry.yarnpkg.com/sha.js/-/sha.js-2.4.11.tgz#37a5cf0b81ecbc6943de109ba2960d1b26584ae7" + integrity sha512-QMEp5B7cftE7APOjk5Y6xgrbWu+WkLVQwk8JNjZ8nKRciZaByEW6MubieAiToS7+dwvrjGhH8jRXz3MVd0AYqQ== + dependencies: + inherits "^2.0.1" + safe-buffer "^5.0.1" + shebang-command@^2.0.0: version "2.0.0" - resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-2.0.0.tgz#ccd0af4f8835fbdc265b82461aaf0c36663f34ea" + resolved "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz" integrity sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA== dependencies: shebang-regex "^3.0.0" shebang-regex@^3.0.0: version "3.0.0" - resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-3.0.0.tgz#ae16f1644d873ecad843b0307b143362d4c42172" + resolved "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz" integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A== -shell-quote@^1.6.1: +shell-quote@^1.6.1, shell-quote@^1.8.1: version "1.8.1" - resolved "https://registry.yarnpkg.com/shell-quote/-/shell-quote-1.8.1.tgz#6dbf4db75515ad5bac63b4f1894c3a154c766680" - integrity sha512-6j1W9l1iAs/4xYBI1SYOVZyFcCis9b4KCLQ8fgAGG07QvzaRLVVRQvAy85yNmmZSjYjg4MWh4gNvlPujU/5LpA== + resolved "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.1.tgz" + integrity "sha1-bb9Nt1UVrVusY7TxiUw6FUx2ZoA= sha512-6j1W9l1iAs/4xYBI1SYOVZyFcCis9b4KCLQ8fgAGG07QvzaRLVVRQvAy85yNmmZSjYjg4MWh4gNvlPujU/5LpA==" side-channel@^1.0.4: version "1.0.4" - resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.4.tgz#efce5c8fdc104ee751b25c58d4290011fa5ea2cf" + resolved "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz" integrity sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw== dependencies: call-bind "^1.0.0" get-intrinsic "^1.0.2" object-inspect "^1.9.0" +side-channel@^1.0.6: + version "1.0.6" + resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.6.tgz#abd25fb7cd24baf45466406b1096b7831c9215f2" + integrity sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA== + dependencies: + call-bind "^1.0.7" + es-errors "^1.3.0" + get-intrinsic "^1.2.4" + object-inspect "^1.13.1" + signal-exit@^3.0.2, signal-exit@^3.0.4, signal-exit@^3.0.6: version "3.0.7" - resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.7.tgz#a9a1767f8af84155114eaabd73f99273c8f59ad9" + resolved "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz" integrity sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ== sinon-chai@^3.7.0: version "3.7.0" - resolved "https://registry.yarnpkg.com/sinon-chai/-/sinon-chai-3.7.0.tgz#cfb7dec1c50990ed18c153f1840721cf13139783" + resolved "https://registry.npmjs.org/sinon-chai/-/sinon-chai-3.7.0.tgz" integrity sha512-mf5NURdUaSdnatJx3uhoBOrY9dtL19fiOtAdT1Azxg3+lNJFiuN0uzaU3xX1LeAfL17kHQhTAJgpsfhbMJMY2g== -sinon@^15.2.0: - version "15.2.0" - resolved "https://registry.yarnpkg.com/sinon/-/sinon-15.2.0.tgz#5e44d4bc5a9b5d993871137fd3560bebfac27565" - integrity sha512-nPS85arNqwBXaIsFCkolHjGIkFo+Oxu9vbgmBJizLAhqe6P2o3Qmj3KCUoRkfhHtvgDhZdWD3risLHAUJ8npjw== +sinon@^16.1.3: + version "16.1.3" + resolved "https://registry.yarnpkg.com/sinon/-/sinon-16.1.3.tgz#b760ddafe785356e2847502657b4a0da5501fba8" + integrity sha512-mjnWWeyxcAf9nC0bXcPmiDut+oE8HYridTNzBbF98AYVLmWwGRp2ISEpyhYflG1ifILT+eNn3BmKUJPxjXUPlA== dependencies: "@sinonjs/commons" "^3.0.0" "@sinonjs/fake-timers" "^10.3.0" @@ -4540,14 +4491,9 @@ sinon@^15.2.0: nise "^5.1.4" supports-color "^7.2.0" -slash@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/slash/-/slash-3.0.0.tgz#6539be870c165adbd5240220dbe361f1bc4d4634" - integrity sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q== - slice-ansi@^3.0.0: version "3.0.0" - resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-3.0.0.tgz#31ddc10930a1b7e0b67b08c96c2f49b77a789787" + resolved "https://registry.npmjs.org/slice-ansi/-/slice-ansi-3.0.0.tgz" integrity sha512-pSyv7bSTC7ig9Dcgbw9AuRNUb5k5V6oDudjZoMBSr13qpLBG7tB+zgCkARjq7xIUgdz5P1Qe8u+rSGdouOOIyQ== dependencies: ansi-styles "^4.0.0" @@ -4556,7 +4502,7 @@ slice-ansi@^3.0.0: source-map-support@^0.5.16: version "0.5.21" - resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.21.tgz#04fe7c7f9e1ed2d662233c28cb2b35b9f63f6e4f" + resolved "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz" integrity sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w== dependencies: buffer-from "^1.0.0" @@ -4569,12 +4515,12 @@ source-map@^0.6.0, source-map@^0.6.1: source-map@^0.7.4: version "0.7.4" - resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.7.4.tgz#a9bbe705c9d8846f4e08ff6765acf0f1b0898656" + resolved "https://registry.npmjs.org/source-map/-/source-map-0.7.4.tgz" integrity sha512-l3BikUxvPOcn5E74dZiq5BGsTb5yEwhaTSzccU6t4sDOH8NWJCstKO5QT2CvtFoK6F0saL7p9xHAqHOlCPJygA== spawn-wrap@^2.0.0: version "2.0.0" - resolved "https://registry.yarnpkg.com/spawn-wrap/-/spawn-wrap-2.0.0.tgz#103685b8b8f9b79771318827aa78650a610d457e" + resolved "https://registry.npmjs.org/spawn-wrap/-/spawn-wrap-2.0.0.tgz" integrity sha512-EeajNjfN9zMnULLwhZZQU3GWBoFNkbngTUPfaawT4RkMiviTxcX0qfhVbGey39mfctfDHkWtuecgQ8NJcyQWHg== dependencies: foreground-child "^2.0.0" @@ -4586,36 +4532,29 @@ spawn-wrap@^2.0.0: sprintf-js@~1.0.2: version "1.0.3" - resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" + resolved "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz" integrity sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g== stack-utils@^2.0.2, stack-utils@^2.0.4: version "2.0.6" - resolved "https://registry.yarnpkg.com/stack-utils/-/stack-utils-2.0.6.tgz#aaf0748169c02fc33c8232abccf933f54a1cc34f" + resolved "https://registry.npmjs.org/stack-utils/-/stack-utils-2.0.6.tgz" integrity sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ== dependencies: escape-string-regexp "^2.0.0" statuses@2.0.1: version "2.0.1" - resolved "https://registry.yarnpkg.com/statuses/-/statuses-2.0.1.tgz#55cb000ccf1d48728bd23c685a063998cf1a1b63" + resolved "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz" integrity sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ== -stop-iteration-iterator@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/stop-iteration-iterator/-/stop-iteration-iterator-1.0.0.tgz#6a60be0b4ee757d1ed5254858ec66b10c49285e4" - integrity sha512-iCGQj+0l0HOdZ2AEeBADlsRC+vsnDsZsbdSiH1yNSjcfKM7fdpCMfqAL/dwF5BLiw/XhRft/Wax6zQbhq2BcjQ== - dependencies: - internal-slot "^1.0.4" - streamsearch@^1.1.0: version "1.1.0" - resolved "https://registry.yarnpkg.com/streamsearch/-/streamsearch-1.1.0.tgz#404dd1e2247ca94af554e841a8ef0eaa238da764" + resolved "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz" integrity sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg== -"string-width@^1.0.2 || 2", string-width@^2.1.1: +string-width@^2.1.1: version "2.1.1" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-2.1.1.tgz#ab93f27a8dc13d28cac815c462143a6d9012ae9e" + resolved "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz" integrity sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw== dependencies: is-fullwidth-code-point "^2.0.0" @@ -4623,123 +4562,106 @@ streamsearch@^1.1.0: string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2: version "4.2.3" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" + resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== dependencies: emoji-regex "^8.0.0" is-fullwidth-code-point "^3.0.0" strip-ansi "^6.0.1" -string.prototype.trim@^1.2.7: - version "1.2.7" - resolved "https://registry.yarnpkg.com/string.prototype.trim/-/string.prototype.trim-1.2.7.tgz#a68352740859f6893f14ce3ef1bb3037f7a90533" - integrity sha512-p6TmeT1T3411M8Cgg9wBTMRtY2q9+PNy9EV1i2lIXUN/btt763oIfxwN3RR8VU6wHX8j/1CFy0L+YuThm6bgOg== - dependencies: - call-bind "^1.0.2" - define-properties "^1.1.4" - es-abstract "^1.20.4" - -string.prototype.trimend@^1.0.5: - version "1.0.5" - resolved "https://registry.yarnpkg.com/string.prototype.trimend/-/string.prototype.trimend-1.0.5.tgz#914a65baaab25fbdd4ee291ca7dde57e869cb8d0" - integrity sha512-I7RGvmjV4pJ7O3kdf+LXFpVfdNOxtCW/2C8f6jNiW4+PQchwxkCDzlk1/7p+Wl4bqFIZeF47qAHXLuHHWKAxog== - dependencies: - call-bind "^1.0.2" - define-properties "^1.1.4" - es-abstract "^1.19.5" - -string.prototype.trimend@^1.0.6: - version "1.0.6" - resolved "https://registry.yarnpkg.com/string.prototype.trimend/-/string.prototype.trimend-1.0.6.tgz#c4a27fa026d979d79c04f17397f250a462944533" - integrity sha512-JySq+4mrPf9EsDBEDYMOb/lM7XQLulwg5R/m1r0PXEFqrV0qHvl58sdTilSXtKOflCsK2E8jxf+GKC0T07RWwQ== +string.prototype.trim@^1.2.9: + version "1.2.9" + resolved "https://registry.yarnpkg.com/string.prototype.trim/-/string.prototype.trim-1.2.9.tgz#b6fa326d72d2c78b6df02f7759c73f8f6274faa4" + integrity sha512-klHuCNxiMZ8MlsOihJhJEBJAiMVqU3Z2nEXWfWnIqjN0gEFS9J9+IxKozWWtQGcgoa1WUZzLjKPTr4ZHNFTFxw== dependencies: - call-bind "^1.0.2" - define-properties "^1.1.4" - es-abstract "^1.20.4" + call-bind "^1.0.7" + define-properties "^1.2.1" + es-abstract "^1.23.0" + es-object-atoms "^1.0.0" -string.prototype.trimstart@^1.0.5: - version "1.0.5" - resolved "https://registry.yarnpkg.com/string.prototype.trimstart/-/string.prototype.trimstart-1.0.5.tgz#5466d93ba58cfa2134839f81d7f42437e8c01fef" - integrity sha512-THx16TJCGlsN0o6dl2o6ncWUsdgnLRSA23rRE5pyGBw/mLr3Ej/R2LaqCtgP8VNMGZsvMWnf9ooZPyY2bHvUFg== +string.prototype.trimend@^1.0.8: + version "1.0.8" + resolved "https://registry.yarnpkg.com/string.prototype.trimend/-/string.prototype.trimend-1.0.8.tgz#3651b8513719e8a9f48de7f2f77640b26652b229" + integrity sha512-p73uL5VCHCO2BZZ6krwwQE3kCzM7NKmis8S//xEC6fQonchbum4eP6kR4DLEjQFO3Wnj3Fuo8NM0kOSjVdHjZQ== dependencies: - call-bind "^1.0.2" - define-properties "^1.1.4" - es-abstract "^1.19.5" + call-bind "^1.0.7" + define-properties "^1.2.1" + es-object-atoms "^1.0.0" -string.prototype.trimstart@^1.0.6: - version "1.0.6" - resolved "https://registry.yarnpkg.com/string.prototype.trimstart/-/string.prototype.trimstart-1.0.6.tgz#e90ab66aa8e4007d92ef591bbf3cd422c56bdcf4" - integrity sha512-omqjMDaY92pbn5HOX7f9IccLA+U1tA9GvtU4JrodiXFfYB7jPzzHpRzpglLAjtUV6bB557zwClJezTqnAiYnQA== +string.prototype.trimstart@^1.0.8: + version "1.0.8" + resolved "https://registry.yarnpkg.com/string.prototype.trimstart/-/string.prototype.trimstart-1.0.8.tgz#7ee834dda8c7c17eff3118472bb35bfedaa34dde" + integrity sha512-UXSH262CSZY1tfu3G3Secr6uGLCFVPMhIqHjlgCUtCCcgihYc/xKs9djMTMUOb2j1mVSeU8EU6NWc/iQKU6Gfg== dependencies: - call-bind "^1.0.2" - define-properties "^1.1.4" - es-abstract "^1.20.4" + call-bind "^1.0.7" + define-properties "^1.2.1" + es-object-atoms "^1.0.0" string_decoder@~1.1.1: version "1.1.1" - resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8" + resolved "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz" integrity sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg== dependencies: safe-buffer "~5.1.0" strip-ansi@^4.0.0: version "4.0.0" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-4.0.0.tgz#a8479022eb1ac368a871389b635262c505ee368f" + resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz" integrity sha512-4XaJ2zQdCzROZDivEVIDPkcQn8LMFSa8kj8Gxb/Lnwzv9A8VctNZ+lfivC/sV3ivW8ElJTERXZoPBRrZKkNKow== dependencies: ansi-regex "^3.0.0" strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" + resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== dependencies: ansi-regex "^5.0.1" strip-bom@^3.0.0: version "3.0.0" - resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-3.0.0.tgz#2334c18e9c759f7bdd56fdef7e9ae3d588e68ed3" + resolved "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz" integrity sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA== strip-bom@^4.0.0: version "4.0.0" - resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-4.0.0.tgz#9c3505c1db45bcedca3d9cf7a16f5c5aa3901878" + resolved "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz" integrity sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w== -strip-json-comments@3.1.1, strip-json-comments@^3.1.0, strip-json-comments@^3.1.1: +strip-json-comments@3.1.1, strip-json-comments@^3.1.1: version "3.1.1" - resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-3.1.1.tgz#31f1281b3832630434831c310c01cccda8cbe006" + resolved "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz" integrity sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig== supports-color@8.1.1: version "8.1.1" - resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-8.1.1.tgz#cd6fc17e28500cff56c1b86c0a7fd4a54a73005c" + resolved "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz" integrity sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q== dependencies: has-flag "^4.0.0" supports-color@^5.3.0: version "5.5.0" - resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.5.0.tgz#e2e69a44ac8772f78a1ec0b35b689df6530efc8f" + resolved "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz" integrity sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow== dependencies: has-flag "^3.0.0" supports-color@^7.1.0, supports-color@^7.2.0: version "7.2.0" - resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-7.2.0.tgz#1b7dcdcb32b8138801b3e478ba6a51caa89648da" + resolved "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz" integrity sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw== dependencies: has-flag "^4.0.0" supports-preserve-symlinks-flag@^1.0.0: version "1.0.0" - resolved "https://registry.yarnpkg.com/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz#6eda4bd344a3c94aea376d4cc31bc77311039e09" + resolved "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz" integrity sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w== tap-mocha-reporter@^5.0.3: version "5.0.3" - resolved "https://registry.yarnpkg.com/tap-mocha-reporter/-/tap-mocha-reporter-5.0.3.tgz#3e261b2a43092ba8bc0cb67a89b33e283decee05" + resolved "https://registry.npmjs.org/tap-mocha-reporter/-/tap-mocha-reporter-5.0.3.tgz" integrity sha512-6zlGkaV4J+XMRFkN0X+yuw6xHbE9jyCZ3WUKfw4KxMyRGOpYSRuuQTRJyWX88WWuLdVTuFbxzwXhXuS2XE6o0g== dependencies: color-support "^1.1.0" @@ -4753,7 +4675,7 @@ tap-mocha-reporter@^5.0.3: tap-parser@^11.0.0, tap-parser@^11.0.2: version "11.0.2" - resolved "https://registry.yarnpkg.com/tap-parser/-/tap-parser-11.0.2.tgz#5d3e76e2cc521e23a8c50201487b273ca0fba800" + resolved "https://registry.npmjs.org/tap-parser/-/tap-parser-11.0.2.tgz" integrity sha512-6qGlC956rcORw+fg7Fv1iCRAY8/bU9UabUAhs3mXRH6eRmVZcNPLheSXCYaVaYeSwx5xa/1HXZb1537YSvwDZg== dependencies: events-to-array "^1.0.1" @@ -4762,15 +4684,15 @@ tap-parser@^11.0.0, tap-parser@^11.0.2: tap-yaml@^1.0.0, tap-yaml@^1.0.2: version "1.0.2" - resolved "https://registry.yarnpkg.com/tap-yaml/-/tap-yaml-1.0.2.tgz#62032a459e5524e10661c19ee9df5d33d78812fa" + resolved "https://registry.npmjs.org/tap-yaml/-/tap-yaml-1.0.2.tgz" integrity sha512-GegASpuqBnRNdT1U+yuUPZ8rEU64pL35WPBpCISWwff4dErS2/438barz7WFJl4Nzh3Y05tfPidZnH+GaV1wMg== dependencies: yaml "^1.10.2" tap@^16.3.7: version "16.3.7" - resolved "https://registry.yarnpkg.com/tap/-/tap-16.3.7.tgz#1d3561b58dd7af3aed172a2f6fc3ad8252b040ab" - integrity sha512-AaovVsfXVKcIf9eD1NxgwIqSDz5LauvybTpS6bjAKVYqz3+iavHC1abwxTkXmswb2n7eq8qKLt8DvY3D6iWcYA== + resolved "https://registry.npmjs.org/tap/-/tap-16.3.7.tgz" + integrity "sha1-HTVhtY3XrzrtFyovb8OtglKwQKs= sha512-AaovVsfXVKcIf9eD1NxgwIqSDz5LauvybTpS6bjAKVYqz3+iavHC1abwxTkXmswb2n7eq8qKLt8DvY3D6iWcYA==" dependencies: "@isaacs/import-jsx" "^4.0.1" "@types/react" "^17.0.52" @@ -4799,48 +4721,21 @@ tap@^16.3.7: treport "^3.0.4" which "^2.0.2" -tape@^5.6.5: - version "5.6.5" - resolved "https://registry.yarnpkg.com/tape/-/tape-5.6.5.tgz#a4dd5c6fb035fcee5b89a069cf8e98c6cbf40959" - integrity sha512-r6XcLeO3h5rOFpkYWifAjlhSSSXbFSSBF86lhb6J0KAQbY91H1MzOeIWG6TH0iWS52ypwr6fenJgCGQGtL8CxA== - dependencies: - array.prototype.every "^1.1.4" - call-bind "^1.0.2" - deep-equal "^2.2.2" - defined "^1.0.1" - dotignore "^0.1.2" - for-each "^0.3.3" - get-package-type "^0.1.0" - glob "^7.2.3" - has "^1.0.3" - has-dynamic-import "^2.0.1" - inherits "^2.0.4" - is-regex "^1.1.4" - minimist "^1.2.8" - object-inspect "^1.12.3" - object-is "^1.1.5" - object-keys "^1.1.1" - object.assign "^4.1.4" - resolve "^2.0.0-next.4" - resumer "^0.0.0" - string.prototype.trim "^1.2.7" - through "^2.3.8" - tarn@^3.0.2: version "3.0.2" - resolved "https://registry.yarnpkg.com/tarn/-/tarn-3.0.2.tgz#73b6140fbb881b71559c4f8bfde3d9a4b3d27693" + resolved "https://registry.npmjs.org/tarn/-/tarn-3.0.2.tgz" integrity sha512-51LAVKUSZSVfI05vjPESNc5vwqqZpbXCsU+/+wxlOrUjk2SnFTt97v9ZgQrD4YmxYW1Px6w2KjaDitCfkvgxMQ== tcompare@^5.0.6, tcompare@^5.0.7: version "5.0.7" - resolved "https://registry.yarnpkg.com/tcompare/-/tcompare-5.0.7.tgz#8c2d647208031ed5cac5e573428149e16f795bbf" + resolved "https://registry.npmjs.org/tcompare/-/tcompare-5.0.7.tgz" integrity sha512-d9iddt6YYGgyxJw5bjsN7UJUO1kGOtjSlNy/4PoGYAjQS5pAT/hzIoLf1bZCw+uUxRmZJh7Yy1aA7xKVRT9B4w== dependencies: diff "^4.0.2" test-exclude@^6.0.0: version "6.0.0" - resolved "https://registry.yarnpkg.com/test-exclude/-/test-exclude-6.0.0.tgz#04a8698661d805ea6fa293b6cb9e63ac044ef15e" + resolved "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz" integrity sha512-cAGWPIyOHU6zlmg88jwm7VRyXnMN7iV68OGAbYDk/Mh/xC/pzVPlQtY6ngoIH/5/tciuhGfvESU8GrHrcxD56w== dependencies: "@istanbuljs/schema" "^0.1.2" @@ -4849,44 +4744,54 @@ test-exclude@^6.0.0: text-table@^0.2.0: version "0.2.0" - resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4" + resolved "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz" integrity sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw== -through@^2.3.8, through@~2.3.4: - version "2.3.8" - resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" - integrity sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg== +tiktoken@^1.0.15: + version "1.0.15" + resolved "https://registry.yarnpkg.com/tiktoken/-/tiktoken-1.0.15.tgz#a1e11681fa51b50c81bb7eaaee53b7a66e844a23" + integrity sha512-sCsrq/vMWUSEW29CJLNmPvWxlVp7yh2tlkAjpJltIKqp5CKf98ZNpdeHRmAlPVFlGEbswDc6SmI8vz64W/qErw== tildify@2.0.0: version "2.0.0" - resolved "https://registry.yarnpkg.com/tildify/-/tildify-2.0.0.tgz#f205f3674d677ce698b7067a99e949ce03b4754a" + resolved "https://registry.npmjs.org/tildify/-/tildify-2.0.0.tgz" integrity sha512-Cc+OraorugtXNfs50hU9KS369rFXCfgGLpfCfvlc+Ud5u6VWmUQsOAa9HbTvheQdYnrdJqqv1e5oIqXppMYnSw== timestring@^6.0.0: version "6.0.0" - resolved "https://registry.yarnpkg.com/timestring/-/timestring-6.0.0.tgz#b0c7c331981ecf2066ce88bcfb8ee3ae32e7a0f6" + resolved "https://registry.npmjs.org/timestring/-/timestring-6.0.0.tgz" integrity sha512-wMctrWD2HZZLuIlchlkE2dfXJh7J2KDI9Dwl+2abPYg0mswQHfOAyQW3jJg1pY5VfttSINZuKcXoB3FGypVklA== +tlhunter-sorted-set@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/tlhunter-sorted-set/-/tlhunter-sorted-set-0.1.0.tgz#1c3eae28c0fa4dff97e9501d2e3c204b86406f4b" + integrity sha512-eGYW4bjf1DtrHzUYxYfAcSytpOkA44zsr7G2n3PV7yOUR23vmkGe3LL4R+1jL9OsXtbsFOwe8XtbCrabeaEFnw== + to-fast-properties@^2.0.0: version "2.0.0" - resolved "https://registry.yarnpkg.com/to-fast-properties/-/to-fast-properties-2.0.0.tgz#dc5e698cbd079265bc73e0377681a4e4e83f616e" + resolved "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz" integrity sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog== to-regex-range@^5.0.1: version "5.0.1" - resolved "https://registry.yarnpkg.com/to-regex-range/-/to-regex-range-5.0.1.tgz#1648c44aae7c8d988a326018ed72f5b4dd0392e4" + resolved "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz" integrity sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ== dependencies: is-number "^7.0.0" toidentifier@1.0.1: version "1.0.1" - resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.1.tgz#3be34321a88a820ed1bd80dfaa33e479fbb8dd35" + resolved "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz" integrity sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA== +tr46@~0.0.3: + version "0.0.3" + resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" + integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw== + treport@^3.0.4: version "3.0.4" - resolved "https://registry.yarnpkg.com/treport/-/treport-3.0.4.tgz#05247fa7820ad3afe92355e4cf08fe41a933084b" + resolved "https://registry.npmjs.org/treport/-/treport-3.0.4.tgz" integrity sha512-zUw1sfJypuoZi0I54woo6CNsfvMrv+OwLBD0/wc4LhMW8MA0MbSE+4fNObn22JSR8x9lOYccuAzfBfZ2IemzoQ== dependencies: "@isaacs/import-jsx" "^4.0.1" @@ -4900,113 +4805,128 @@ treport@^3.0.4: trivial-deferred@^1.0.1: version "1.0.1" - resolved "https://registry.yarnpkg.com/trivial-deferred/-/trivial-deferred-1.0.1.tgz#376d4d29d951d6368a6f7a0ae85c2f4d5e0658f3" - integrity sha512-dagAKX7vaesNNAwOc9Np9C2mJ+7YopF4lk+jE2JML9ta4kZ91Y6UruJNH65bLRYoUROD8EY+Pmi44qQWwXR7sw== + resolved "https://registry.npmjs.org/trivial-deferred/-/trivial-deferred-1.0.1.tgz" + integrity "sha1-N21NKdlR1jaKb3oK6FwvTV4GWPM= sha512-dagAKX7vaesNNAwOc9Np9C2mJ+7YopF4lk+jE2JML9ta4kZ91Y6UruJNH65bLRYoUROD8EY+Pmi44qQWwXR7sw==" -tsconfig-paths@^3.14.1: - version "3.14.1" - resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-3.14.1.tgz#ba0734599e8ea36c862798e920bcf163277b137a" - integrity sha512-fxDhWnFSLt3VuTwtvJt5fpwxBHg5AdKWMsgcPOOIilyjymcYVZoCQF8fvFRezCNfblEXmi+PcM1eYHeOAgXCOQ== +tsconfig-paths@^3.15.0: + version "3.15.0" + resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-3.15.0.tgz#5299ec605e55b1abb23ec939ef15edaf483070d4" + integrity sha512-2Ac2RgzDe/cn48GvOe3M+o82pEFewD3UPbyoUHHdKasHwJKjds4fLXWf/Ux5kATBKN20oaFGu+jbElp1pos0mg== dependencies: "@types/json5" "^0.0.29" - json5 "^1.0.1" + json5 "^1.0.2" minimist "^1.2.6" strip-bom "^3.0.0" +tslib@^2.4.0: + version "2.8.0" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.8.0.tgz#d124c86c3c05a40a91e6fdea4021bd31d377971b" + integrity sha512-jWVzBLplnCmoaTr13V9dYbiQ99wvZRd0vNWaDRg+aVYRcjDF3nDksxFDE/+fkXnKhpnUUkmx5pK/v8mCtLVqZA== + type-check@^0.4.0, type-check@~0.4.0: version "0.4.0" - resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.4.0.tgz#07b8203bfa7056c0657050e3ccd2c37730bab8f1" + resolved "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz" integrity sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew== dependencies: prelude-ls "^1.2.1" -type-detect@4.0.8, type-detect@^4.0.0, type-detect@^4.0.5, type-detect@^4.0.8: +type-detect@4.0.8, type-detect@^4.0.0, type-detect@^4.0.5: version "4.0.8" - resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-4.0.8.tgz#7646fb5f18871cfbb7749e69bd39a6388eb7450c" + resolved "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz" integrity sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g== +type-detect@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-4.1.0.tgz#deb2453e8f08dcae7ae98c626b13dddb0155906c" + integrity sha512-Acylog8/luQ8L7il+geoSxhEkazvkslg7PSNKOX59mbB9cOveP5aq9h74Y7YU8yDpJwetzQQrfIwtf4Wp4LKcw== + type-fest@^0.12.0: version "0.12.0" - resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.12.0.tgz#f57a27ab81c68d136a51fd71467eff94157fa1ee" + resolved "https://registry.npmjs.org/type-fest/-/type-fest-0.12.0.tgz" integrity sha512-53RyidyjvkGpnWPMF9bQgFtWp+Sl8O2Rp13VavmJgfAP9WWG6q6TkrKU8iyJdnwnfgHI6k2hTlgqH4aSdjoTbg== type-fest@^0.20.2: version "0.20.2" - resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.20.2.tgz#1bf207f4b28f91583666cb5fbd327887301cd5f4" + resolved "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz" integrity sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ== type-fest@^0.21.3: version "0.21.3" - resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.21.3.tgz#d260a24b0198436e133fa26a524a6d65fa3b2e37" + resolved "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz" integrity sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w== type-fest@^0.8.0: version "0.8.1" - resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.8.1.tgz#09e249ebde851d3b1e48d27c105444667f17b83d" + resolved "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz" integrity sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA== type-is@^1.6.4, type-is@~1.6.18: version "1.6.18" - resolved "https://registry.yarnpkg.com/type-is/-/type-is-1.6.18.tgz#4e552cd05df09467dcbc4ef739de89f2cf37c131" + resolved "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz" integrity sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g== dependencies: media-typer "0.3.0" mime-types "~2.1.24" -typed-array-buffer@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/typed-array-buffer/-/typed-array-buffer-1.0.0.tgz#18de3e7ed7974b0a729d3feecb94338d1472cd60" - integrity sha512-Y8KTSIglk9OZEr8zywiIHG/kmQ7KWyjseXs1CbSo8vC42w7hg2HgYTxSWwP0+is7bWDc1H+Fo026CpHFwm8tkw== +typed-array-buffer@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/typed-array-buffer/-/typed-array-buffer-1.0.2.tgz#1867c5d83b20fcb5ccf32649e5e2fc7424474ff3" + integrity sha512-gEymJYKZtKXzzBzM4jqa9w6Q1Jjm7x2d+sh19AdsD4wqnMPDYyvwpsIc2Q/835kHuo3BEQ7CjelGhfTsoBb2MQ== dependencies: - call-bind "^1.0.2" - get-intrinsic "^1.2.1" - is-typed-array "^1.1.10" + call-bind "^1.0.7" + es-errors "^1.3.0" + is-typed-array "^1.1.13" -typed-array-byte-length@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/typed-array-byte-length/-/typed-array-byte-length-1.0.0.tgz#d787a24a995711611fb2b87a4052799517b230d0" - integrity sha512-Or/+kvLxNpeQ9DtSydonMxCx+9ZXOswtwJn17SNLvhptaXYDJvkFFP5zbfU/uLmvnBJlI4yrnXRxpdWH/M5tNA== +typed-array-byte-length@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/typed-array-byte-length/-/typed-array-byte-length-1.0.1.tgz#d92972d3cff99a3fa2e765a28fcdc0f1d89dec67" + integrity sha512-3iMJ9q0ao7WE9tWcaYKIptkNBuOIcZCCT0d4MRvuuH88fEoEH62IuQe0OtraD3ebQEoTRk8XCBoknUNc1Y67pw== dependencies: - call-bind "^1.0.2" + call-bind "^1.0.7" for-each "^0.3.3" - has-proto "^1.0.1" - is-typed-array "^1.1.10" + gopd "^1.0.1" + has-proto "^1.0.3" + is-typed-array "^1.1.13" -typed-array-byte-offset@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/typed-array-byte-offset/-/typed-array-byte-offset-1.0.0.tgz#cbbe89b51fdef9cd6aaf07ad4707340abbc4ea0b" - integrity sha512-RD97prjEt9EL8YgAgpOkf3O4IF9lhJFr9g0htQkm0rchFp/Vx7LW5Q8fSXXub7BXAODyUQohRMyOc3faCPd0hg== +typed-array-byte-offset@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/typed-array-byte-offset/-/typed-array-byte-offset-1.0.2.tgz#f9ec1acb9259f395093e4567eb3c28a580d02063" + integrity sha512-Ous0vodHa56FviZucS2E63zkgtgrACj7omjwd/8lTEMEPFFyjfixMZ1ZXenpgCFBBt4EC1J2XsyVS2gkG0eTFA== dependencies: - available-typed-arrays "^1.0.5" - call-bind "^1.0.2" + available-typed-arrays "^1.0.7" + call-bind "^1.0.7" for-each "^0.3.3" - has-proto "^1.0.1" - is-typed-array "^1.1.10" + gopd "^1.0.1" + has-proto "^1.0.3" + is-typed-array "^1.1.13" -typed-array-length@^1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/typed-array-length/-/typed-array-length-1.0.4.tgz#89d83785e5c4098bec72e08b319651f0eac9c1bb" - integrity sha512-KjZypGq+I/H7HI5HlOoGHkWUUGq+Q0TPhQurLbyrVrvnKTBgzLhIJ7j6J/XTQOi0d1RjyZ0wdas8bKs2p0x3Ng== +typed-array-length@^1.0.6: + version "1.0.6" + resolved "https://registry.yarnpkg.com/typed-array-length/-/typed-array-length-1.0.6.tgz#57155207c76e64a3457482dfdc1c9d1d3c4c73a3" + integrity sha512-/OxDN6OtAk5KBpGb28T+HZc2M+ADtvRxXrKKbUwtsLgdoxgX13hyy7ek6bFRl5+aBs2yZzB0c4CnQfAtVypW/g== dependencies: - call-bind "^1.0.2" + call-bind "^1.0.7" for-each "^0.3.3" - is-typed-array "^1.1.9" + gopd "^1.0.1" + has-proto "^1.0.3" + is-typed-array "^1.1.13" + possible-typed-array-names "^1.0.0" typedarray-to-buffer@^3.1.5: version "3.1.5" - resolved "https://registry.yarnpkg.com/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz#a97ee7a9ff42691b9f783ff1bc5112fe3fca9080" + resolved "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz" integrity sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q== dependencies: is-typedarray "^1.0.0" typedarray@^0.0.6: version "0.0.6" - resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" + resolved "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz" integrity sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA== unbox-primitive@^1.0.2: version "1.0.2" - resolved "https://registry.yarnpkg.com/unbox-primitive/-/unbox-primitive-1.0.2.tgz#29032021057d5e6cdbd08c5129c226dff8ed6f9e" + resolved "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.2.tgz" integrity sha512-61pPlCD9h51VoreyJ0BReideM3MDKMKnh6+V9L08331ipq6Q8OFXZYiqP6n/tbHx4s5I9uRhcye6BrbkizkBDw== dependencies: call-bind "^1.0.2" @@ -5014,50 +4934,60 @@ unbox-primitive@^1.0.2: has-symbols "^1.0.3" which-boxed-primitive "^1.0.2" +undici-types@~5.26.4: + version "5.26.5" + resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617" + integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA== + +undici-types@~6.19.2: + version "6.19.8" + resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-6.19.8.tgz#35111c9d1437ab83a7cdc0abae2f26d88eda0a02" + integrity sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw== + unicode-length@^2.0.2: version "2.1.0" - resolved "https://registry.yarnpkg.com/unicode-length/-/unicode-length-2.1.0.tgz#425202b99f21854f5ca3530cc2a08dc262ce619f" + resolved "https://registry.npmjs.org/unicode-length/-/unicode-length-2.1.0.tgz" integrity sha512-4bV582zTV9Q02RXBxSUMiuN/KHo5w4aTojuKTNT96DIKps/SIawFp7cS5Mu25VuY1AioGXrmYyzKZUzh8OqoUw== dependencies: punycode "^2.0.0" unpipe@1.0.0, unpipe@~1.0.0: version "1.0.0" - resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" + resolved "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz" integrity sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ== -update-browserslist-db@^1.0.9: - version "1.0.9" - resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.0.9.tgz#2924d3927367a38d5c555413a7ce138fc95fcb18" - integrity sha512-/xsqn21EGVdXI3EXSum1Yckj3ZVZugqyOZQ/CxYPBD/R+ko9NSUScf8tFF4dOKY+2pvSSJA/S+5B8s4Zr4kyvg== +update-browserslist-db@^1.0.13: + version "1.0.13" + resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.0.13.tgz#3c5e4f5c083661bd38ef64b6328c26ed6c8248c4" + integrity "sha1-PF5PXAg2Yb0472S2Mowm7WyCSMQ= sha512-xebP81SNcPuNpPP3uzeW1NYXxI3rxyJzF3pD6sH4jE7o/IX+WtSpwnVU+qIsDPyk0d3hmFQ7mjqc6AtV604hbg==" dependencies: escalade "^3.1.1" picocolors "^1.0.0" uri-js@^4.2.2: version "4.4.1" - resolved "https://registry.yarnpkg.com/uri-js/-/uri-js-4.4.1.tgz#9b1a52595225859e55f669d928f88c6c57f2a77e" + resolved "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz" integrity sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg== dependencies: punycode "^2.1.0" url@0.10.3: version "0.10.3" - resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64" - integrity sha512-hzSUW2q06EqL1gKM/a+obYHLIO6ct2hwPuviqTTOcfFVc61UbfJ2Q32+uGL/HCPxKqrdGB5QUwIe7UqlDgwsOQ== + resolved "https://registry.npmjs.org/url/-/url-0.10.3.tgz" + integrity "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ= sha512-hzSUW2q06EqL1gKM/a+obYHLIO6ct2hwPuviqTTOcfFVc61UbfJ2Q32+uGL/HCPxKqrdGB5QUwIe7UqlDgwsOQ==" dependencies: punycode "1.3.2" querystring "0.2.0" util-deprecate@~1.0.1: version "1.0.2" - resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" + resolved "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz" integrity sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw== util@^0.12.4: version "0.12.5" - resolved "https://registry.yarnpkg.com/util/-/util-0.12.5.tgz#5f17a6059b73db61a875668781a1c2b136bd6fbc" - integrity sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA== + resolved "https://registry.npmjs.org/util/-/util-0.12.5.tgz" + integrity "sha1-XxemBZtz22GodWaHgaHCsTa9b7w= sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==" dependencies: inherits "^2.0.3" is-arguments "^1.0.4" @@ -5067,32 +4997,60 @@ util@^0.12.4: utils-merge@1.0.1: version "1.0.1" - resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713" + resolved "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz" integrity sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA== uuid-parse@^1.1.0: version "1.1.0" - resolved "https://registry.yarnpkg.com/uuid-parse/-/uuid-parse-1.1.0.tgz#7061c5a1384ae0e1f943c538094597e1b5f3a65b" + resolved "https://registry.npmjs.org/uuid-parse/-/uuid-parse-1.1.0.tgz" integrity sha512-OdmXxA8rDsQ7YpNVbKSJkNzTw2I+S5WsbMDnCtIWSQaosNAcWtFuI/YK1TjzUI6nbkgiqEyh8gWngfcv8Asd9A== uuid@8.0.0: version "8.0.0" - resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.0.0.tgz#bc6ccf91b5ff0ac07bbcdbf1c7c4e150db4dbb6c" - integrity sha512-jOXGuXZAWdsTH7eZLtyXMqUb9EcWMGZNbL9YcGBJl4MH4nrxHmZJhEHvyLFrkxo+28uLb/NYRcStH48fnD0Vzw== + resolved "https://registry.npmjs.org/uuid/-/uuid-8.0.0.tgz" + integrity "sha1-vGzPkbX/CsB7vNvxx8ThUNtNu2w= sha512-jOXGuXZAWdsTH7eZLtyXMqUb9EcWMGZNbL9YcGBJl4MH4nrxHmZJhEHvyLFrkxo+28uLb/NYRcStH48fnD0Vzw==" uuid@^8.3.2: version "8.3.2" - resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" + resolved "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz" integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== -vary@~1.1.2: +uuid@^9.0.0: + version "9.0.1" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.1.tgz#e188d4c8853cc722220392c424cd637f32293f30" + integrity sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA== + +value-or-promise@^1.0.12: + version "1.0.12" + resolved "https://registry.yarnpkg.com/value-or-promise/-/value-or-promise-1.0.12.tgz#0e5abfeec70148c78460a849f6b003ea7986f15c" + integrity sha512-Z6Uz+TYwEqE7ZN50gwn+1LCVo9ZVrpxRPOhOLnncYkY1ZzOYtrX8Fwf/rFktZ8R5mJms6EZf5TqNOMeZmnPq9Q== + +vary@^1, vary@~1.1.2: version "1.1.2" - resolved "https://registry.yarnpkg.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc" + resolved "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz" integrity sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg== +webidl-conversions@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" + integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ== + +whatwg-mimetype@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/whatwg-mimetype/-/whatwg-mimetype-3.0.0.tgz#5fa1a7623867ff1af6ca3dc72ad6b8a4208beba7" + integrity sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q== + +whatwg-url@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" + integrity sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw== + dependencies: + tr46 "~0.0.3" + webidl-conversions "^3.0.0" + which-boxed-primitive@^1.0.2: version "1.0.2" - resolved "https://registry.yarnpkg.com/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz#13757bc89b209b049fe5d86430e21cf40a89a8e6" + resolved "https://registry.npmjs.org/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz" integrity sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg== dependencies: is-bigint "^1.0.1" @@ -5101,83 +5059,54 @@ which-boxed-primitive@^1.0.2: is-string "^1.0.5" is-symbol "^1.0.3" -which-collection@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/which-collection/-/which-collection-1.0.1.tgz#70eab71ebbbd2aefaf32f917082fc62cdcb70906" - integrity sha512-W8xeTUwaln8i3K/cY1nGXzdnVZlidBcagyNFtBdD5kxnb4TvGKR7FfSIS3mYpwWS1QUCutfKz8IY8RjftB0+1A== - dependencies: - is-map "^2.0.1" - is-set "^2.0.1" - is-weakmap "^2.0.1" - is-weakset "^2.0.1" - which-module@^2.0.0: version "2.0.0" - resolved "https://registry.yarnpkg.com/which-module/-/which-module-2.0.0.tgz#d9ef07dce77b9902b8a3a8fa4b31c3e3f7e6e87a" + resolved "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz" integrity sha512-B+enWhmw6cjfVC7kS8Pj9pCrKSc5txArRyaYGe088shv/FGWH+0Rjx/xPgtsWfsUtS27FkP697E4DDhgrgoc0Q== -which-typed-array@^1.1.10, which-typed-array@^1.1.9: - version "1.1.10" - resolved "https://registry.yarnpkg.com/which-typed-array/-/which-typed-array-1.1.10.tgz#74baa2789991905c2076abb317103b866c64e69e" - integrity sha512-uxoA5vLUfRPdjCuJ1h5LlYdmTLbYfums398v3WLkM+i/Wltl2/XyZpQWKbN++ck5L64SR/grOHqtXCUKmlZPNA== - dependencies: - available-typed-arrays "^1.0.5" - call-bind "^1.0.2" - for-each "^0.3.3" - gopd "^1.0.1" - has-tostringtag "^1.0.0" - is-typed-array "^1.1.10" - -which-typed-array@^1.1.11, which-typed-array@^1.1.2: - version "1.1.13" - resolved "https://registry.yarnpkg.com/which-typed-array/-/which-typed-array-1.1.13.tgz#870cd5be06ddb616f504e7b039c4c24898184d36" - integrity sha512-P5Nra0qjSncduVPEAr7xhoF5guty49ArDTwzJ/yNuPIbZppyRxFQsRCWrocxIY+CnMVG+qfbU2FmDKyvSGClow== +which-typed-array@^1.1.14, which-typed-array@^1.1.15, which-typed-array@^1.1.2: + version "1.1.15" + resolved "https://registry.yarnpkg.com/which-typed-array/-/which-typed-array-1.1.15.tgz#264859e9b11a649b388bfaaf4f767df1f779b38d" + integrity sha512-oV0jmFtUky6CXfkqehVvBP/LSWJ2sy4vWMioiENyJLePrBO/yKyV9OyJySfAKosh+RYkIl5zJCNZ8/4JncrpdA== dependencies: - available-typed-arrays "^1.0.5" - call-bind "^1.0.4" + available-typed-arrays "^1.0.7" + call-bind "^1.0.7" for-each "^0.3.3" gopd "^1.0.1" - has-tostringtag "^1.0.0" + has-tostringtag "^1.0.2" which@2.0.2, which@^2.0.1, which@^2.0.2: version "2.0.2" - resolved "https://registry.yarnpkg.com/which/-/which-2.0.2.tgz#7c6a8dd0a636a0327e10b59c9286eee93f3f51b1" + resolved "https://registry.npmjs.org/which/-/which-2.0.2.tgz" integrity sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA== dependencies: isexe "^2.0.0" -wide-align@1.1.3: - version "1.1.3" - resolved "https://registry.yarnpkg.com/wide-align/-/wide-align-1.1.3.tgz#ae074e6bdc0c14a431e804e624549c633b000457" - integrity sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA== - dependencies: - string-width "^1.0.2 || 2" - widest-line@^3.1.0: version "3.1.0" - resolved "https://registry.yarnpkg.com/widest-line/-/widest-line-3.1.0.tgz#8292333bbf66cb45ff0de1603b136b7ae1496eca" + resolved "https://registry.npmjs.org/widest-line/-/widest-line-3.1.0.tgz" integrity sha512-NsmoXalsWVDMGupxZ5R08ka9flZjjiLvHVAWYOKtiKM8ujtZWr9cRffak+uSE48+Ob8ObalXpwyeUiyDD6QFgg== dependencies: string-width "^4.0.0" -word-wrap@^1.2.3: - version "1.2.3" - resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c" - integrity sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ== +word-wrap@^1.2.5: + version "1.2.5" + resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.5.tgz#d2c45c6dd4fbce621a66f136cbe328afd0410b34" + integrity sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA== wordwrap@~0.0.2: version "0.0.3" - resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-0.0.3.tgz#a3d5da6cd5c0bc0008d37234bbaf1bed63059107" + resolved "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz" integrity sha512-1tMA907+V4QmxV7dbRvb4/8MaRALK6q9Abid3ndMYnbyo8piisCmeONVqVSXqQA3KaP4SLt5b7ud6E2sqP8TFw== -workerpool@6.1.0: - version "6.1.0" - resolved "https://registry.yarnpkg.com/workerpool/-/workerpool-6.1.0.tgz#a8e038b4c94569596852de7a8ea4228eefdeb37b" - integrity sha512-toV7q9rWNYha963Pl/qyeZ6wG+3nnsyvolaNUS8+R5Wtw6qJPTxIlOP1ZSvcGhEJw+l3HMMmtiNo9Gl61G4GVg== +workerpool@6.2.0: + version "6.2.0" + resolved "https://registry.yarnpkg.com/workerpool/-/workerpool-6.2.0.tgz#827d93c9ba23ee2019c3ffaff5c27fccea289e8b" + integrity sha512-Rsk5qQHJ9eowMH28Jwhe8HEbmdYDX4lwoMWshiCXugjtHqMD9ZbiqSDLxcsfdqsETPzVUtX5s1Z5kStiIM6l4A== wrap-ansi@^6.2.0: version "6.2.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-6.2.0.tgz#e9393ba07102e6c91a3b221478f0257cd2856e53" + resolved "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz" integrity sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA== dependencies: ansi-styles "^4.0.0" @@ -5186,7 +5115,7 @@ wrap-ansi@^6.2.0: wrap-ansi@^7.0.0: version "7.0.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" + resolved "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== dependencies: ansi-styles "^4.0.0" @@ -5195,12 +5124,12 @@ wrap-ansi@^7.0.0: wrappy@1: version "1.0.2" - resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" + resolved "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz" integrity sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ== write-file-atomic@^3.0.0: version "3.0.3" - resolved "https://registry.yarnpkg.com/write-file-atomic/-/write-file-atomic-3.0.3.tgz#56bd5c5a5c70481cd19c571bd39ab965a5de56e8" + resolved "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-3.0.3.tgz" integrity sha512-AvHcyZ5JnSfq3ioSyjrBkH9yW4m7Ayk8/9My/DD9onKeu/94fwrMocemO2QAJFAlnnDN+ZDS+ZjAR5ua1/PV/Q== dependencies: imurmurhash "^0.1.4" @@ -5209,61 +5138,61 @@ write-file-atomic@^3.0.0: typedarray-to-buffer "^3.1.5" ws@^7, ws@^7.5.5: - version "7.5.9" - resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.9.tgz#54fa7db29f4c7cec68b1ddd3a89de099942bb591" - integrity sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q== + version "7.5.10" + resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.10.tgz#58b5c20dc281633f6c19113f39b349bd8bd558d9" + integrity sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ== xml2js@0.5.0: version "0.5.0" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.5.0.tgz#d9440631fbb2ed800203fad106f2724f62c493b7" - integrity sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA== + resolved "https://registry.npmjs.org/xml2js/-/xml2js-0.5.0.tgz" + integrity "sha1-2UQGMfuy7YACA/rRBvJyT2LEk7c= sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA==" dependencies: sax ">=0.6.0" xmlbuilder "~11.0.0" xmlbuilder@~11.0.0: version "11.0.1" - resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" - integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA== + resolved "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz" + integrity "sha1-vpuuHIoEbnazESdyY0fQrXACvrM= sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==" xtend@^4.0.0: version "4.0.2" - resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" + resolved "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz" integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ== y18n@^4.0.0: version "4.0.3" - resolved "https://registry.yarnpkg.com/y18n/-/y18n-4.0.3.tgz#b5f259c82cd6e336921efd7bfd8bf560de9eeedf" + resolved "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz" integrity sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ== y18n@^5.0.5: version "5.0.8" - resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55" + resolved "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz" integrity sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA== yallist@^3.0.2: version "3.1.1" - resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd" + resolved "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz" integrity sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g== yallist@^4.0.0: version "4.0.0" - resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" + resolved "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz" integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== yaml@^1.10.2: version "1.10.2" - resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.2.tgz#2301c5ffbf12b467de8da2333a459e29e7920e4b" + resolved "https://registry.npmjs.org/yaml/-/yaml-1.10.2.tgz" integrity sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg== yargs-parser@20.2.4: version "20.2.4" - resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.4.tgz#b42890f14566796f85ae8e3a25290d205f154a54" + resolved "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.4.tgz" integrity sha512-WOkpgNhPTlE73h4VFAFsOnomJVaovO8VqLDzy5saChRBFQFBoMYirowyW+Q9HB4HFF4Z7VZTiG3iSzJJA29yRA== yargs-parser@^18.1.2: version "18.1.3" - resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-18.1.3.tgz#be68c4975c6b2abf469236b0c870362fab09a7b0" + resolved "https://registry.npmjs.org/yargs-parser/-/yargs-parser-18.1.3.tgz" integrity sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ== dependencies: camelcase "^5.0.0" @@ -5271,12 +5200,12 @@ yargs-parser@^18.1.2: yargs-parser@^20.2.2: version "20.2.9" - resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.9.tgz#2eb7dc3b0289718fc295f362753845c41a0c94ee" + resolved "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz" integrity sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w== yargs-unparser@2.0.0: version "2.0.0" - resolved "https://registry.yarnpkg.com/yargs-unparser/-/yargs-unparser-2.0.0.tgz#f131f9226911ae5d9ad38c432fe809366c2325eb" + resolved "https://registry.npmjs.org/yargs-unparser/-/yargs-unparser-2.0.0.tgz" integrity sha512-7pRTIA9Qc1caZ0bZ6RYRGbHJthJWuakf+WmHK0rVeLkNrrGhfoabBNdue6kdINI6r4if7ocq9aD/n7xwKOdzOA== dependencies: camelcase "^6.0.0" @@ -5286,7 +5215,7 @@ yargs-unparser@2.0.0: yargs@16.2.0: version "16.2.0" - resolved "https://registry.yarnpkg.com/yargs/-/yargs-16.2.0.tgz#1c82bf0f6b6a66eafce7ef30e376f49a12477f66" + resolved "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz" integrity sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw== dependencies: cliui "^7.0.2" @@ -5299,7 +5228,7 @@ yargs@16.2.0: yargs@^15.0.2: version "15.4.1" - resolved "https://registry.yarnpkg.com/yargs/-/yargs-15.4.1.tgz#0d87a16de01aee9d8bec2bfbf74f67851730f4f8" + resolved "https://registry.npmjs.org/yargs/-/yargs-15.4.1.tgz" integrity sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A== dependencies: cliui "^6.0.0" @@ -5316,12 +5245,12 @@ yargs@^15.0.2: yocto-queue@^0.1.0: version "0.1.0" - resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b" + resolved "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz" integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q== yoga-layout-prebuilt@^1.9.6: version "1.10.0" - resolved "https://registry.yarnpkg.com/yoga-layout-prebuilt/-/yoga-layout-prebuilt-1.10.0.tgz#2936fbaf4b3628ee0b3e3b1df44936d6c146faa6" + resolved "https://registry.npmjs.org/yoga-layout-prebuilt/-/yoga-layout-prebuilt-1.10.0.tgz" integrity sha512-YnOmtSbv4MTf7RGJMK0FvZ+KD8OEe/J5BNnR0GHhD8J/XcG/Qvxgszm0Un6FTHWW4uHlTgP0IztiXQnGyIR45g== dependencies: "@types/yoga-layout" "1.9.2"