v3.24.0 proposal

.github/workflows/appsec.yml

@@ -136,3 +136,18 @@ jobs:
       - uses: ./.github/actions/node/latest
       - run: yarn test:appsec:plugins:ci
       - uses: codecov/codecov-action@v2
+
+  sourcing:
+    runs-on: ubuntu-latest
+    env:
+      PLUGINS: cookie
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ./.github/actions/node/setup
+      - run: yarn install
+      - uses: ./.github/actions/node/16
+      - run: yarn test:appsec:plugins:ci
+      - uses: ./.github/actions/node/18
+      - run: yarn test:appsec:plugins:ci
+      - uses: ./.github/actions/node/latest
+      - run: yarn test:appsec:plugins:ci

.github/workflows/plugins.yml

@@ -332,6 +332,23 @@ jobs:
         uses: ./.github/actions/testagent/logs
       - uses: codecov/codecov-action@v2
 
+  fetch:
+    runs-on: ubuntu-latest
+    env:
+      PLUGINS: fetch
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ./.github/actions/testagent/start
+      - uses: ./.github/actions/node/setup
+      - run: yarn install
+      - uses: ./.github/actions/node/oldest
+      - run: yarn test:plugins:ci
+      - uses: ./.github/actions/node/latest
+      - run: yarn test:plugins:ci
+      - if: always()
+        uses: ./.github/actions/testagent/logs
+      - uses: codecov/codecov-action@v2
+
   generic-pool:
     runs-on: ubuntu-latest
     env:
@@ -823,7 +840,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - uses: ./.github/actions/node/setup
-      - run: yarn install --ignore-engine
+      - run: yarn install --ignore-engines
       - run: yarn services
       - run: yarn test:plugins
       - uses: codecov/codecov-action@v2

.github/workflows/project.yml

@@ -5,7 +5,7 @@ on:
   push:
     branches: [master]
   schedule:
-    - cron: '0 4 * * *'
+    - cron: "0 4 * * *"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
@@ -22,20 +22,34 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: ./.github/actions/node/setup
-      - run: yarn install
       - uses: actions/setup-node@v3
         with:
           node-version: ${{ matrix.version }}
       # Disable core dumps since some integration tests intentionally abort and core dump generation takes around 5-10s
+      - run: yarn install
       - run: sudo sysctl -w kernel.core_pattern='|/bin/false'
       - run: yarn test:integration
 
   integration-ci:
     strategy:
       matrix:
         version: [16, latest]
-        framework: [cucumber, cypress, playwright]
+        framework: [cucumber, playwright]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.version }}
+      - run: yarn install
+      - run: yarn test:integration:${{ matrix.framework }}
+
+  integration-cypress:
+    strategy:
+      matrix:
+        version: [16, latest]
+        # 6.7.0 is the minimum version we support
+        cypress-version: [6.7.0, latest]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -44,7 +58,9 @@ jobs:
       - uses: actions/setup-node@v3
         with:
           node-version: ${{ matrix.version }}
-      - run: yarn test:integration:${{ matrix.framework }}
+      - run: yarn test:integration:cypress
+        env:
+          CYPRESS_VERSION: ${{ matrix.cypress-version }}
 
   lint:
     runs-on: ubuntu-latest

.github/workflows/release-dev.yml

@@ -23,3 +23,26 @@ jobs:
           echo "::set-output name=json::$content"
       - run: npm version --no-git-tag-version ${{ fromJson(steps.pkg.outputs.json).version }}-$(git rev-parse --short HEAD)+${{ github.run_id }}.${{ github.run_attempt }}
       - run: npm publish --tag dev
+      - run: |
+          git tag --force dev
+          git push origin :refs/tags/dev
+          git push origin --tags
+
+  injection-image-publish:
+    runs-on: ubuntu-latest
+    needs: ['publish']
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+      - name: Log in to the Container registry
+        uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: npm pack for injection image
+        run: |
+          npm pack dd-trace@dev
+      - uses: ./.github/actions/injection
+        with:
+          init-image-version: dev

.gitlab-ci.yml

@@ -48,7 +48,7 @@ deploy_to_reliability_env:
 deploy_to_docker_registries:
   stage: deploy
   rules:
-    - if: '$CI_COMMIT_TAG =~ /^v.*/'
+    - if: '$CI_COMMIT_TAG =~ /^v.*/ || $CI_COMMIT_TAG == "dev"'
       when: on_success
     - when: manual
       allow_failure: true
@@ -78,3 +78,5 @@ deploy_latest_to_docker_registries:
     IMG_SOURCES: ghcr.io/datadog/dd-trace-js/dd-lib-js-init:$CI_COMMIT_TAG
     IMG_DESTINATIONS: dd-lib-js-init:latest
     IMG_SIGNING: "false"
+    RETRY_COUNT: 5
+    RETRY_DELAY: 300

MIGRATING.md

@@ -4,6 +4,45 @@ This guide describes the steps to upgrade dd-trace from a major version to the
 next. If you are having any issues related to migrating, please feel free to
 open an issue or contact our [support](https://www.datadoghq.com/support/) team.
 
+## 3.0 to 4.0
+
+### Node 14 is no longer supported
+
+Node.js 14 has reached EOL in April 2023 and is no longer supported. Generally
+speaking, we highly recommend always keeping Node.js up to date regardless of
+our support policy.
+
+### The `orphanable` option was removed
+
+This option was only useful internally for a single integration that has since
+been removed. It was never useful for manual instrumentation since all that is
+needed to orphan a span on creation is to use
+`tracer.trace('web.request', { childOf: null })`.
+
+### Support for `jest-jasmine2` has been removed
+
+The default test runner for Jest was changed to `jest-circus` around 2 years ago and
+is no longer supported by our Jest integration for CI Visibility. We recommend
+switching to `jest-circus` to anyone still using `jest-jasmine2`.
+
+### Support for older Next.js versions was removed
+
+We now support only Next.js 10.2 and up.
+
+### W3C headers are now prioritized over Datadog headers
+
+As we move towards open standards, we have decided to prioritize W3C Trace
+Context headers over our own vendor-specific headers for context propagation
+across services. For most applications this shouldn't change anything and
+distributed tracing should continue to work seamlessly.
+
+In some rare cases it's possible that some of the services involved in a trace
+are not instrumented by Datadog at all which can cause spans within the trace to
+become disconnected. While the data would still be available in the UI, the
+relationship between spans would no longer be visible. This can be addressed by
+restoring the previous behaviour using
+`DD_TRACE_PROPAGATION_STYLE='datadog,tracecontext'`.
+
 ## 2.0 to 3.0
 
 ### Node 12 is no longer supported

README.md

@@ -1,6 +1,7 @@
 # `dd-trace`: Node.js APM Tracer Library
 
-[![npm v3](https://img.shields.io/npm/v/dd-trace/latest?color=blue&label=dd-trace%40v3&logo=npm)](https://www.npmjs.com/package/dd-trace)
+[![npm v4](https://img.shields.io/npm/v/dd-trace/latest?color=blue&label=dd-trace%40v4&logo=npm)](https://www.npmjs.com/package/dd-trace)
+[![npm v3](https://img.shields.io/npm/v/dd-trace/latest-node14?color=blue&label=dd-trace%40v3&logo=npm)](https://www.npmjs.com/package/dd-trace/v/latest-node12)
 [![npm v2](https://img.shields.io/npm/v/dd-trace/latest-node12?color=blue&label=dd-trace%40v2&logo=npm)](https://www.npmjs.com/package/dd-trace/v/latest-node12)
 [![npm dev](https://img.shields.io/npm/v/dd-trace/dev?color=orange&label=dd-trace%40dev&logo=npm)](https://www.npmjs.com/package/dd-trace/v/dev)
 [![codecov](https://codecov.io/gh/DataDog/dd-trace-js/branch/master/graph/badge.svg)](https://codecov.io/gh/DataDog/dd-trace-js)
@@ -28,27 +29,28 @@ Most of the documentation for `dd-trace` is available on these webpages:
 | :---:                                                    | :---:                                                                                                  | :---:    | :---:           | :---:          | :---:       |
 | [`v1`](https://github.com/DataDog/dd-trace-js/tree/v1.x) | ![npm v1](https://img.shields.io/npm/v/dd-trace/legacy-v1?color=white&label=%20&style=flat-square)     | `>= v12` | **End of Life** | 2021-07-13     | 2022-02-25  |
 | [`v2`](https://github.com/DataDog/dd-trace-js/tree/v2.x) | ![npm v2](https://img.shields.io/npm/v/dd-trace/latest-node12?color=white&label=%20&style=flat-square) | `>= v12` | **Maintenance** | 2022-01-28     | 2023-08-15  |
-| [`v3`](https://github.com/DataDog/dd-trace-js/tree/v3.x) | ![npm v3](https://img.shields.io/npm/v/dd-trace/latest?color=white&label=%20&style=flat-square)        | `>= v14` | **Current**     | 2022-08-15     | Unknown     |
+| [`v3`](https://github.com/DataDog/dd-trace-js/tree/v3.x) | ![npm v3](https://img.shields.io/npm/v/dd-trace/latest-node14?color=white&label=%20&style=flat-square) | `>= v14` | **Maintenance** | 2022-08-15     | 2024-05-15  |
+| [`v4`](https://github.com/DataDog/dd-trace-js/tree/v4.x) | ![npm v4](https://img.shields.io/npm/v/dd-trace/latest?color=white&label=%20&style=flat-square)        | `>= v16` | **Current**     | 2023-05-12     | Unknown     |
 
-We currently maintain two release lines, namely `v2` and `v3`.
-Features and bug fixes that are merged are released to the `v3` line and, if appropriate, also the `v2` line.
+We currently maintain three release lines, namely `v2`, `v3` and `v4`.
+Features and bug fixes that are merged are released to the `v4` line and, if appropriate, also the `v2` and `v3` line.
 
-For any new projects it is recommended to use the `v3` release line:
+For any new projects it is recommended to use the `v4` release line:
 
 ```sh
 $ npm install dd-trace
 $ yarn add dd-trace
 ```
 
-However, existing projects that already use the `v2` release line, or projects that need to support Node.js v12, may use the `v2` release line.
+However, existing projects that already use the `v2` or `v3` release lines, or projects that need to support EOL versions of Node.js, may continue to use these release lines.
 This is done by specifying the version when installing the package.
-Note that we also publish to npm using a `latest-node12` tag that can also be used for install:
+Note that we also publish to npm using a `latest-node12` and `latest-node14` tag that can also be used for install:
 
 ```sh
-$ npm install dd-trace@2
-$ yarn add dd-trace@2
-$ npm install dd-trace@latest-node12
-$ yarn add dd-trace@latest-node12
+$ npm install dd-trace@3
+$ yarn add dd-trace@3
+$ npm install dd-trace@latest-node14
+$ yarn add dd-trace@latest-node14
 ```
 
 Any backwards-breaking functionality that is introduced into the library will result in an increase of the major version of the library and therefore a new release line.
@@ -153,6 +155,11 @@ $ yarn lint
 
 ### Experimental ESM Support
 
+> **Warning**
+> 
+> ESM support has been temporarily disabled starting from Node 20 as significant
+> changes are in progress.
+
 ESM support is currently in the experimental stages, while CJS has been supported
 since inception. This means that code loaded using `require()` should work fine
 but code loaded using `import` might not always work.

benchmark/sirun/appsec-iast/README.md

@@ -0,0 +1,9 @@
+This creates 150 HTTP requests from client to server.
+
+The variants are:
+- control tracer with non vulnerable endpoint without iast 
+- tracer with non vulnerable endpoint with iast active and default configuration
+- tracer with non vulnerable endpoint with iast active and sampling 100
+- control tracer with vulnerable endpoint without iast
+- tracer with vulnerable endpoint with iast active and default configuration
+- tracer with vulnerable endpoint with iast active and sampling 100

benchmark/sirun/appsec-iast/client.js

@@ -0,0 +1,30 @@
+'use strict'
+
+const { port, reqs } = require('./common')
+const http = require('http')
+
+let connectionsMade = 0
+function request (opts) {
+  http.get(opts, (res) => {
+    res.on('data', () => {})
+    res.on('end', () => {
+      if (++connectionsMade !== reqs) {
+        request(opts)
+      }
+    })
+  }).on('error', (e) => {
+    setTimeout(() => {
+      request(opts)
+    }, 10)
+  })
+}
+
+const path = '/?param=value'
+const opts = {
+  headers: {
+    accept: 'text/html'
+  },
+  port,
+  path
+}
+request(opts)

benchmark/sirun/appsec-iast/common.js

@@ -0,0 +1,6 @@
+'use strict'
+
+module.exports = {
+  port: 3331 + parseInt(process.env.CPU_AFFINITY || '0'),
+  reqs: 200
+}

benchmark/sirun/appsec-iast/meta.json

@@ -0,0 +1,66 @@
+{
+  "name": "appsec-iast",
+  "cachegrind": false,
+  "instructions": true,
+  "iterations": 30,
+  "variants": {
+    "no-vulnerability-control": {
+      "setup": "bash -c \"nohup node client.js >/dev/null 2>&1 &\"",
+      "run": "node --require ../../../init.js server-without-vulnerability.js",
+      "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node --require ../../../init.js server-without-vulnerability.js\"",
+      "env": {
+        "DD_IAST_ENABLED": "0"
+      }
+    },
+    "no-vulnerability-iast-enabled-default-config": {
+      "setup": "bash -c \"nohup node client.js >/dev/null 2>&1 &\"",
+      "run": "node --require ../../../init.js server-without-vulnerability.js",
+      "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node --require ../../../init.js server-without-vulnerability.js\"",
+      "baseline": "no-vulnerability-control",
+      "env": {
+        "DD_IAST_ENABLED": "1"
+      }
+    },
+    "no-vulnerability-iast-enabled-always-active": {
+      "setup": "bash -c \"nohup node client.js >/dev/null 2>&1 &\"",
+      "run": "node --require ../../../init.js server-without-vulnerability.js",
+      "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node --require ../../../init.js server-without-vulnerability.js\"",
+      "baseline": "no-vulnerability-control",
+      "env": {
+        "DD_IAST_ENABLED": "1",
+        "DD_IAST_REQUEST_SAMPLING": "100",
+        "DD_IAST_MAX_CONCURRENT_REQUESTS": "1000",
+        "DD_IAST_MAX_CONTEXT_OPERATIONS": "100"
+      }
+    },
+    "with-vulnerability-control": {
+      "setup": "bash -c \"nohup node client.js >/dev/null 2>&1 &\"",
+      "run": "node --require ../../../init.js server-with-vulnerability.js",
+      "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node --require ../../../init.js server-with-vulnerability.js\"",
+      "env": {
+        "DD_IAST_ENABLED": "0"
+      }
+    },
+    "with-vulnerability-iast-enabled-default-config": {
+      "setup": "bash -c \"nohup node client.js >/dev/null 2>&1 &\"",
+      "run": "node --require ../../../init.js server-with-vulnerability.js",
+      "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node --require ../../../init.js server-with-vulnerability.js\"",
+      "baseline": "with-vulnerability-control",
+      "env": {
+        "DD_IAST_ENABLED": "1"
+      }
+    },
+    "with-vulnerability-iast-enabled-always-active": {
+      "setup": "bash -c \"nohup node client.js >/dev/null 2>&1 &\"",
+      "run": "node --require ../../../init.js server-with-vulnerability.js",
+      "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node --require ../../../init.js server-with-vulnerability.js\"",
+      "baseline": "with-vulnerability-control",
+      "env": {
+        "DD_IAST_ENABLED": "1",
+        "DD_IAST_REQUEST_SAMPLING": "100",
+        "DD_IAST_MAX_CONCURRENT_REQUESTS": "1000",
+        "DD_IAST_MAX_CONTEXT_OPERATIONS": "100"
+      }
+    }
+  }
+}

benchmark/sirun/appsec-iast/server-with-vulnerability.js

@@ -0,0 +1,25 @@
+'use strict'
+
+const { port, reqs } = require('./common')
+const express = require('../../../versions/express').get()
+const cookieParser = require('../../../versions/cookie-parser').get()
+const childProcess = require('child_process')
+
+const app = express()
+app.use(cookieParser())
+
+let connectionsMade = 0
+
+function noop () {}
+
+app.get('/', (req, res) => {
+  childProcess.exec('echo #' + req.query.param, noop)
+  res.writeHead(200)
+  res.end('Hello, World!')
+
+  if (++connectionsMade === reqs) {
+    server.close()
+  }
+})
+
+const server = app.listen(port)