v4.49.0 proposal

.github/workflows/appsec.yml

@@ -126,7 +126,7 @@ jobs:
   express:
     runs-on: ubuntu-latest
     env:
-      PLUGINS: express|body-parser|cookie-parser
+      PLUGINS: express|body-parser|cookie-parser|multer
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/node/setup

.github/workflows/llmobs.yml

@@ -0,0 +1,49 @@
+name: LLMObs
+
+on:
+  pull_request:
+  push:
+    branches: [master]
+  schedule:
+    - cron: '0 4 * * *'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  sdk:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/testagent/start
+      - uses: ./.github/actions/node/setup
+      - uses: ./.github/actions/install
+      - uses: ./.github/actions/node/18
+      - run: yarn test:llmobs:sdk:ci
+      - uses: ./.github/actions/node/20
+      - run: yarn test:llmobs:sdk:ci
+      - uses: ./.github/actions/node/latest
+      - run: yarn test:llmobs:sdk:ci
+      - if: always()
+        uses: ./.github/actions/testagent/logs
+      - uses: codecov/codecov-action@v3
+
+  openai:
+    runs-on: ubuntu-latest
+    env:
+      PLUGINS: openai
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/testagent/start
+      - uses: ./.github/actions/node/setup
+      - uses: ./.github/actions/install
+      - uses: ./.github/actions/node/oldest
+      - run: yarn test:llmobs:plugins:ci
+        shell: bash
+      - uses: ./.github/actions/node/latest
+      - run: yarn test:llmobs:plugins:ci
+        shell: bash
+      - uses: codecov/codecov-action@v3
+      - if: always()
+        uses: ./.github/actions/testagent/logs

.github/workflows/plugins.yml

@@ -212,6 +212,14 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/plugins/test
 
+  body-parser:
+    runs-on: ubuntu-latest
+    env:
+      PLUGINS: body-parser
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/plugins/test
+
   bunyan:
     runs-on: ubuntu-latest
     env:
@@ -250,6 +258,14 @@ jobs:
       - run: yarn test:plugins:ci
       - uses: codecov/codecov-action@v2
 
+  cookie-parser:
+    runs-on: ubuntu-latest
+    env:
+      PLUGINS: cookie-parser
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/plugins/test
+
   couchbase:
     strategy:
       matrix:
@@ -359,7 +375,22 @@ jobs:
   express:
     runs-on: ubuntu-latest
     env:
-      PLUGINS: express|body-parser|cookie-parser
+      PLUGINS: express
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/plugins/test
+
+  express-mongo-sanitize:
+    runs-on: ubuntu-latest
+    services:
+      mongodb:
+        image: circleci/mongo
+        ports:
+          - 27017:27017
+    env:
+      PLUGINS: express-mongo-sanitize
+      PACKAGE_NAMES: express-mongo-sanitize
+      SERVICES: mongo
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/plugins/test
@@ -542,6 +573,23 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/plugins/test
+
+  mariadb:
+    runs-on: ubuntu-latest
+    services:
+      mysql:
+        image: mariadb:10.4
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: 'yes'
+          MYSQL_DATABASE: 'db'
+        ports:
+          - 3306:3306
+    env:
+      PLUGINS: mariadb
+      SERVICES: mariadb
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/plugins/test
 
   memcached:
     runs-on: ubuntu-latest
@@ -636,12 +684,29 @@ jobs:
         ports:
           - 3306:3306
     env:
-      PLUGINS: mysql|mysql2|mariadb # TODO: move mysql2 to its own job
+      PLUGINS: mysql
       SERVICES: mysql
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/plugins/test
 
+  mysql2:
+    runs-on: ubuntu-latest
+    services:
+      mysql:
+        image: mariadb:10.4
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: 'yes'
+          MYSQL_DATABASE: 'db'
+        ports:
+          - 3306:3306
+    env:
+      PLUGINS: mysql2
+      SERVICES: mysql2
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/plugins/test
+
   net:
     runs-on: ubuntu-latest
     env:

.github/workflows/system-tests.yml

@@ -31,7 +31,6 @@ jobs:
     uses: DataDog/system-tests/.github/workflows/compute-workflow-parameters.yml@main
     with:
       library: nodejs
-      scenarios: CROSSED_TRACING_LIBRARIES
       scenarios_groups: essentials
 
   system-tests:

.gitlab-ci.yml

@@ -24,3 +24,10 @@ onboarding_tests_installer:
 onboarding_tests_k8s_injection:
   variables:
     WEBLOG_VARIANT: sample-app
+
+requirements_json_test:
+  rules:
+    - when: on_success
+  variables:
+    REQUIREMENTS_BLOCK_JSON_PATH: ".gitlab/requirements_block.json"
+    REQUIREMENTS_ALLOW_JSON_PATH: ".gitlab/requirements_allow.json"

.gitlab/prepare-oci-package.sh

@@ -21,3 +21,5 @@ fi
 echo -n $JS_PACKAGE_VERSION > packaging/sources/version
 
 cd packaging
+
+cp ../requirements.json sources/requirements.json

.gitlab/requirements_allow.json

@@ -0,0 +1,19 @@
+[
+  {"name": "min glibc x64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:2.17"}},
+  {"name": "ok glibc x64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:2.23"}},
+  {"name": "high glibc x64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:3.0"}},
+  {"name": "musl x64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "musl:1.2.2"}},
+  {"name": "min glibc arm64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "arm64", "libc": "glibc:2.17"}},
+  {"name": "ok glibc arm64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "arm64", "libc": "glibc:2.27"}},
+  {"name": "glibc x86","filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x86", "libc": "glibc:2.19"}},
+  {"name": "musl arm","filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "arm", "libc": "musl:1.2.2"}},
+  {"name": "musl arm64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "arm64", "libc": "musl:1.2.2"}},
+  {"name": "musl x64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "musl:1.2.2"}},
+  {"name": "musl x86", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x86", "libc": "musl:1.2.2"}},
+  {"name": "windows x64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "windows", "arch": "x64"}},
+  {"name": "windows x86", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "windows", "arch": "x86"}},
+  {"name": "macos x64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "darwin", "arch": "x64"}},
+  {"name": "macos arm64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "darwin", "arch": "arm64"}},
+  {"name": "node app", "filepath": "/pathto/node", "args": ["/pathto/node", "./app.js"], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:2.40"}},
+  {"name": "ts-node app", "filepath": "/pathto/ts-node", "args": ["/pathto/ts-node", "./app.js"], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:2.40"}}
+]

.gitlab/requirements_block.json

@@ -0,0 +1,11 @@
+[
+  {"name": "unsupported 2.x glibc x64","filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:2.16"}},
+  {"name": "unsupported 1.x glibc x64","filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:1.22"}},
+  {"name": "unsupported 2.x.x glibc x64","filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:2.16.9"}},
+  {"name": "unsupported 2.x glibc arm64","filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "arm64", "libc": "glibc:2.16"}},
+  {"name": "unsupported 2.x.x glibc x64","filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "arm64", "libc": "glibc:2.16.9"}},
+  {"name": "unsupported 2.x.x glibc x86","filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x86", "libc": "glibc:2.17"}},
+  {"name": "npm","filepath": "/pathto/node", "args": ["/pathto/node", "/pathto/npm-cli.js"], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:2.40"}},
+  {"name": "yarn","filepath": "/pathto/node", "args": ["/pathto/node", "/pathto/yarn.js"], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:2.40"}},
+  {"name": "pnpm","filepath": "/pathto/node", "args": ["/pathto/node", "/pathto/pnpm.cjs"], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:2.40"}}
+]

LICENSE-3rdparty.csv

@@ -29,6 +29,7 @@ require,retry,MIT,Copyright 2011 Tim Koschützki Felix Geisendörfer
 require,rfdc,MIT,Copyright 2019 David Mark Clements
 require,semver,ISC,Copyright Isaac Z. Schlueter and Contributors
 require,shell-quote,mit,Copyright (c) 2013 James Halliday
+dev,@apollo/server,MIT,Copyright (c) 2016-2020 Apollo Graph, Inc. (Formerly Meteor Development Group, Inc.)
 dev,@types/node,MIT,Copyright Authors
 dev,autocannon,MIT,Copyright 2016 Matteo Collina
 dev,aws-sdk,Apache 2.0,Copyright 2012-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.

docker-compose.yml

@@ -129,7 +129,7 @@ services:
       - KAFKA_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093
       - KAFKA_CONTROLLER_QUORUM_VOTERS=1@127.0.0.1:9093
       - KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER
-      - KAFKA_CLUSTER_ID=r4zt_wrqTRuT7W2NJsB_GA
+      - CLUSTER_ID=5L6g3nShT-eMCtK--X86sw
       - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://127.0.0.1:9092
       - KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT
       - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT

docs/test.ts

@@ -536,3 +536,80 @@ const otelTraceId: string = spanContext.traceId
 const otelSpanId: string = spanContext.spanId
 const otelTraceFlags: number = spanContext.traceFlags
 const otelTraceState: opentelemetry.TraceState = spanContext.traceState!
+
+// -- LLM Observability --
+const llmobsEnableOptions = {
+  mlApp: 'mlApp',
+  agentlessEnabled: true
+}
+tracer.init({
+  llmobs: llmobsEnableOptions,
+})
+const llmobs = tracer.llmobs
+const enabled = llmobs.enabled
+
+// manually enable
+llmobs.enable({
+  mlApp: 'mlApp',
+  agentlessEnabled: true
+})
+
+// manually disable
+llmobs.disable()
+
+// trace block of code
+llmobs.trace({ name: 'name', kind: 'llm' }, () => {})
+llmobs.trace({ kind: 'llm', name: 'myLLM', modelName: 'myModel', modelProvider: 'myProvider' }, () => {})
+llmobs.trace({ name: 'name', kind: 'llm' }, (span, cb) => {
+  llmobs.annotate(span, {})
+  span.setTag('foo', 'bar')
+  cb(new Error('boom'))
+})
+
+// wrap a function
+llmobs.wrap({ kind: 'llm' }, function myLLM () {})()
+llmobs.wrap({ kind: 'llm', name: 'myLLM', modelName: 'myModel', modelProvider: 'myProvider' }, function myFunction () {})()
+
+// export a span
+llmobs.enable({ mlApp: 'myApp' })
+llmobs.trace({ kind: 'llm', name: 'myLLM' }, (span) => {
+  const llmobsSpanCtx = llmobs.exportSpan(span)
+  llmobsSpanCtx.traceId;
+  llmobsSpanCtx.spanId;
+
+  // submit evaluation
+  llmobs.disable()
+  llmobs.submitEvaluation(llmobsSpanCtx, {
+    label: 'my-eval-metric',
+    metricType: 'categorical',
+    value: 'good',
+    mlApp: 'myApp',
+    tags: {},
+    timestampMs: Date.now()
+  })
+})
+
+// annotate a span
+llmobs.annotate({
+  inputData: 'input',
+  outputData: 'output',
+  metadata: {},
+  metrics: {
+    inputTokens: 10,
+    outputTokens: 5,
+    totalTokens: 15
+  },
+  tags: {}
+})
+llmobs.annotate(span, {
+  inputData: 'input',
+  outputData: 'output',
+  metadata: {},
+  metrics: {},
+  tags: {}
+})
+
+
+
+// flush
+llmobs.flush()