diff --git a/.github/workflows/appsec.yml b/.github/workflows/appsec.yml index 9679f7e3b24..5c3ad393c87 100644 --- a/.github/workflows/appsec.yml +++ b/.github/workflows/appsec.yml @@ -126,7 +126,7 @@ jobs: express: runs-on: ubuntu-latest env: - PLUGINS: express|body-parser|cookie-parser + PLUGINS: express|body-parser|cookie-parser|multer steps: - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup diff --git a/.github/workflows/llmobs.yml b/.github/workflows/llmobs.yml new file mode 100644 index 00000000000..a1e3502a8a0 --- /dev/null +++ b/.github/workflows/llmobs.yml @@ -0,0 +1,49 @@ +name: LLMObs + +on: + pull_request: + push: + branches: [master] + schedule: + - cron: '0 4 * * *' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref || github.run_id }} + cancel-in-progress: true + +jobs: + sdk: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/testagent/start + - uses: ./.github/actions/node/setup + - uses: ./.github/actions/install + - uses: ./.github/actions/node/18 + - run: yarn test:llmobs:sdk:ci + - uses: ./.github/actions/node/20 + - run: yarn test:llmobs:sdk:ci + - uses: ./.github/actions/node/latest + - run: yarn test:llmobs:sdk:ci + - if: always() + uses: ./.github/actions/testagent/logs + - uses: codecov/codecov-action@v3 + + openai: + runs-on: ubuntu-latest + env: + PLUGINS: openai + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/testagent/start + - uses: ./.github/actions/node/setup + - uses: ./.github/actions/install + - uses: ./.github/actions/node/oldest + - run: yarn test:llmobs:plugins:ci + shell: bash + - uses: ./.github/actions/node/latest + - run: yarn test:llmobs:plugins:ci + shell: bash + - uses: codecov/codecov-action@v3 + - if: always() + uses: ./.github/actions/testagent/logs diff --git a/.github/workflows/plugins.yml b/.github/workflows/plugins.yml index 48e33395efe..b2c177b2a14 100644 --- a/.github/workflows/plugins.yml +++ b/.github/workflows/plugins.yml @@ -212,6 +212,14 @@ jobs: - uses: actions/checkout@v4 - uses: ./.github/actions/plugins/test + body-parser: + runs-on: ubuntu-latest + env: + PLUGINS: body-parser + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test + bunyan: runs-on: ubuntu-latest env: @@ -250,6 +258,14 @@ jobs: - run: yarn test:plugins:ci - uses: codecov/codecov-action@v2 + cookie-parser: + runs-on: ubuntu-latest + env: + PLUGINS: cookie-parser + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test + couchbase: strategy: matrix: @@ -359,7 +375,22 @@ jobs: express: runs-on: ubuntu-latest env: - PLUGINS: express|body-parser|cookie-parser + PLUGINS: express + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test + + express-mongo-sanitize: + runs-on: ubuntu-latest + services: + mongodb: + image: circleci/mongo + ports: + - 27017:27017 + env: + PLUGINS: express-mongo-sanitize + PACKAGE_NAMES: express-mongo-sanitize + SERVICES: mongo steps: - uses: actions/checkout@v4 - uses: ./.github/actions/plugins/test @@ -542,6 +573,23 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/plugins/test + + mariadb: + runs-on: ubuntu-latest + services: + mysql: + image: mariadb:10.4 + env: + MYSQL_ALLOW_EMPTY_PASSWORD: 'yes' + MYSQL_DATABASE: 'db' + ports: + - 3306:3306 + env: + PLUGINS: mariadb + SERVICES: mariadb + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test memcached: runs-on: ubuntu-latest @@ -636,12 +684,29 @@ jobs: ports: - 3306:3306 env: - PLUGINS: mysql|mysql2|mariadb # TODO: move mysql2 to its own job + PLUGINS: mysql SERVICES: mysql steps: - uses: actions/checkout@v4 - uses: ./.github/actions/plugins/test + mysql2: + runs-on: ubuntu-latest + services: + mysql: + image: mariadb:10.4 + env: + MYSQL_ALLOW_EMPTY_PASSWORD: 'yes' + MYSQL_DATABASE: 'db' + ports: + - 3306:3306 + env: + PLUGINS: mysql2 + SERVICES: mysql2 + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test + net: runs-on: ubuntu-latest env: diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index c53c5b3064c..0a7d4094b8b 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -31,7 +31,6 @@ jobs: uses: DataDog/system-tests/.github/workflows/compute-workflow-parameters.yml@main with: library: nodejs - scenarios: CROSSED_TRACING_LIBRARIES scenarios_groups: essentials system-tests: diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 714eb493581..87d896df458 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,3 +24,10 @@ onboarding_tests_installer: onboarding_tests_k8s_injection: variables: WEBLOG_VARIANT: sample-app + +requirements_json_test: + rules: + - when: on_success + variables: + REQUIREMENTS_BLOCK_JSON_PATH: ".gitlab/requirements_block.json" + REQUIREMENTS_ALLOW_JSON_PATH: ".gitlab/requirements_allow.json" diff --git a/.gitlab/prepare-oci-package.sh b/.gitlab/prepare-oci-package.sh index b65b3e73d5c..af579f04355 100755 --- a/.gitlab/prepare-oci-package.sh +++ b/.gitlab/prepare-oci-package.sh @@ -21,3 +21,5 @@ fi echo -n $JS_PACKAGE_VERSION > packaging/sources/version cd packaging + +cp ../requirements.json sources/requirements.json diff --git a/.gitlab/requirements_allow.json b/.gitlab/requirements_allow.json new file mode 100644 index 00000000000..e832f6e7132 --- /dev/null +++ b/.gitlab/requirements_allow.json @@ -0,0 +1,19 @@ +[ + {"name": "min glibc x64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:2.17"}}, + {"name": "ok glibc x64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:2.23"}}, + {"name": "high glibc x64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:3.0"}}, + {"name": "musl x64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "musl:1.2.2"}}, + {"name": "min glibc arm64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "arm64", "libc": "glibc:2.17"}}, + {"name": "ok glibc arm64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "arm64", "libc": "glibc:2.27"}}, + {"name": "glibc x86","filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x86", "libc": "glibc:2.19"}}, + {"name": "musl arm","filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "arm", "libc": "musl:1.2.2"}}, + {"name": "musl arm64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "arm64", "libc": "musl:1.2.2"}}, + {"name": "musl x64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "musl:1.2.2"}}, + {"name": "musl x86", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x86", "libc": "musl:1.2.2"}}, + {"name": "windows x64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "windows", "arch": "x64"}}, + {"name": "windows x86", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "windows", "arch": "x86"}}, + {"name": "macos x64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "darwin", "arch": "x64"}}, + {"name": "macos arm64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "darwin", "arch": "arm64"}}, + {"name": "node app", "filepath": "/pathto/node", "args": ["/pathto/node", "./app.js"], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:2.40"}}, + {"name": "ts-node app", "filepath": "/pathto/ts-node", "args": ["/pathto/ts-node", "./app.js"], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:2.40"}} +] diff --git a/.gitlab/requirements_block.json b/.gitlab/requirements_block.json new file mode 100644 index 00000000000..e728f802915 --- /dev/null +++ b/.gitlab/requirements_block.json @@ -0,0 +1,11 @@ +[ + {"name": "unsupported 2.x glibc x64","filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:2.16"}}, + {"name": "unsupported 1.x glibc x64","filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:1.22"}}, + {"name": "unsupported 2.x.x glibc x64","filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:2.16.9"}}, + {"name": "unsupported 2.x glibc arm64","filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "arm64", "libc": "glibc:2.16"}}, + {"name": "unsupported 2.x.x glibc x64","filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "arm64", "libc": "glibc:2.16.9"}}, + {"name": "unsupported 2.x.x glibc x86","filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x86", "libc": "glibc:2.17"}}, + {"name": "npm","filepath": "/pathto/node", "args": ["/pathto/node", "/pathto/npm-cli.js"], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:2.40"}}, + {"name": "yarn","filepath": "/pathto/node", "args": ["/pathto/node", "/pathto/yarn.js"], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:2.40"}}, + {"name": "pnpm","filepath": "/pathto/node", "args": ["/pathto/node", "/pathto/pnpm.cjs"], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:2.40"}} +] diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index f36fac2da6c..0ce2aba174a 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -29,6 +29,7 @@ require,retry,MIT,Copyright 2011 Tim Koschützki Felix Geisendörfer require,rfdc,MIT,Copyright 2019 David Mark Clements require,semver,ISC,Copyright Isaac Z. Schlueter and Contributors require,shell-quote,mit,Copyright (c) 2013 James Halliday +dev,@apollo/server,MIT,Copyright (c) 2016-2020 Apollo Graph, Inc. (Formerly Meteor Development Group, Inc.) dev,@types/node,MIT,Copyright Authors dev,autocannon,MIT,Copyright 2016 Matteo Collina dev,aws-sdk,Apache 2.0,Copyright 2012-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. diff --git a/docker-compose.yml b/docker-compose.yml index a16fef8893d..81bdd3c2032 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -129,7 +129,7 @@ services: - KAFKA_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093 - KAFKA_CONTROLLER_QUORUM_VOTERS=1@127.0.0.1:9093 - KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER - - KAFKA_CLUSTER_ID=r4zt_wrqTRuT7W2NJsB_GA + - CLUSTER_ID=5L6g3nShT-eMCtK--X86sw - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://127.0.0.1:9092 - KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT diff --git a/docs/test.ts b/docs/test.ts index 9c6c7df6211..6c3f54c2598 100644 --- a/docs/test.ts +++ b/docs/test.ts @@ -536,3 +536,80 @@ const otelTraceId: string = spanContext.traceId const otelSpanId: string = spanContext.spanId const otelTraceFlags: number = spanContext.traceFlags const otelTraceState: opentelemetry.TraceState = spanContext.traceState! + +// -- LLM Observability -- +const llmobsEnableOptions = { + mlApp: 'mlApp', + agentlessEnabled: true +} +tracer.init({ + llmobs: llmobsEnableOptions, +}) +const llmobs = tracer.llmobs +const enabled = llmobs.enabled + +// manually enable +llmobs.enable({ + mlApp: 'mlApp', + agentlessEnabled: true +}) + +// manually disable +llmobs.disable() + +// trace block of code +llmobs.trace({ name: 'name', kind: 'llm' }, () => {}) +llmobs.trace({ kind: 'llm', name: 'myLLM', modelName: 'myModel', modelProvider: 'myProvider' }, () => {}) +llmobs.trace({ name: 'name', kind: 'llm' }, (span, cb) => { + llmobs.annotate(span, {}) + span.setTag('foo', 'bar') + cb(new Error('boom')) +}) + +// wrap a function +llmobs.wrap({ kind: 'llm' }, function myLLM () {})() +llmobs.wrap({ kind: 'llm', name: 'myLLM', modelName: 'myModel', modelProvider: 'myProvider' }, function myFunction () {})() + +// export a span +llmobs.enable({ mlApp: 'myApp' }) +llmobs.trace({ kind: 'llm', name: 'myLLM' }, (span) => { + const llmobsSpanCtx = llmobs.exportSpan(span) + llmobsSpanCtx.traceId; + llmobsSpanCtx.spanId; + + // submit evaluation + llmobs.disable() + llmobs.submitEvaluation(llmobsSpanCtx, { + label: 'my-eval-metric', + metricType: 'categorical', + value: 'good', + mlApp: 'myApp', + tags: {}, + timestampMs: Date.now() + }) +}) + +// annotate a span +llmobs.annotate({ + inputData: 'input', + outputData: 'output', + metadata: {}, + metrics: { + inputTokens: 10, + outputTokens: 5, + totalTokens: 15 + }, + tags: {} +}) +llmobs.annotate(span, { + inputData: 'input', + outputData: 'output', + metadata: {}, + metrics: {}, + tags: {} +}) + + + +// flush +llmobs.flush() diff --git a/index.d.ts b/index.d.ts index f969984162a..86673bd2c98 100644 --- a/index.d.ts +++ b/index.d.ts @@ -137,6 +137,11 @@ interface Tracer extends opentracing.Tracer { TracerProvider: tracer.opentelemetry.TracerProvider; dogstatsd: tracer.DogStatsD; + + /** + * LLM Observability SDK + */ + llmobs: tracer.llmobs.LLMObs; } // left out of the namespace, so it @@ -752,6 +757,11 @@ declare namespace tracer { */ maxDepth?: number } + + /** + * Configuration enabling LLM Observability. Enablement is superceded by the DD_LLMOBS_ENABLED environment variable. + */ + llmobs?: llmobs.LLMObsEnableOptions } /** @@ -2198,6 +2208,331 @@ declare namespace tracer { */ telemetryVerbosity?: string } + + export namespace llmobs { + export interface LLMObs { + + /** + * Whether or not LLM Observability is enabled. + */ + enabled: boolean, + + /** + * Enable LLM Observability tracing. + */ + enable (options: LLMObsEnableOptions): void, + + /** + * Disable LLM Observability tracing. + */ + disable (): void, + + /** + * Instruments a function by automatically creating a span activated on its + * scope. + * + * The span will automatically be finished when one of these conditions is + * met: + * + * * The function returns a promise, in which case the span will finish when + * the promise is resolved or rejected. + * * The function takes a callback as its second parameter, in which case the + * span will finish when that callback is called. + * * The function doesn't accept a callback and doesn't return a promise, in + * which case the span will finish at the end of the function execution. + * @param fn The function to instrument. + * @param options Optional LLM Observability span options. + * @returns The return value of the function. + */ + trace (options: LLMObsNamedSpanOptions, fn: (span: tracer.Span, done: (error?: Error) => void) => T): T + + /** + * Wrap a function to automatically create a span activated on its + * scope when it's called. + * + * The span will automatically be finished when one of these conditions is + * met: + * + * * The function returns a promise, in which case the span will finish when + * the promise is resolved or rejected. + * * The function takes a callback as its last parameter, in which case the + * span will finish when that callback is called. + * * The function doesn't accept a callback and doesn't return a promise, in + * which case the span will finish at the end of the function execution. + * @param fn The function to instrument. + * @param options Optional LLM Observability span options. + * @returns A new function that wraps the provided function with span creation. + */ + wrap any> (options: LLMObsNamelessSpanOptions, fn: T): T + + /** + * Decorate a function in a javascript runtime that supports function decorators. + * Note that this is **not** supported in the Node.js runtime, but is in TypeScript. + * + * In TypeScript, this decorator is only supported in contexts where general TypeScript + * function decorators are supported. + * + * @param options Optional LLM Observability span options. + */ + decorate (options: llmobs.LLMObsNamelessSpanOptions): any + + /** + * Returns a representation of a span to export its span and trace IDs. + * If no span is provided, the current LLMObs-type span will be used. + * @param span Optional span to export. + * @returns An object containing the span and trace IDs. + */ + exportSpan (span?: tracer.Span): llmobs.ExportedLLMObsSpan + + + /** + * Sets inputs, outputs, tags, metadata, and metrics as provided for a given LLM Observability span. + * Note that with the exception of tags, this method will override any existing values for the provided fields. + * + * For example: + * ```javascript + * llmobs.trace({ kind: 'llm', name: 'myLLM', modelName: 'gpt-4o', modelProvider: 'openai' }, () => { + * llmobs.annotate({ + * inputData: [{ content: 'system prompt, role: 'system' }, { content: 'user prompt', role: 'user' }], + * outputData: { content: 'response', role: 'ai' }, + * metadata: { temperature: 0.7 }, + * tags: { host: 'localhost' }, + * metrics: { inputTokens: 10, outputTokens: 20, totalTokens: 30 } + * }) + * }) + * ``` + * + * @param span The span to annotate (defaults to the current LLM Observability span if not provided) + * @param options An object containing the inputs, outputs, tags, metadata, and metrics to set on the span. + */ + annotate (options: llmobs.AnnotationOptions): void + annotate (span: tracer.Span | undefined, options: llmobs.AnnotationOptions): void + + /** + * Submits a custom evalutation metric for a given span ID and trace ID. + * @param spanContext The span context of the span to submit the evaluation metric for. + * @param options An object containing the label, metric type, value, and tags of the evaluation metric. + */ + submitEvaluation (spanContext: llmobs.ExportedLLMObsSpan, options: llmobs.EvaluationOptions): void + + /** + * Flushes any remaining spans and evaluation metrics to LLM Observability. + */ + flush (): void + } + + interface EvaluationOptions { + /** + * The name of the evalutation metric + */ + label: string, + + /** + * The type of evaluation metric, one of 'categorical' or 'score' + */ + metricType: 'categorical' | 'score', + + /** + * The value of the evaluation metric. + * Must be string for 'categorical' metrics and number for 'score' metrics. + */ + value: string | number, + + /** + * An object of string key-value pairs to tag the evaluation metric with. + */ + tags?: { [key: string]: any }, + + /** + * The name of the ML application + */ + mlApp?: string, + + /** + * The timestamp in milliseconds when the evaluation metric result was generated. + */ + timestampMs?: number + } + + interface Document { + /** + * Document text + */ + text?: string, + + /** + * Document name + */ + name?: string, + + /** + * Document ID + */ + id?: string, + + /** + * Score of the document retrieval as a source of ground truth + */ + score?: number + } + + /** + * Represents a single LLM chat model message + */ + interface Message { + /** + * Content of the message. + */ + content: string, + + /** + * Role of the message (ie system, user, ai) + */ + role?: string, + + /** + * Tool calls of the message + */ + toolCalls?: ToolCall[], + } + + /** + * Represents a single tool call for an LLM chat model message + */ + interface ToolCall { + /** + * Name of the tool + */ + name?: string, + + /** + * Arguments passed to the tool + */ + arguments?: { [key: string]: any }, + + /** + * The tool ID + */ + toolId?: string, + + /** + * The tool type + */ + type?: string + } + + /** + * Annotation options for LLM Observability spans. + */ + interface AnnotationOptions { + /** + * A single input string, object, or a list of objects based on the span kind: + * 1. LLM spans: accepts a string, or an object of the form {content: "...", role: "..."}, or a list of objects with the same signature. + * 2. Embedding spans: accepts a string, list of strings, or an object of the form {text: "...", ...}, or a list of objects with the same signature. + * 3. Other: any JSON serializable type + */ + inputData?: string | Message | Message[] | Document | Document[] | { [key: string]: any }, + + /** + * A single output string, object, or a list of objects based on the span kind: + * 1. LLM spans: accepts a string, or an object of the form {content: "...", role: "..."}, or a list of objects with the same signature. + * 2. Retrieval spans: An object containing any of the key value pairs {name: str, id: str, text: str, source: number} or a list of dictionaries with the same signature. + * 3. Other: any JSON serializable type + */ + outputData?: string | Message | Message[] | Document | Document[] | { [key: string]: any }, + + /** + * Object of JSON serializable key-value metadata pairs relevant to the input/output operation described by the LLM Observability span. + */ + metadata?: { [key: string]: any }, + + /** + * Object of JSON seraliazable key-value metrics (number) pairs, such as `{input,output,total}Tokens` + */ + metrics?: { [key: string]: number }, + + /** + * Object of JSON serializable key-value tag pairs to set or update on the LLM Observability span regarding the span's context. + */ + tags?: { [key: string]: any } + } + + /** + * An object containing the span ID and trace ID of interest + */ + interface ExportedLLMObsSpan { + /** + * Trace ID associated with the span of interest + */ + traceId: string, + + /** + * Span ID associated with the span of interest + */ + spanId: string, + } + + interface LLMObsSpanOptions extends SpanOptions { + /** + * LLM Observability span kind. One of `agent`, `workflow`, `task`, `tool`, `retrieval`, `embedding`, or `llm`. + */ + kind: llmobs.spanKind, + + /** + * The ID of the underlying user session. Required for tracking sessions. + */ + sessionId?: string, + + /** + * The name of the ML application that the agent is orchestrating. + * If not provided, the default value will be set to mlApp provided during initalization, or `DD_LLMOBS_ML_APP`. + */ + mlApp?: string, + + /** + * The name of the invoked LLM or embedding model. Only used on `llm` and `embedding` spans. + */ + modelName?: string, + + /** + * The name of the invoked LLM or embedding model provider. Only used on `llm` and `embedding` spans. + * If not provided for LLM or embedding spans, a default value of 'custom' will be set. + */ + modelProvider?: string, + } + + interface LLMObsNamedSpanOptions extends LLMObsSpanOptions { + /** + * The name of the traced operation. This is a required option. + */ + name: string, + } + + interface LLMObsNamelessSpanOptions extends LLMObsSpanOptions { + /** + * The name of the traced operation. + */ + name?: string, + } + + /** + * Options for enabling LLM Observability tracing. + */ + interface LLMObsEnableOptions { + /** + * The name of your ML application. + */ + mlApp?: string, + + /** + * Set to `true` to disbale sending data that requires a Datadog Agent. + */ + agentlessEnabled?: boolean, + } + + /** @hidden */ + type spanKind = 'agent' | 'workflow' | 'task' | 'tool' | 'retrieval' | 'embedding' | 'llm' + } } /** diff --git a/integration-tests/appsec/multer.spec.js b/integration-tests/appsec/multer.spec.js new file mode 100644 index 00000000000..91b3e93d531 --- /dev/null +++ b/integration-tests/appsec/multer.spec.js @@ -0,0 +1,138 @@ +'use strict' + +const { assert } = require('chai') +const path = require('path') +const axios = require('axios') + +const { + createSandbox, + FakeAgent, + spawnProc +} = require('../helpers') + +describe('multer', () => { + let sandbox, cwd, startupTestFile, agent, proc, env + + ['1.4.4-lts.1', '1.4.5-lts.1'].forEach((version) => { + describe(`v${version}`, () => { + before(async () => { + sandbox = await createSandbox(['express', `multer@${version}`]) + cwd = sandbox.folder + startupTestFile = path.join(cwd, 'appsec', 'multer', 'index.js') + }) + + after(async () => { + await sandbox.remove() + }) + + beforeEach(async () => { + agent = await new FakeAgent().start() + + env = { + AGENT_PORT: agent.port, + DD_APPSEC_RULES: path.join(cwd, 'appsec', 'multer', 'body-parser-rules.json') + } + + const execArgv = [] + + proc = await spawnProc(startupTestFile, { cwd, env, execArgv }) + }) + + afterEach(async () => { + proc.kill() + await agent.stop() + }) + + describe('Suspicious request blocking', () => { + describe('using middleware', () => { + it('should not block the request without an attack', async () => { + const form = new FormData() + form.append('key', 'value') + + const res = await axios.post(proc.url, form) + + assert.equal(res.data, 'DONE') + }) + + it('should block the request when attack is detected', async () => { + try { + const form = new FormData() + form.append('key', 'testattack') + + await axios.post(proc.url, form) + + return Promise.reject(new Error('Request should not return 200')) + } catch (e) { + assert.equal(e.response.status, 403) + } + }) + }) + + describe('not using middleware', () => { + it('should not block the request without an attack', async () => { + const form = new FormData() + form.append('key', 'value') + + const res = await axios.post(`${proc.url}/no-middleware`, form) + + assert.equal(res.data, 'DONE') + }) + + it('should block the request when attack is detected', async () => { + try { + const form = new FormData() + form.append('key', 'testattack') + + await axios.post(`${proc.url}/no-middleware`, form) + + return Promise.reject(new Error('Request should not return 200')) + } catch (e) { + assert.equal(e.response.status, 403) + } + }) + }) + }) + + describe('IAST', () => { + function assertCmdInjection ({ payload }) { + assert.isArray(payload) + assert.strictEqual(payload.length, 1) + assert.isArray(payload[0]) + + const { meta } = payload[0][0] + + assert.property(meta, '_dd.iast.json') + + const iastJson = JSON.parse(meta['_dd.iast.json']) + + assert.isTrue(iastJson.vulnerabilities.some(v => v.type === 'COMMAND_INJECTION')) + assert.isTrue(iastJson.sources.some(s => s.origin === 'http.request.body')) + } + + describe('using middleware', () => { + it('should taint multipart body', async () => { + const resultPromise = agent.assertMessageReceived(assertCmdInjection) + + const formData = new FormData() + formData.append('command', 'echo 1') + await axios.post(`${proc.url}/cmd`, formData) + + return resultPromise + }) + }) + + describe('not using middleware', () => { + it('should taint multipart body', async () => { + const resultPromise = agent.assertMessageReceived(assertCmdInjection) + + const formData = new FormData() + formData.append('command', 'echo 1') + await axios.post(`${proc.url}/cmd-no-middleware`, formData) + + return resultPromise + }) + }) + }) + }) + }) +}) diff --git a/integration-tests/appsec/multer/body-parser-rules.json b/integration-tests/appsec/multer/body-parser-rules.json new file mode 100644 index 00000000000..6b22c7cbbf6 --- /dev/null +++ b/integration-tests/appsec/multer/body-parser-rules.json @@ -0,0 +1,33 @@ +{ + "version": "2.2", + "metadata": { + "rules_version": "1.5.0" + }, + "rules": [ + { + "id": "test-rule-id-1", + "name": "test-rule-name-1", + "tags": { + "type": "security_scanner", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.body" + } + ], + "list": [ + "testattack" + ] + }, + "operator": "phrase_match" + } + ], + "transformers": ["lowercase"], + "on_match": ["block"] + } + ] +} diff --git a/integration-tests/appsec/multer/index.js b/integration-tests/appsec/multer/index.js new file mode 100644 index 00000000000..b872af9dc8e --- /dev/null +++ b/integration-tests/appsec/multer/index.js @@ -0,0 +1,64 @@ +'use strict' + +const options = { + appsec: { + enabled: true + }, + iast: { + enabled: true, + requestSampling: 100 + } +} + +if (process.env.AGENT_PORT) { + options.port = process.env.AGENT_PORT +} + +if (process.env.AGENT_URL) { + options.url = process.env.AGENT_URL +} + +const tracer = require('dd-trace') +tracer.init(options) + +const http = require('http') +const express = require('express') +const childProcess = require('child_process') + +const multer = require('multer') +const uploadToMemory = multer({ storage: multer.memoryStorage(), limits: { fileSize: 200000 } }) + +const app = express() + +app.post('/', uploadToMemory.single('file'), (req, res) => { + res.end('DONE') +}) + +app.post('/no-middleware', (req, res) => { + uploadToMemory.none()(req, res, () => { + res.end('DONE') + }) +}) + +app.post('/cmd', uploadToMemory.single('file'), (req, res) => { + childProcess.exec(req.body.command, () => { + res.end('DONE') + }) +}) + +app.post('/cmd-no-middleware', (req, res) => { + uploadToMemory.none()(req, res, () => { + childProcess.exec(req.body.command, () => { + res.end('DONE') + }) + }) +}) + +app.get('/', (req, res) => { + res.status(200).send('hello world') +}) + +const server = http.createServer(app).listen(0, () => { + const port = server.address().port + process.send?.({ port }) +}) diff --git a/integration-tests/debugger/index.spec.js b/integration-tests/debugger/basic.spec.js similarity index 50% rename from integration-tests/debugger/index.spec.js rename to integration-tests/debugger/basic.spec.js index 613c4eeb695..3330a6c32d3 100644 --- a/integration-tests/debugger/index.spec.js +++ b/integration-tests/debugger/basic.spec.js @@ -1,59 +1,18 @@ 'use strict' -const path = require('path') -const { randomUUID } = require('crypto') const os = require('os') -const getPort = require('get-port') -const Axios = require('axios') const { assert } = require('chai') -const { assertObjectContains, assertUUID, createSandbox, FakeAgent, spawnProc } = require('../helpers') +const { pollInterval, setup } = require('./utils') +const { assertObjectContains, assertUUID } = require('../helpers') const { ACKNOWLEDGED, ERROR } = require('../../packages/dd-trace/src/appsec/remote_config/apply_states') const { version } = require('../../package.json') -const probeFile = 'debugger/target-app/index.js' -const probeLineNo = 14 -const pollInterval = 1 - describe('Dynamic Instrumentation', function () { - let axios, sandbox, cwd, appPort, appFile, agent, proc, rcConfig - - before(async function () { - sandbox = await createSandbox(['fastify']) - cwd = sandbox.folder - appFile = path.join(cwd, ...probeFile.split('/')) - }) - - after(async function () { - await sandbox.remove() - }) - - beforeEach(async function () { - rcConfig = generateRemoteConfig() - appPort = await getPort() - agent = await new FakeAgent().start() - proc = await spawnProc(appFile, { - cwd, - env: { - APP_PORT: appPort, - DD_DYNAMIC_INSTRUMENTATION_ENABLED: true, - DD_TRACE_AGENT_PORT: agent.port, - DD_TRACE_DEBUG: process.env.DD_TRACE_DEBUG, // inherit to make debugging the sandbox easier - DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS: pollInterval - } - }) - axios = Axios.create({ - baseURL: `http://localhost:${appPort}` - }) - }) - - afterEach(async function () { - proc.kill() - await agent.stop() - }) + const t = setup() it('base case: target app should work as expected if no test probe has been added', async function () { - const response = await axios.get('/foo') + const response = await t.axios.get('/foo') assert.strictEqual(response.status, 200) assert.deepStrictEqual(response.data, { hello: 'foo' }) }) @@ -61,7 +20,7 @@ describe('Dynamic Instrumentation', function () { describe('diagnostics messages', function () { it('should send expected diagnostics messages if probe is received and triggered', function (done) { let receivedAckUpdate = false - const probeId = rcConfig.config.id + const probeId = t.rcConfig.config.id const expectedPayloads = [{ ddsource: 'dd_debugger', service: 'node', @@ -76,8 +35,8 @@ describe('Dynamic Instrumentation', function () { debugger: { diagnostics: { probeId, version: 0, status: 'EMITTING' } } }] - agent.on('remote-config-ack-update', (id, version, state, error) => { - assert.strictEqual(id, rcConfig.id) + t.agent.on('remote-config-ack-update', (id, version, state, error) => { + assert.strictEqual(id, t.rcConfig.id) assert.strictEqual(version, 1) assert.strictEqual(state, ACKNOWLEDGED) assert.notOk(error) // falsy check since error will be an empty string, but that's an implementation detail @@ -86,13 +45,13 @@ describe('Dynamic Instrumentation', function () { endIfDone() }) - agent.on('debugger-diagnostics', ({ payload }) => { + t.agent.on('debugger-diagnostics', ({ payload }) => { const expected = expectedPayloads.shift() assertObjectContains(payload, expected) assertUUID(payload.debugger.diagnostics.runtimeId) if (payload.debugger.diagnostics.status === 'INSTALLED') { - axios.get('/foo') + t.axios.get('/foo') .then((response) => { assert.strictEqual(response.status, 200) assert.deepStrictEqual(response.data, { hello: 'foo' }) @@ -103,7 +62,7 @@ describe('Dynamic Instrumentation', function () { } }) - agent.addRemoteConfig(rcConfig) + t.agent.addRemoteConfig(t.rcConfig) function endIfDone () { if (receivedAckUpdate && expectedPayloads.length === 0) done() @@ -112,7 +71,7 @@ describe('Dynamic Instrumentation', function () { it('should send expected diagnostics messages if probe is first received and then updated', function (done) { let receivedAckUpdates = 0 - const probeId = rcConfig.config.id + const probeId = t.rcConfig.config.id const expectedPayloads = [{ ddsource: 'dd_debugger', service: 'node', @@ -132,14 +91,14 @@ describe('Dynamic Instrumentation', function () { }] const triggers = [ () => { - rcConfig.config.version++ - agent.updateRemoteConfig(rcConfig.id, rcConfig.config) + t.rcConfig.config.version++ + t.agent.updateRemoteConfig(t.rcConfig.id, t.rcConfig.config) }, () => {} ] - agent.on('remote-config-ack-update', (id, version, state, error) => { - assert.strictEqual(id, rcConfig.id) + t.agent.on('remote-config-ack-update', (id, version, state, error) => { + assert.strictEqual(id, t.rcConfig.id) assert.strictEqual(version, ++receivedAckUpdates) assert.strictEqual(state, ACKNOWLEDGED) assert.notOk(error) // falsy check since error will be an empty string, but that's an implementation detail @@ -147,7 +106,7 @@ describe('Dynamic Instrumentation', function () { endIfDone() }) - agent.on('debugger-diagnostics', ({ payload }) => { + t.agent.on('debugger-diagnostics', ({ payload }) => { const expected = expectedPayloads.shift() assertObjectContains(payload, expected) assertUUID(payload.debugger.diagnostics.runtimeId) @@ -155,7 +114,7 @@ describe('Dynamic Instrumentation', function () { endIfDone() }) - agent.addRemoteConfig(rcConfig) + t.agent.addRemoteConfig(t.rcConfig) function endIfDone () { if (receivedAckUpdates === 2 && expectedPayloads.length === 0) done() @@ -165,7 +124,7 @@ describe('Dynamic Instrumentation', function () { it('should send expected diagnostics messages if probe is first received and then deleted', function (done) { let receivedAckUpdate = false let payloadsProcessed = false - const probeId = rcConfig.config.id + const probeId = t.rcConfig.config.id const expectedPayloads = [{ ddsource: 'dd_debugger', service: 'node', @@ -176,8 +135,8 @@ describe('Dynamic Instrumentation', function () { debugger: { diagnostics: { probeId, version: 0, status: 'INSTALLED' } } }] - agent.on('remote-config-ack-update', (id, version, state, error) => { - assert.strictEqual(id, rcConfig.id) + t.agent.on('remote-config-ack-update', (id, version, state, error) => { + assert.strictEqual(id, t.rcConfig.id) assert.strictEqual(version, 1) assert.strictEqual(state, ACKNOWLEDGED) assert.notOk(error) // falsy check since error will be an empty string, but that's an implementation detail @@ -186,13 +145,13 @@ describe('Dynamic Instrumentation', function () { endIfDone() }) - agent.on('debugger-diagnostics', ({ payload }) => { + t.agent.on('debugger-diagnostics', ({ payload }) => { const expected = expectedPayloads.shift() assertObjectContains(payload, expected) assertUUID(payload.debugger.diagnostics.runtimeId) if (payload.debugger.diagnostics.status === 'INSTALLED') { - agent.removeRemoteConfig(rcConfig.id) + t.agent.removeRemoteConfig(t.rcConfig.id) // Wait a little to see if we get any follow-up `debugger-diagnostics` messages setTimeout(() => { payloadsProcessed = true @@ -201,7 +160,7 @@ describe('Dynamic Instrumentation', function () { } }) - agent.addRemoteConfig(rcConfig) + t.agent.addRemoteConfig(t.rcConfig) function endIfDone () { if (receivedAckUpdate && payloadsProcessed) done() @@ -214,17 +173,17 @@ describe('Dynamic Instrumentation', function () { { status: 'ERROR' } ], [ 'should send expected error diagnostics messages if probe type isn\'t supported', - generateProbeConfig({ type: 'INVALID_PROBE' }) + t.generateProbeConfig({ type: 'INVALID_PROBE' }) ], [ 'should send expected error diagnostics messages if it isn\'t a line-probe', - generateProbeConfig({ where: { foo: 'bar' } }) // TODO: Use valid schema for method probe instead + t.generateProbeConfig({ where: { foo: 'bar' } }) // TODO: Use valid schema for method probe instead ]] for (const [title, config, customErrorDiagnosticsObj] of unsupporedOrInvalidProbes) { it(title, function (done) { let receivedAckUpdate = false - agent.on('remote-config-ack-update', (id, version, state, error) => { + t.agent.on('remote-config-ack-update', (id, version, state, error) => { assert.strictEqual(id, `logProbe_${config.id}`) assert.strictEqual(version, 1) assert.strictEqual(state, ERROR) @@ -245,7 +204,7 @@ describe('Dynamic Instrumentation', function () { debugger: { diagnostics: customErrorDiagnosticsObj ?? { probeId, version: 0, status: 'ERROR' } } }] - agent.on('debugger-diagnostics', ({ payload }) => { + t.agent.on('debugger-diagnostics', ({ payload }) => { const expected = expectedPayloads.shift() assertObjectContains(payload, expected) const { diagnostics } = payload.debugger @@ -261,7 +220,7 @@ describe('Dynamic Instrumentation', function () { endIfDone() }) - agent.addRemoteConfig({ + t.agent.addRemoteConfig({ product: 'LIVE_DEBUGGING', id: `logProbe_${config.id}`, config @@ -276,29 +235,25 @@ describe('Dynamic Instrumentation', function () { describe('input messages', function () { it('should capture and send expected payload when a log line probe is triggered', function (done) { - agent.on('debugger-diagnostics', ({ payload }) => { - if (payload.debugger.diagnostics.status === 'INSTALLED') { - axios.get('/foo') - } - }) + t.triggerBreakpoint() - agent.on('debugger-input', ({ payload }) => { + t.agent.on('debugger-input', ({ payload }) => { const expected = { ddsource: 'dd_debugger', hostname: os.hostname(), service: 'node', message: 'Hello World!', logger: { - name: 'debugger/target-app/index.js', + name: t.breakpoint.file, method: 'handler', version, thread_name: 'MainThread' }, 'debugger.snapshot': { probe: { - id: rcConfig.config.id, + id: t.rcConfig.config.id, version: 0, - location: { file: probeFile, lines: [String(probeLineNo)] } + location: { file: t.breakpoint.file, lines: [String(t.breakpoint.line)] } }, language: 'javascript' } @@ -322,50 +277,51 @@ describe('Dynamic Instrumentation', function () { assert.isAbove(frame.columnNumber, 0) } const topFrame = payload['debugger.snapshot'].stack[0] - assert.match(topFrame.fileName, new RegExp(`${appFile}$`)) // path seems to be prefeixed with `/private` on Mac + // path seems to be prefeixed with `/private` on Mac + assert.match(topFrame.fileName, new RegExp(`${t.appFile}$`)) assert.strictEqual(topFrame.function, 'handler') - assert.strictEqual(topFrame.lineNumber, probeLineNo) + assert.strictEqual(topFrame.lineNumber, t.breakpoint.line) assert.strictEqual(topFrame.columnNumber, 3) done() }) - agent.addRemoteConfig(rcConfig) + t.agent.addRemoteConfig(t.rcConfig) }) it('should respond with updated message if probe message is updated', function (done) { const expectedMessages = ['Hello World!', 'Hello Updated World!'] const triggers = [ async () => { - await axios.get('/foo') - rcConfig.config.version++ - rcConfig.config.template = 'Hello Updated World!' - agent.updateRemoteConfig(rcConfig.id, rcConfig.config) + await t.axios.get('/foo') + t.rcConfig.config.version++ + t.rcConfig.config.template = 'Hello Updated World!' + t.agent.updateRemoteConfig(t.rcConfig.id, t.rcConfig.config) }, async () => { - await axios.get('/foo') + await t.axios.get('/foo') } ] - agent.on('debugger-diagnostics', ({ payload }) => { + t.agent.on('debugger-diagnostics', ({ payload }) => { if (payload.debugger.diagnostics.status === 'INSTALLED') triggers.shift()().catch(done) }) - agent.on('debugger-input', ({ payload }) => { + t.agent.on('debugger-input', ({ payload }) => { assert.strictEqual(payload.message, expectedMessages.shift()) if (expectedMessages.length === 0) done() }) - agent.addRemoteConfig(rcConfig) + t.agent.addRemoteConfig(t.rcConfig) }) it('should not trigger if probe is deleted', function (done) { - agent.on('debugger-diagnostics', async ({ payload }) => { + t.agent.on('debugger-diagnostics', async ({ payload }) => { try { if (payload.debugger.diagnostics.status === 'INSTALLED') { - agent.once('remote-confg-responded', async () => { + t.agent.once('remote-confg-responded', async () => { try { - await axios.get('/foo') + await t.axios.get('/foo') // We want to wait enough time to see if the client triggers on the breakpoint so that the test can fail // if it does, but not so long that the test times out. // TODO: Is there some signal we can use instead of a timer? @@ -377,7 +333,7 @@ describe('Dynamic Instrumentation', function () { } }) - agent.removeRemoteConfig(rcConfig.id) + t.agent.removeRemoteConfig(t.rcConfig.id) } } catch (err) { // Nessecary hack: Any errors thrown inside of an async function is invisible to Mocha unless the outer `it` @@ -386,190 +342,25 @@ describe('Dynamic Instrumentation', function () { } }) - agent.on('debugger-input', () => { + t.agent.on('debugger-input', () => { assert.fail('should not capture anything when the probe is deleted') }) - agent.addRemoteConfig(rcConfig) - }) - - describe('with snapshot', () => { - beforeEach(() => { - // Trigger the breakpoint once probe is successfully installed - agent.on('debugger-diagnostics', ({ payload }) => { - if (payload.debugger.diagnostics.status === 'INSTALLED') { - axios.get('/foo') - } - }) - }) - - it('should capture a snapshot', (done) => { - agent.on('debugger-input', ({ payload: { 'debugger.snapshot': { captures } } }) => { - assert.deepEqual(Object.keys(captures), ['lines']) - assert.deepEqual(Object.keys(captures.lines), [String(probeLineNo)]) - - const { locals } = captures.lines[probeLineNo] - const { request, fastify, getSomeData } = locals - delete locals.request - delete locals.fastify - delete locals.getSomeData - - // from block scope - assert.deepEqual(locals, { - nil: { type: 'null', isNull: true }, - undef: { type: 'undefined' }, - bool: { type: 'boolean', value: 'true' }, - num: { type: 'number', value: '42' }, - bigint: { type: 'bigint', value: '42' }, - str: { type: 'string', value: 'foo' }, - lstr: { - type: 'string', - // eslint-disable-next-line max-len - value: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor i', - truncated: true, - size: 445 - }, - sym: { type: 'symbol', value: 'Symbol(foo)' }, - regex: { type: 'RegExp', value: '/bar/i' }, - arr: { - type: 'Array', - elements: [ - { type: 'number', value: '1' }, - { type: 'number', value: '2' }, - { type: 'number', value: '3' } - ] - }, - obj: { - type: 'Object', - fields: { - foo: { - type: 'Object', - fields: { - baz: { type: 'number', value: '42' }, - nil: { type: 'null', isNull: true }, - undef: { type: 'undefined' }, - deep: { - type: 'Object', - fields: { nested: { type: 'Object', notCapturedReason: 'depth' } } - } - } - }, - bar: { type: 'boolean', value: 'true' } - } - }, - emptyObj: { type: 'Object', fields: {} }, - fn: { - type: 'Function', - fields: { - length: { type: 'number', value: '0' }, - name: { type: 'string', value: 'fn' } - } - }, - p: { - type: 'Promise', - fields: { - '[[PromiseState]]': { type: 'string', value: 'fulfilled' }, - '[[PromiseResult]]': { type: 'undefined' } - } - } - }) - - // from local scope - // There's no reason to test the `request` object 100%, instead just check its fingerprint - assert.deepEqual(Object.keys(request), ['type', 'fields']) - assert.equal(request.type, 'Request') - assert.deepEqual(request.fields.id, { type: 'string', value: 'req-1' }) - assert.deepEqual(request.fields.params, { - type: 'NullObject', fields: { name: { type: 'string', value: 'foo' } } - }) - assert.deepEqual(request.fields.query, { type: 'Object', fields: {} }) - assert.deepEqual(request.fields.body, { type: 'undefined' }) - - // from closure scope - // There's no reason to test the `fastify` object 100%, instead just check its fingerprint - assert.deepEqual(Object.keys(fastify), ['type', 'fields']) - assert.equal(fastify.type, 'Object') - - assert.deepEqual(getSomeData, { - type: 'Function', - fields: { - length: { type: 'number', value: '0' }, - name: { type: 'string', value: 'getSomeData' } - } - }) - - done() - }) - - agent.addRemoteConfig(generateRemoteConfig({ captureSnapshot: true })) - }) - - it('should respect maxReferenceDepth', (done) => { - agent.on('debugger-input', ({ payload: { 'debugger.snapshot': { captures } } }) => { - const { locals } = captures.lines[probeLineNo] - delete locals.request - delete locals.fastify - delete locals.getSomeData - - assert.deepEqual(locals, { - nil: { type: 'null', isNull: true }, - undef: { type: 'undefined' }, - bool: { type: 'boolean', value: 'true' }, - num: { type: 'number', value: '42' }, - bigint: { type: 'bigint', value: '42' }, - str: { type: 'string', value: 'foo' }, - lstr: { - type: 'string', - // eslint-disable-next-line max-len - value: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor i', - truncated: true, - size: 445 - }, - sym: { type: 'symbol', value: 'Symbol(foo)' }, - regex: { type: 'RegExp', value: '/bar/i' }, - arr: { type: 'Array', notCapturedReason: 'depth' }, - obj: { type: 'Object', notCapturedReason: 'depth' }, - emptyObj: { type: 'Object', notCapturedReason: 'depth' }, - fn: { type: 'Function', notCapturedReason: 'depth' }, - p: { type: 'Promise', notCapturedReason: 'depth' } - }) - - done() - }) - - agent.addRemoteConfig(generateRemoteConfig({ captureSnapshot: true, capture: { maxReferenceDepth: 0 } })) - }) - - it('should respect maxLength', (done) => { - agent.on('debugger-input', ({ payload: { 'debugger.snapshot': { captures } } }) => { - const { locals } = captures.lines[probeLineNo] - - assert.deepEqual(locals.lstr, { - type: 'string', - value: 'Lorem ipsu', - truncated: true, - size: 445 - }) - - done() - }) - - agent.addRemoteConfig(generateRemoteConfig({ captureSnapshot: true, capture: { maxLength: 10 } })) - }) + t.agent.addRemoteConfig(t.rcConfig) }) }) - describe('race conditions', () => { - it('should remove the last breakpoint completely before trying to add a new one', (done) => { - const rcConfig2 = generateRemoteConfig() + describe('race conditions', function () { + it('should remove the last breakpoint completely before trying to add a new one', function (done) { + const rcConfig2 = t.generateRemoteConfig() - agent.on('debugger-diagnostics', ({ payload: { debugger: { diagnostics: { status, probeId } } } }) => { + t.agent.on('debugger-diagnostics', ({ payload: { debugger: { diagnostics: { status, probeId } } } }) => { if (status !== 'INSTALLED') return - if (probeId === rcConfig.config.id) { + if (probeId === t.rcConfig.config.id) { // First INSTALLED payload: Try to trigger the race condition. - agent.removeRemoteConfig(rcConfig.id) - agent.addRemoteConfig(rcConfig2) + t.agent.removeRemoteConfig(t.rcConfig.id) + t.agent.addRemoteConfig(rcConfig2) } else { // Second INSTALLED payload: Perform an HTTP request to see if we successfully handled the race condition. let finished = false @@ -582,14 +373,14 @@ describe('Dynamic Instrumentation', function () { // If we successfully handled the race condition, the probe will trigger, we'll get a probe result and the // following event listener will be called: - agent.once('debugger-input', () => { + t.agent.once('debugger-input', () => { clearTimeout(timer) finished = true done() }) // Perform HTTP request to try and trigger the probe - axios.get('/foo').catch((err) => { + t.axios.get('/foo').catch((err) => { // If the request hasn't fully completed by the time the tests ends and the target app is destroyed, Axios // will complain with a "socket hang up" error. Hence this sanity check before calling `done(err)`. If we // later add more tests below this one, this shouuldn't be an issue. @@ -598,34 +389,7 @@ describe('Dynamic Instrumentation', function () { } }) - agent.addRemoteConfig(rcConfig) + t.agent.addRemoteConfig(t.rcConfig) }) }) }) - -function generateRemoteConfig (overrides = {}) { - overrides.id = overrides.id || randomUUID() - return { - product: 'LIVE_DEBUGGING', - id: `logProbe_${overrides.id}`, - config: generateProbeConfig(overrides) - } -} - -function generateProbeConfig (overrides) { - return { - id: randomUUID(), - version: 0, - type: 'LOG_PROBE', - language: 'javascript', - where: { sourceFile: probeFile, lines: [String(probeLineNo)] }, - tags: [], - template: 'Hello World!', - segments: [{ str: 'Hello World!' }], - captureSnapshot: false, - capture: { maxReferenceDepth: 3 }, - sampling: { snapshotsPerSecond: 5000 }, - evaluateAt: 'EXIT', - ...overrides - } -} diff --git a/integration-tests/debugger/snapshot-pruning.spec.js b/integration-tests/debugger/snapshot-pruning.spec.js new file mode 100644 index 00000000000..91190a1c25d --- /dev/null +++ b/integration-tests/debugger/snapshot-pruning.spec.js @@ -0,0 +1,43 @@ +'use strict' + +const { assert } = require('chai') +const { setup, getBreakpointInfo } = require('./utils') + +const { line } = getBreakpointInfo() + +describe('Dynamic Instrumentation', function () { + const t = setup() + + describe('input messages', function () { + describe('with snapshot', function () { + beforeEach(t.triggerBreakpoint) + + it('should prune snapshot if payload is too large', function (done) { + t.agent.on('debugger-input', ({ payload }) => { + assert.isBelow(Buffer.byteLength(JSON.stringify(payload)), 1024 * 1024) // 1MB + assert.deepEqual(payload['debugger.snapshot'].captures, { + lines: { + [line]: { + locals: { + notCapturedReason: 'Snapshot was too large', + size: 6 + } + } + } + }) + done() + }) + + t.agent.addRemoteConfig(t.generateRemoteConfig({ + captureSnapshot: true, + capture: { + // ensure we get a large snapshot + maxCollectionSize: Number.MAX_SAFE_INTEGER, + maxFieldCount: Number.MAX_SAFE_INTEGER, + maxLength: Number.MAX_SAFE_INTEGER + } + })) + }) + }) + }) +}) diff --git a/integration-tests/debugger/snapshot.spec.js b/integration-tests/debugger/snapshot.spec.js new file mode 100644 index 00000000000..94ef323f6a7 --- /dev/null +++ b/integration-tests/debugger/snapshot.spec.js @@ -0,0 +1,239 @@ +'use strict' + +const { assert } = require('chai') +const { setup } = require('./utils') + +describe('Dynamic Instrumentation', function () { + const t = setup() + + describe('input messages', function () { + describe('with snapshot', function () { + beforeEach(t.triggerBreakpoint) + + it('should capture a snapshot', function (done) { + t.agent.on('debugger-input', ({ payload: { 'debugger.snapshot': { captures } } }) => { + assert.deepEqual(Object.keys(captures), ['lines']) + assert.deepEqual(Object.keys(captures.lines), [String(t.breakpoint.line)]) + + const { locals } = captures.lines[t.breakpoint.line] + const { request, fastify, getSomeData } = locals + delete locals.request + delete locals.fastify + delete locals.getSomeData + + // from block scope + assert.deepEqual(locals, { + nil: { type: 'null', isNull: true }, + undef: { type: 'undefined' }, + bool: { type: 'boolean', value: 'true' }, + num: { type: 'number', value: '42' }, + bigint: { type: 'bigint', value: '42' }, + str: { type: 'string', value: 'foo' }, + lstr: { + type: 'string', + // eslint-disable-next-line max-len + value: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor i', + truncated: true, + size: 445 + }, + sym: { type: 'symbol', value: 'Symbol(foo)' }, + regex: { type: 'RegExp', value: '/bar/i' }, + arr: { + type: 'Array', + elements: [ + { type: 'number', value: '1' }, + { type: 'number', value: '2' }, + { type: 'number', value: '3' }, + { type: 'number', value: '4' }, + { type: 'number', value: '5' } + ] + }, + obj: { + type: 'Object', + fields: { + foo: { + type: 'Object', + fields: { + baz: { type: 'number', value: '42' }, + nil: { type: 'null', isNull: true }, + undef: { type: 'undefined' }, + deep: { + type: 'Object', + fields: { nested: { type: 'Object', notCapturedReason: 'depth' } } + } + } + }, + bar: { type: 'boolean', value: 'true' } + } + }, + emptyObj: { type: 'Object', fields: {} }, + fn: { + type: 'Function', + fields: { + length: { type: 'number', value: '0' }, + name: { type: 'string', value: 'fn' } + } + }, + p: { + type: 'Promise', + fields: { + '[[PromiseState]]': { type: 'string', value: 'fulfilled' }, + '[[PromiseResult]]': { type: 'undefined' } + } + } + }) + + // from local scope + // There's no reason to test the `request` object 100%, instead just check its fingerprint + assert.deepEqual(Object.keys(request), ['type', 'fields']) + assert.equal(request.type, 'Request') + assert.deepEqual(request.fields.id, { type: 'string', value: 'req-1' }) + assert.deepEqual(request.fields.params, { + type: 'NullObject', fields: { name: { type: 'string', value: 'foo' } } + }) + assert.deepEqual(request.fields.query, { type: 'Object', fields: {} }) + assert.deepEqual(request.fields.body, { type: 'undefined' }) + + // from closure scope + // There's no reason to test the `fastify` object 100%, instead just check its fingerprint + assert.equal(fastify.type, 'Object') + assert.typeOf(fastify.fields, 'Object') + + assert.deepEqual(getSomeData, { + type: 'Function', + fields: { + length: { type: 'number', value: '0' }, + name: { type: 'string', value: 'getSomeData' } + } + }) + + done() + }) + + t.agent.addRemoteConfig(t.generateRemoteConfig({ captureSnapshot: true })) + }) + + it('should respect maxReferenceDepth', function (done) { + t.agent.on('debugger-input', ({ payload: { 'debugger.snapshot': { captures } } }) => { + const { locals } = captures.lines[t.breakpoint.line] + delete locals.request + delete locals.fastify + delete locals.getSomeData + + assert.deepEqual(locals, { + nil: { type: 'null', isNull: true }, + undef: { type: 'undefined' }, + bool: { type: 'boolean', value: 'true' }, + num: { type: 'number', value: '42' }, + bigint: { type: 'bigint', value: '42' }, + str: { type: 'string', value: 'foo' }, + lstr: { + type: 'string', + // eslint-disable-next-line max-len + value: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor i', + truncated: true, + size: 445 + }, + sym: { type: 'symbol', value: 'Symbol(foo)' }, + regex: { type: 'RegExp', value: '/bar/i' }, + arr: { type: 'Array', notCapturedReason: 'depth' }, + obj: { type: 'Object', notCapturedReason: 'depth' }, + emptyObj: { type: 'Object', notCapturedReason: 'depth' }, + fn: { type: 'Function', notCapturedReason: 'depth' }, + p: { type: 'Promise', notCapturedReason: 'depth' } + }) + + done() + }) + + t.agent.addRemoteConfig(t.generateRemoteConfig({ captureSnapshot: true, capture: { maxReferenceDepth: 0 } })) + }) + + it('should respect maxLength', function (done) { + t.agent.on('debugger-input', ({ payload: { 'debugger.snapshot': { captures } } }) => { + const { locals } = captures.lines[t.breakpoint.line] + + assert.deepEqual(locals.lstr, { + type: 'string', + value: 'Lorem ipsu', + truncated: true, + size: 445 + }) + + done() + }) + + t.agent.addRemoteConfig(t.generateRemoteConfig({ captureSnapshot: true, capture: { maxLength: 10 } })) + }) + + it('should respect maxCollectionSize', function (done) { + t.agent.on('debugger-input', ({ payload: { 'debugger.snapshot': { captures } } }) => { + const { locals } = captures.lines[t.breakpoint.line] + + assert.deepEqual(locals.arr, { + type: 'Array', + elements: [ + { type: 'number', value: '1' }, + { type: 'number', value: '2' }, + { type: 'number', value: '3' } + ], + notCapturedReason: 'collectionSize', + size: 5 + }) + + done() + }) + + t.agent.addRemoteConfig(t.generateRemoteConfig({ captureSnapshot: true, capture: { maxCollectionSize: 3 } })) + }) + + it('should respect maxFieldCount', (done) => { + const maxFieldCount = 3 + + function assertMaxFieldCount (prop) { + if ('fields' in prop) { + if (prop.notCapturedReason === 'fieldCount') { + assert.strictEqual(Object.keys(prop.fields).length, maxFieldCount) + assert.isAbove(prop.size, maxFieldCount) + } else { + assert.isBelow(Object.keys(prop.fields).length, maxFieldCount) + } + } + + for (const value of Object.values(prop.fields || prop.elements || prop.entries || {})) { + assertMaxFieldCount(value) + } + } + + t.agent.on('debugger-input', ({ payload: { 'debugger.snapshot': { captures } } }) => { + const { locals } = captures.lines[t.breakpoint.line] + + assert.deepEqual(Object.keys(locals), [ + // Up to 3 properties from the local scope + 'request', 'nil', 'undef', + // Up to 3 properties from the closure scope + 'fastify', 'getSomeData' + ]) + + assert.strictEqual(locals.request.type, 'Request') + assert.strictEqual(Object.keys(locals.request.fields).length, maxFieldCount) + assert.strictEqual(locals.request.notCapturedReason, 'fieldCount') + assert.isAbove(locals.request.size, maxFieldCount) + + assert.strictEqual(locals.fastify.type, 'Object') + assert.strictEqual(Object.keys(locals.fastify.fields).length, maxFieldCount) + assert.strictEqual(locals.fastify.notCapturedReason, 'fieldCount') + assert.isAbove(locals.fastify.size, maxFieldCount) + + for (const value of Object.values(locals)) { + assertMaxFieldCount(value) + } + + done() + }) + + t.agent.addRemoteConfig(t.generateRemoteConfig({ captureSnapshot: true, capture: { maxFieldCount } })) + }) + }) + }) +}) diff --git a/integration-tests/debugger/target-app/basic.js b/integration-tests/debugger/target-app/basic.js new file mode 100644 index 00000000000..f8330012278 --- /dev/null +++ b/integration-tests/debugger/target-app/basic.js @@ -0,0 +1,18 @@ +'use strict' + +require('dd-trace/init') +const Fastify = require('fastify') + +const fastify = Fastify() + +fastify.get('/:name', function handler (request) { + return { hello: request.params.name } // BREAKPOINT +}) + +fastify.listen({ port: process.env.APP_PORT }, (err) => { + if (err) { + fastify.log.error(err) + process.exit(1) + } + process.send({ port: process.env.APP_PORT }) +}) diff --git a/integration-tests/debugger/target-app/snapshot-pruning.js b/integration-tests/debugger/target-app/snapshot-pruning.js new file mode 100644 index 00000000000..58752006192 --- /dev/null +++ b/integration-tests/debugger/target-app/snapshot-pruning.js @@ -0,0 +1,41 @@ +'use strict' + +require('dd-trace/init') + +const { randomBytes } = require('crypto') +const Fastify = require('fastify') + +const fastify = Fastify() + +const TARGET_SIZE = 1024 * 1024 // 1MB +const LARGE_STRING = randomBytes(1024).toString('hex') + +fastify.get('/:name', function handler (request) { + // eslint-disable-next-line no-unused-vars + const obj = generateObjectWithJSONSizeLargerThan1MB() + + return { hello: request.params.name } // BREAKPOINT +}) + +fastify.listen({ port: process.env.APP_PORT }, (err) => { + if (err) { + fastify.log.error(err) + process.exit(1) + } + process.send({ port: process.env.APP_PORT }) +}) + +function generateObjectWithJSONSizeLargerThan1MB () { + const obj = {} + let i = 0 + + while (++i) { + if (i % 100 === 0) { + const size = JSON.stringify(obj).length + if (size > TARGET_SIZE) break + } + obj[i] = LARGE_STRING + } + + return obj +} diff --git a/integration-tests/debugger/target-app/index.js b/integration-tests/debugger/target-app/snapshot.js similarity index 91% rename from integration-tests/debugger/target-app/index.js rename to integration-tests/debugger/target-app/snapshot.js index dd7f5e6328a..a7b1810c10b 100644 --- a/integration-tests/debugger/target-app/index.js +++ b/integration-tests/debugger/target-app/snapshot.js @@ -11,11 +11,9 @@ const fastify = Fastify() fastify.get('/:name', function handler (request) { // eslint-disable-next-line no-unused-vars const { nil, undef, bool, num, bigint, str, lstr, sym, regex, arr, obj, emptyObj, fn, p } = getSomeData() - return { hello: request.params.name } + return { hello: request.params.name } // BREAKPOINT }) -// WARNING: Breakpoints present above this line - Any changes to the lines above might influence tests! - fastify.listen({ port: process.env.APP_PORT }, (err) => { if (err) { fastify.log.error(err) @@ -36,7 +34,7 @@ function getSomeData () { lstr: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.', sym: Symbol('foo'), regex: /bar/i, - arr: [1, 2, 3], + arr: [1, 2, 3, 4, 5], obj: { foo: { baz: 42, diff --git a/integration-tests/debugger/utils.js b/integration-tests/debugger/utils.js new file mode 100644 index 00000000000..c5760a0e9d4 --- /dev/null +++ b/integration-tests/debugger/utils.js @@ -0,0 +1,124 @@ +'use strict' + +const { basename, join } = require('path') +const { readFileSync } = require('fs') +const { randomUUID } = require('crypto') + +const getPort = require('get-port') +const Axios = require('axios') + +const { createSandbox, FakeAgent, spawnProc } = require('../helpers') + +const pollInterval = 1 + +module.exports = { + pollInterval, + setup, + getBreakpointInfo +} + +function setup () { + let sandbox, cwd, appPort, proc + const breakpoint = getBreakpointInfo(1) // `1` to disregard the `setup` function + const t = { + breakpoint, + axios: null, + appFile: null, + agent: null, + rcConfig: null, + triggerBreakpoint, + generateRemoteConfig, + generateProbeConfig + } + + function triggerBreakpoint () { + // Trigger the breakpoint once probe is successfully installed + t.agent.on('debugger-diagnostics', ({ payload }) => { + if (payload.debugger.diagnostics.status === 'INSTALLED') { + t.axios.get('/foo') + } + }) + } + + function generateRemoteConfig (overrides = {}) { + overrides.id = overrides.id || randomUUID() + return { + product: 'LIVE_DEBUGGING', + id: `logProbe_${overrides.id}`, + config: generateProbeConfig(overrides) + } + } + + function generateProbeConfig (overrides = {}) { + overrides.capture = { maxReferenceDepth: 3, ...overrides.capture } + overrides.sampling = { snapshotsPerSecond: 5000, ...overrides.sampling } + return { + id: randomUUID(), + version: 0, + type: 'LOG_PROBE', + language: 'javascript', + where: { sourceFile: breakpoint.file, lines: [String(breakpoint.line)] }, + tags: [], + template: 'Hello World!', + segments: [{ str: 'Hello World!' }], + captureSnapshot: false, + evaluateAt: 'EXIT', + ...overrides + } + } + + before(async function () { + sandbox = await createSandbox(['fastify']) + cwd = sandbox.folder + t.appFile = join(cwd, ...breakpoint.file.split('/')) + }) + + after(async function () { + await sandbox.remove() + }) + + beforeEach(async function () { + t.rcConfig = generateRemoteConfig(breakpoint) + appPort = await getPort() + t.agent = await new FakeAgent().start() + proc = await spawnProc(t.appFile, { + cwd, + env: { + APP_PORT: appPort, + DD_DYNAMIC_INSTRUMENTATION_ENABLED: true, + DD_TRACE_AGENT_PORT: t.agent.port, + DD_TRACE_DEBUG: process.env.DD_TRACE_DEBUG, // inherit to make debugging the sandbox easier + DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS: pollInterval + } + }) + t.axios = Axios.create({ + baseURL: `http://localhost:${appPort}` + }) + }) + + afterEach(async function () { + proc.kill() + await t.agent.stop() + }) + + return t +} + +function getBreakpointInfo (stackIndex = 0) { + // First, get the filename of file that called this function + const testFile = new Error().stack + .split('\n')[stackIndex + 2] // +2 to skip this function + the first line, which is the error message + .split(' (')[1] + .slice(0, -1) + .split(':')[0] + + // Then, find the corresponding file in which the breakpoint exists + const filename = basename(testFile).replace('.spec', '') + + // Finally, find the line number of the breakpoint + const line = readFileSync(join(__dirname, 'target-app', filename), 'utf8') + .split('\n') + .findIndex(line => line.includes('// BREAKPOINT')) + 1 + + return { file: `debugger/target-app/${filename}`, line } +} diff --git a/integration-tests/esbuild/basic-test.js b/integration-tests/esbuild/basic-test.js index dc41b4efa53..5e95234eddf 100755 --- a/integration-tests/esbuild/basic-test.js +++ b/integration-tests/esbuild/basic-test.js @@ -6,6 +6,7 @@ const assert = require('assert') const express = require('express') const http = require('http') require('knex') // has dead code paths for multiple instrumented packages +require('@apollo/server') const app = express() const PORT = 31415 diff --git a/integration-tests/esbuild/package.json b/integration-tests/esbuild/package.json index cc027c59bcf..63e8caa8372 100644 --- a/integration-tests/esbuild/package.json +++ b/integration-tests/esbuild/package.json @@ -18,6 +18,7 @@ "author": "Thomas Hunter II ", "license": "ISC", "dependencies": { + "@apollo/server": "^4.11.0", "aws-sdk": "^2.1446.0", "axios": "^1.6.7", "esbuild": "0.16.12", diff --git a/integration-tests/helpers/fake-agent.js b/integration-tests/helpers/fake-agent.js index 70aff2ecfa8..f1054720d92 100644 --- a/integration-tests/helpers/fake-agent.js +++ b/integration-tests/helpers/fake-agent.js @@ -188,6 +188,46 @@ module.exports = class FakeAgent extends EventEmitter { return resultPromise } + + assertLlmObsPayloadReceived (fn, timeout, expectedMessageCount = 1, resolveAtFirstSuccess) { + timeout = timeout || 30000 + let resultResolve + let resultReject + let msgCount = 0 + const errors = [] + + const timeoutObj = setTimeout(() => { + const errorsMsg = errors.length === 0 ? '' : `, additionally:\n${errors.map(e => e.stack).join('\n')}\n===\n` + resultReject(new Error(`timeout${errorsMsg}`, { cause: { errors } })) + }, timeout) + + const resultPromise = new Promise((resolve, reject) => { + resultResolve = () => { + clearTimeout(timeoutObj) + resolve() + } + resultReject = (e) => { + clearTimeout(timeoutObj) + reject(e) + } + }) + + const messageHandler = msg => { + try { + msgCount += 1 + fn(msg) + if (resolveAtFirstSuccess || msgCount === expectedMessageCount) { + resultResolve() + this.removeListener('llmobs', messageHandler) + } + } catch (e) { + errors.push(e) + } + } + this.on('llmobs', messageHandler) + + return resultPromise + } } function buildExpressServer (agent) { @@ -315,6 +355,14 @@ function buildExpressServer (agent) { }) }) + app.post('/evp_proxy/v2/api/v2/llmobs', (req, res) => { + res.status(200).send() + agent.emit('llmobs', { + headers: req.headers, + payload: req.body + }) + }) + return app } diff --git a/integration-tests/mocha/mocha.spec.js b/integration-tests/mocha/mocha.spec.js index dac0a9e3bff..3fa11871204 100644 --- a/integration-tests/mocha/mocha.spec.js +++ b/integration-tests/mocha/mocha.spec.js @@ -1875,7 +1875,7 @@ describe('mocha CommonJS', function () { }) }) - context('flaky test retries', () => { + context('auto test retries', () => { it('retries failed tests automatically', (done) => { receiver.setSettings({ itr_enabled: false, @@ -1911,6 +1911,10 @@ describe('mocha CommonJS', function () { const failedAttempts = tests.filter(test => test.meta[TEST_STATUS] === 'fail') assert.equal(failedAttempts.length, 2) + failedAttempts.forEach((failedTest, index) => { + assert.include(failedTest.meta[ERROR_MESSAGE], `expected ${index + 1} to equal 3`) + }) + // The first attempt is not marked as a retry const retriedFailure = failedAttempts.filter(test => test.meta[TEST_IS_RETRY] === 'true') assert.equal(retriedFailure.length, 1) diff --git a/integration-tests/opentelemetry.spec.js b/integration-tests/opentelemetry.spec.js index 73adf812360..ee307568cb4 100644 --- a/integration-tests/opentelemetry.spec.js +++ b/integration-tests/opentelemetry.spec.js @@ -348,6 +348,52 @@ describe('opentelemetry', () => { }, true) }) + it('should capture auto-instrumentation telemetry', async () => { + const SERVER_PORT = 6666 + proc = fork(join(cwd, 'opentelemetry/auto-instrumentation.js'), { + cwd, + env: { + DD_TRACE_AGENT_PORT: agent.port, + DD_TRACE_OTEL_ENABLED: 1, + SERVER_PORT, + DD_TRACE_DISABLED_INSTRUMENTATIONS: 'http,dns,express,net', + DD_TELEMETRY_HEARTBEAT_INTERVAL: 1 + } + }) + await new Promise(resolve => setTimeout(resolve, 1000)) // Adjust the delay as necessary + await axios.get(`http://localhost:${SERVER_PORT}/first-endpoint`) + + return check(agent, proc, 10000, ({ payload }) => { + assert.strictEqual(payload.request_type, 'generate-metrics') + + const metrics = payload.payload + assert.strictEqual(metrics.namespace, 'tracers') + + const spanCreated = metrics.series.find(({ metric }) => metric === 'spans_created') + const spanFinished = metrics.series.find(({ metric }) => metric === 'spans_finished') + + // Validate common fields between start and finish + for (const series of [spanCreated, spanFinished]) { + assert.ok(series) + + assert.strictEqual(series.points.length, 1) + assert.strictEqual(series.points[0].length, 2) + + const [ts, value] = series.points[0] + assert.ok(nearNow(ts, Date.now() / 1e3)) + assert.strictEqual(value, 9) + + assert.strictEqual(series.type, 'count') + assert.strictEqual(series.common, true) + assert.deepStrictEqual(series.tags, [ + 'integration_name:otel.library', + 'otel_enabled:true', + `version:${process.version}` + ]) + } + }, true) + }) + it('should work within existing datadog-traced http request', async () => { proc = fork(join(cwd, 'opentelemetry/server.js'), { cwd, diff --git a/integration-tests/standalone-asm.spec.js b/integration-tests/standalone-asm.spec.js index d57a96f738e..4e57b25bad6 100644 --- a/integration-tests/standalone-asm.spec.js +++ b/integration-tests/standalone-asm.spec.js @@ -10,6 +10,7 @@ const { curlAndAssertMessage, curl } = require('./helpers') +const { USER_KEEP, AUTO_REJECT, AUTO_KEEP } = require('../ext/priority') describe('Standalone ASM', () => { let sandbox, cwd, startupTestFile, agent, proc, env @@ -43,22 +44,18 @@ describe('Standalone ASM', () => { await agent.stop() }) - function assertKeep (payload, manual = true) { + function assertKeep (payload) { const { meta, metrics } = payload - if (manual) { - assert.propertyVal(meta, 'manual.keep', 'true') - } else { - assert.notProperty(meta, 'manual.keep') - } + assert.propertyVal(meta, '_dd.p.appsec', '1') - assert.propertyVal(metrics, '_sampling_priority_v1', 2) + assert.propertyVal(metrics, '_sampling_priority_v1', USER_KEEP) assert.propertyVal(metrics, '_dd.apm.enabled', 0) } function assertDrop (payload) { const { metrics } = payload - assert.propertyVal(metrics, '_sampling_priority_v1', 0) + assert.propertyVal(metrics, '_sampling_priority_v1', AUTO_REJECT) assert.propertyVal(metrics, '_dd.apm.enabled', 0) assert.notProperty(metrics, '_dd.p.appsec') } @@ -103,7 +100,7 @@ describe('Standalone ASM', () => { assert.notProperty(meta, 'manual.keep') assert.notProperty(meta, '_dd.p.appsec') - assert.propertyVal(metrics, '_sampling_priority_v1', 1) + assert.propertyVal(metrics, '_sampling_priority_v1', AUTO_KEEP) assert.propertyVal(metrics, '_dd.apm.enabled', 0) assertDrop(payload[2][0]) @@ -213,7 +210,7 @@ describe('Standalone ASM', () => { const innerReq = payload.find(p => p[0].resource === 'GET /down') assert.notStrictEqual(innerReq, undefined) - assertKeep(innerReq[0], false) + assertKeep(innerReq[0]) }, undefined, undefined, true) }) diff --git a/package.json b/package.json index 52982e0cce6..e9ad435b5f8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "dd-trace", - "version": "4.48.0", + "version": "4.49.0", "description": "Datadog APM tracing client for JavaScript", "main": "index.js", "typings": "index.d.ts", @@ -12,8 +12,8 @@ "bench:e2e:ci-visibility": "node benchmark/e2e-ci/benchmark-run.js", "type:doc": "cd docs && yarn && yarn build", "type:test": "cd docs && yarn && yarn test", - "lint": "node scripts/check_licenses.js && eslint . && yarn audit --groups dependencies", - "lint-fix": "node scripts/check_licenses.js && eslint . --fix && yarn audit --groups dependencies", + "lint": "node scripts/check_licenses.js && eslint . && yarn audit", + "lint-fix": "node scripts/check_licenses.js && eslint . --fix && yarn audit", "services": "node ./scripts/install_plugin_modules && node packages/dd-trace/test/setup/services", "test": "SERVICES=* yarn services && mocha --expose-gc 'packages/dd-trace/test/setup/node.js' 'packages/*/test/**/*.spec.js'", "test:appsec": "mocha -r \"packages/dd-trace/test/setup/mocha.js\" --exclude \"packages/dd-trace/test/appsec/**/*.plugin.spec.js\" \"packages/dd-trace/test/appsec/**/*.spec.js\"", @@ -30,6 +30,10 @@ "test:core:ci": "npm run test:core -- --coverage --nyc-arg=--include=\"packages/datadog-core/src/**/*.js\"", "test:lambda": "mocha -r \"packages/dd-trace/test/setup/mocha.js\" \"packages/dd-trace/test/lambda/**/*.spec.js\"", "test:lambda:ci": "nyc --no-clean --include \"packages/dd-trace/src/lambda/**/*.js\" -- npm run test:lambda", + "test:llmobs:sdk": "mocha -r \"packages/dd-trace/test/setup/mocha.js\" --exclude \"packages/dd-trace/test/llmobs/plugins/**/*.spec.js\" \"packages/dd-trace/test/llmobs/**/*.spec.js\" ", + "test:llmobs:sdk:ci": "nyc --no-clean --include \"packages/dd-trace/src/llmobs/**/*.js\" -- npm run test:llmobs:sdk", + "test:llmobs:plugins": "mocha -r \"packages/dd-trace/test/setup/mocha.js\" \"packages/dd-trace/test/llmobs/plugins/**/*.spec.js\"", + "test:llmobs:plugins:ci": "yarn services && nyc --no-clean --include \"packages/dd-trace/src/llmobs/**/*.js\" -- npm run test:llmobs:plugins", "test:plugins": "mocha -r \"packages/dd-trace/test/setup/mocha.js\" \"packages/datadog-instrumentations/test/@($(echo $PLUGINS)).spec.js\" \"packages/datadog-plugin-@($(echo $PLUGINS))/test/**/*.spec.js\"", "test:plugins:ci": "yarn services && nyc --no-clean --include \"packages/datadog-instrumentations/src/@($(echo $PLUGINS)).js\" --include \"packages/datadog-instrumentations/src/@($(echo $PLUGINS))/**/*.js\" --include \"packages/datadog-plugin-@($(echo $PLUGINS))/src/**/*.js\" -- npm run test:plugins", "test:plugins:upstream": "node ./packages/dd-trace/test/plugins/suite.js", @@ -76,11 +80,11 @@ "node": ">=16" }, "dependencies": { - "@datadog/native-appsec": "8.1.1", + "@datadog/native-appsec": "8.2.1", "@datadog/native-iast-rewriter": "2.5.0", - "@datadog/native-iast-taint-tracking": "3.1.0", - "@datadog/native-metrics": "^2.0.0", - "@datadog/pprof": "5.3.0", + "@datadog/native-iast-taint-tracking": "3.2.0", + "@datadog/native-metrics": "^3.0.1", + "@datadog/pprof": "5.4.1", "@datadog/sketches-js": "^2.1.0", "@opentelemetry/api": ">=1.0.0 <1.9.0", "@opentelemetry/core": "^1.14.0", @@ -108,6 +112,7 @@ "tlhunter-sorted-set": "^0.1.0" }, "devDependencies": { + "@apollo/server": "^4.11.0", "@types/node": ">=16", "autocannon": "^4.5.2", "aws-sdk": "^2.1446.0", diff --git a/packages/datadog-code-origin/index.js b/packages/datadog-code-origin/index.js index 530dd3cc8ae..278aac265ab 100644 --- a/packages/datadog-code-origin/index.js +++ b/packages/datadog-code-origin/index.js @@ -5,15 +5,15 @@ const { getUserLandFrames } = require('../dd-trace/src/plugins/util/stacktrace') const limit = Number(process.env._DD_CODE_ORIGIN_MAX_USER_FRAMES) || 8 module.exports = { - entryTag, - exitTag + entryTags, + exitTags } -function entryTag (topOfStackFunc) { +function entryTags (topOfStackFunc) { return tag('entry', topOfStackFunc) } -function exitTag (topOfStackFunc) { +function exitTags (topOfStackFunc) { return tag('exit', topOfStackFunc) } diff --git a/packages/datadog-core/src/utils/src/parse-tags.js b/packages/datadog-core/src/utils/src/parse-tags.js new file mode 100644 index 00000000000..4142e770e4e --- /dev/null +++ b/packages/datadog-core/src/utils/src/parse-tags.js @@ -0,0 +1,33 @@ +'use strict' + +const digitRegex = /^\d+$/ + +/** + * Converts a flat object of tags into a nested object. For example: + * { 'a.b.c': 'value' } -> { a: { b: { c: 'value' } } } + * Also supports array-keys. For example: + * { 'a.0.b': 'value' } -> { a: [{ b: 'value' }] } + * + * @param {Object} tags - Key/value pairs of tags + * @returns Object - Parsed tags + */ +module.exports = tags => { + const parsedTags = {} + for (const [tag, value] of Object.entries(tags)) { + const keys = tag.split('.') + let current = parsedTags + let depth = 0 + for (const key of keys) { + if (!current[key]) { + if (depth === keys.length - 1) { + current[key] = value + break + } + current[key] = keys[depth + 1]?.match(digitRegex) ? [] : {} + } + current = current[key] + depth++ + } + } + return parsedTags +} diff --git a/packages/datadog-core/test/utils/src/parse-tags.spec.js b/packages/datadog-core/test/utils/src/parse-tags.spec.js new file mode 100644 index 00000000000..ded1bb5974f --- /dev/null +++ b/packages/datadog-core/test/utils/src/parse-tags.spec.js @@ -0,0 +1,23 @@ +'use strict' + +require('../../../../dd-trace/test/setup/tap') + +const parseTags = require('../../../src/utils/src/parse-tags') + +describe('parseTags', () => { + it('should parse tags to object', () => { + const obj = { + 'a.0.a': 'foo', + 'a.0.b': 'bar', + 'a.1.a': 'baz' + } + + expect(parseTags(obj)).to.deep.equal({ + a: [{ a: 'foo', b: 'bar' }, { a: 'baz' }] + }) + }) + + it('should work with empty object', () => { + expect(parseTags({})).to.deep.equal({}) + }) +}) diff --git a/packages/datadog-esbuild/index.js b/packages/datadog-esbuild/index.js index ce263799023..4a69cf32ebc 100644 --- a/packages/datadog-esbuild/index.js +++ b/packages/datadog-esbuild/index.js @@ -96,7 +96,9 @@ module.exports.setup = function (build) { let pathToPackageJson try { - pathToPackageJson = require.resolve(`${extracted.pkg}/package.json`, { paths: [args.resolveDir] }) + // we can't use require.resolve('pkg/package.json') as ESM modules don't make the file available + pathToPackageJson = require.resolve(`${extracted.pkg}`, { paths: [args.resolveDir] }) + pathToPackageJson = extractPackageAndModulePath(pathToPackageJson).pkgJson } catch (err) { if (err.code === 'MODULE_NOT_FOUND') { if (!internal) { @@ -111,7 +113,7 @@ module.exports.setup = function (build) { } } - const packageJson = require(pathToPackageJson) + const packageJson = JSON.parse(fs.readFileSync(pathToPackageJson).toString()) if (DEBUG) console.log(`RESOLVE: ${args.path}@${packageJson.version}`) diff --git a/packages/datadog-instrumentations/src/helpers/hooks.js b/packages/datadog-instrumentations/src/helpers/hooks.js index 62d45e37008..dbcd55a0b86 100644 --- a/packages/datadog-instrumentations/src/helpers/hooks.js +++ b/packages/datadog-instrumentations/src/helpers/hooks.js @@ -79,6 +79,7 @@ module.exports = { 'mongodb-core': () => require('../mongodb-core'), mongoose: () => require('../mongoose'), mquery: () => require('../mquery'), + multer: () => require('../multer'), mysql: () => require('../mysql'), mysql2: () => require('../mysql2'), net: () => require('../net'), diff --git a/packages/datadog-instrumentations/src/helpers/register.js b/packages/datadog-instrumentations/src/helpers/register.js index 2ef9d199f99..4b4185423c0 100644 --- a/packages/datadog-instrumentations/src/helpers/register.js +++ b/packages/datadog-instrumentations/src/helpers/register.js @@ -22,6 +22,15 @@ const disabledInstrumentations = new Set( DD_TRACE_DISABLED_INSTRUMENTATIONS ? DD_TRACE_DISABLED_INSTRUMENTATIONS.split(',') : [] ) +// Check for DD_TRACE__ENABLED environment variables +for (const [key, value] of Object.entries(process.env)) { + const match = key.match(/^DD_TRACE_(.+)_ENABLED$/) + if (match && (value.toLowerCase() === 'false' || value === '0')) { + const integration = match[1].toLowerCase() + disabledInstrumentations.add(integration) + } +} + const loadChannel = channel('dd-trace:instrumentation:load') // Globals diff --git a/packages/datadog-instrumentations/src/kafkajs.js b/packages/datadog-instrumentations/src/kafkajs.js index 395c69de057..e75c03e7e64 100644 --- a/packages/datadog-instrumentations/src/kafkajs.js +++ b/packages/datadog-instrumentations/src/kafkajs.js @@ -52,45 +52,59 @@ addHook({ name: 'kafkajs', file: 'src/index.js', versions: ['>=1.4'] }, (BaseKaf const send = producer.send const bootstrapServers = this._brokers - producer.send = function () { - const innerAsyncResource = new AsyncResource('bound-anonymous-fn') + const kafkaClusterIdPromise = getKafkaClusterId(this) - return innerAsyncResource.runInAsyncScope(() => { - if (!producerStartCh.hasSubscribers) { - return send.apply(this, arguments) - } + producer.send = function () { + const wrappedSend = (clusterId) => { + const innerAsyncResource = new AsyncResource('bound-anonymous-fn') - try { - const { topic, messages = [] } = arguments[0] - for (const message of messages) { - if (message !== null && typeof message === 'object') { - message.headers = message.headers || {} - } + return innerAsyncResource.runInAsyncScope(() => { + if (!producerStartCh.hasSubscribers) { + return send.apply(this, arguments) } - producerStartCh.publish({ topic, messages, bootstrapServers }) - - const result = send.apply(this, arguments) - - result.then( - innerAsyncResource.bind(res => { - producerFinishCh.publish(undefined) - producerCommitCh.publish(res) - }), - innerAsyncResource.bind(err => { - if (err) { - producerErrorCh.publish(err) + + try { + const { topic, messages = [] } = arguments[0] + for (const message of messages) { + if (message !== null && typeof message === 'object') { + message.headers = message.headers || {} } - producerFinishCh.publish(undefined) - }) - ) + } + producerStartCh.publish({ topic, messages, bootstrapServers, clusterId }) - return result - } catch (e) { - producerErrorCh.publish(e) - producerFinishCh.publish(undefined) - throw e - } - }) + const result = send.apply(this, arguments) + + result.then( + innerAsyncResource.bind(res => { + producerFinishCh.publish(undefined) + producerCommitCh.publish(res) + }), + innerAsyncResource.bind(err => { + if (err) { + producerErrorCh.publish(err) + } + producerFinishCh.publish(undefined) + }) + ) + + return result + } catch (e) { + producerErrorCh.publish(e) + producerFinishCh.publish(undefined) + throw e + } + }) + } + + if (!isPromise(kafkaClusterIdPromise)) { + // promise is already resolved + return wrappedSend(kafkaClusterIdPromise) + } else { + // promise is not resolved + return kafkaClusterIdPromise.then((clusterId) => { + return wrappedSend(clusterId) + }) + } } return producer }) @@ -100,15 +114,17 @@ addHook({ name: 'kafkajs', file: 'src/index.js', versions: ['>=1.4'] }, (BaseKaf return createConsumer.apply(this, arguments) } - const eachMessageExtractor = (args) => { + const kafkaClusterIdPromise = getKafkaClusterId(this) + + const eachMessageExtractor = (args, clusterId) => { const { topic, partition, message } = args[0] - return { topic, partition, message, groupId } + return { topic, partition, message, groupId, clusterId } } - const eachBatchExtractor = (args) => { + const eachBatchExtractor = (args, clusterId) => { const { batch } = args[0] const { topic, partition, messages } = batch - return { topic, partition, messages, groupId } + return { topic, partition, messages, groupId, clusterId } } const consumer = createConsumer.apply(this, arguments) @@ -116,43 +132,53 @@ addHook({ name: 'kafkajs', file: 'src/index.js', versions: ['>=1.4'] }, (BaseKaf consumer.on(consumer.events.COMMIT_OFFSETS, commitsFromEvent) const run = consumer.run - const groupId = arguments[0].groupId + consumer.run = function ({ eachMessage, eachBatch, ...runArgs }) { - eachMessage = wrapFunction( - eachMessage, - consumerStartCh, - consumerFinishCh, - consumerErrorCh, - eachMessageExtractor - ) - - eachBatch = wrapFunction( - eachBatch, - batchConsumerStartCh, - batchConsumerFinishCh, - batchConsumerErrorCh, - eachBatchExtractor - ) - - return run({ - eachMessage, - eachBatch, - ...runArgs - }) + const wrapConsume = (clusterId) => { + return run({ + eachMessage: wrappedCallback( + eachMessage, + consumerStartCh, + consumerFinishCh, + consumerErrorCh, + eachMessageExtractor, + clusterId + ), + eachBatch: wrappedCallback( + eachBatch, + batchConsumerStartCh, + batchConsumerFinishCh, + batchConsumerErrorCh, + eachBatchExtractor, + clusterId + ), + ...runArgs + }) + } + + if (!isPromise(kafkaClusterIdPromise)) { + // promise is already resolved + return wrapConsume(kafkaClusterIdPromise) + } else { + // promise is not resolved + return kafkaClusterIdPromise.then((clusterId) => { + return wrapConsume(clusterId) + }) + } } - return consumer }) return Kafka }) -const wrapFunction = (fn, startCh, finishCh, errorCh, extractArgs) => { +const wrappedCallback = (fn, startCh, finishCh, errorCh, extractArgs, clusterId) => { return typeof fn === 'function' ? function (...args) { const innerAsyncResource = new AsyncResource('bound-anonymous-fn') return innerAsyncResource.runInAsyncScope(() => { - const extractedArgs = extractArgs(args) + const extractedArgs = extractArgs(args, clusterId) + startCh.publish(extractedArgs) try { const result = fn.apply(this, args) @@ -179,3 +205,37 @@ const wrapFunction = (fn, startCh, finishCh, errorCh, extractArgs) => { } : fn } + +const getKafkaClusterId = (kafka) => { + if (kafka._ddKafkaClusterId) { + return kafka._ddKafkaClusterId + } + + if (!kafka.admin) { + return null + } + + const admin = kafka.admin() + + if (!admin.describeCluster) { + return null + } + + return admin.connect() + .then(() => { + return admin.describeCluster() + }) + .then((clusterInfo) => { + const clusterId = clusterInfo?.clusterId + kafka._ddKafkaClusterId = clusterId + admin.disconnect() + return clusterId + }) + .catch((error) => { + throw error + }) +} + +function isPromise (obj) { + return !!obj && (typeof obj === 'object' || typeof obj === 'function') && typeof obj.then === 'function' +} diff --git a/packages/datadog-instrumentations/src/mocha/utils.js b/packages/datadog-instrumentations/src/mocha/utils.js index a4da0762039..2b51fd6e73b 100644 --- a/packages/datadog-instrumentations/src/mocha/utils.js +++ b/packages/datadog-instrumentations/src/mocha/utils.js @@ -280,12 +280,12 @@ function getOnFailHandler (isMain) { } function getOnTestRetryHandler () { - return function (test) { + return function (test, err) { const asyncResource = getTestAsyncResource(test) if (asyncResource) { const isFirstAttempt = test._currentRetry === 0 asyncResource.runInAsyncScope(() => { - testRetryCh.publish(isFirstAttempt) + testRetryCh.publish({ isFirstAttempt, err }) }) } const key = getTestToArKey(test) diff --git a/packages/datadog-instrumentations/src/multer.js b/packages/datadog-instrumentations/src/multer.js new file mode 100644 index 00000000000..90fae3a8297 --- /dev/null +++ b/packages/datadog-instrumentations/src/multer.js @@ -0,0 +1,37 @@ +'use strict' + +const shimmer = require('../../datadog-shimmer') +const { channel, addHook, AsyncResource } = require('./helpers/instrument') + +const multerReadCh = channel('datadog:multer:read:finish') + +function publishRequestBodyAndNext (req, res, next) { + return shimmer.wrapFunction(next, next => function () { + if (multerReadCh.hasSubscribers && req) { + const abortController = new AbortController() + const body = req.body + + multerReadCh.publish({ req, res, body, abortController }) + + if (abortController.signal.aborted) return + } + + return next.apply(this, arguments) + }) +} + +addHook({ + name: 'multer', + file: 'lib/make-middleware.js', + versions: ['^1.4.4-lts.1'] +}, makeMiddleware => { + return shimmer.wrapFunction(makeMiddleware, makeMiddleware => function () { + const middleware = makeMiddleware.apply(this, arguments) + + return shimmer.wrapFunction(middleware, middleware => function wrapMulterMiddleware (req, res, next) { + const nextResource = new AsyncResource('bound-anonymous-fn') + arguments[2] = nextResource.bind(publishRequestBodyAndNext(req, res, next)) + return middleware.apply(this, arguments) + }) + }) +}) diff --git a/packages/datadog-instrumentations/src/openai.js b/packages/datadog-instrumentations/src/openai.js index 940b5919d24..3528b1ecc13 100644 --- a/packages/datadog-instrumentations/src/openai.js +++ b/packages/datadog-instrumentations/src/openai.js @@ -3,8 +3,8 @@ const { addHook } = require('./helpers/instrument') const shimmer = require('../../datadog-shimmer') -const tracingChannel = require('dc-polyfill').tracingChannel -const ch = tracingChannel('apm:openai:request') +const dc = require('dc-polyfill') +const ch = dc.tracingChannel('apm:openai:request') const V4_PACKAGE_SHIMS = [ { diff --git a/packages/datadog-instrumentations/src/utils/src/extract-package-and-module-path.js b/packages/datadog-instrumentations/src/utils/src/extract-package-and-module-path.js index 176c3c618ff..7a48565e379 100644 --- a/packages/datadog-instrumentations/src/utils/src/extract-package-and-module-path.js +++ b/packages/datadog-instrumentations/src/utils/src/extract-package-and-module-path.js @@ -6,7 +6,7 @@ const NM = 'node_modules/' * For a given full path to a module, * return the package name it belongs to and the local path to the module * input: '/foo/node_modules/@co/stuff/foo/bar/baz.js' - * output: { pkg: '@co/stuff', path: 'foo/bar/baz.js' } + * output: { pkg: '@co/stuff', path: 'foo/bar/baz.js', pkgJson: '/foo/node_modules/@co/stuff/package.json' } */ module.exports = function extractPackageAndModulePath (fullPath) { const nm = fullPath.lastIndexOf(NM) @@ -17,17 +17,20 @@ module.exports = function extractPackageAndModulePath (fullPath) { const subPath = fullPath.substring(nm + NM.length) const firstSlash = subPath.indexOf('/') + const firstPath = fullPath.substring(fullPath[0], nm + NM.length) + if (subPath[0] === '@') { const secondSlash = subPath.substring(firstSlash + 1).indexOf('/') - return { pkg: subPath.substring(0, firstSlash + 1 + secondSlash), - path: subPath.substring(firstSlash + 1 + secondSlash + 1) + path: subPath.substring(firstSlash + 1 + secondSlash + 1), + pkgJson: firstPath + subPath.substring(0, firstSlash + 1 + secondSlash) + '/package.json' } } return { pkg: subPath.substring(0, firstSlash), - path: subPath.substring(firstSlash + 1) + path: subPath.substring(firstSlash + 1), + pkgJson: firstPath + subPath.substring(0, firstSlash) + '/package.json' } } diff --git a/packages/datadog-instrumentations/test/multer.spec.js b/packages/datadog-instrumentations/test/multer.spec.js new file mode 100644 index 00000000000..f7edcee6cd3 --- /dev/null +++ b/packages/datadog-instrumentations/test/multer.spec.js @@ -0,0 +1,108 @@ +'use strict' + +const dc = require('dc-polyfill') +const axios = require('axios') +const agent = require('../../dd-trace/test/plugins/agent') +const { storage } = require('../../datadog-core') + +withVersions('multer', 'multer', version => { + describe('multer parser instrumentation', () => { + const multerReadCh = dc.channel('datadog:multer:read:finish') + let port, server, middlewareProcessBodyStub, formData + + before(() => { + return agent.load(['http', 'express', 'multer'], { client: false }) + }) + + before((done) => { + const express = require('../../../versions/express').get() + const multer = require(`../../../versions/multer@${version}`).get() + const uploadToMemory = multer({ storage: multer.memoryStorage(), limits: { fileSize: 200000 } }) + + const app = express() + + app.post('/', uploadToMemory.single('file'), (req, res) => { + middlewareProcessBodyStub(req.body.key) + res.end('DONE') + }) + server = app.listen(0, () => { + port = server.address().port + done() + }) + }) + + beforeEach(async () => { + middlewareProcessBodyStub = sinon.stub() + + formData = new FormData() + formData.append('key', 'value') + }) + + after(() => { + server.close() + return agent.close({ ritmReset: false }) + }) + + it('should not abort the request by default', async () => { + const res = await axios.post(`http://localhost:${port}/`, formData) + + expect(middlewareProcessBodyStub).to.be.calledOnceWithExactly(formData.get('key')) + expect(res.data).to.be.equal('DONE') + }) + + it('should not abort the request with non blocker subscription', async () => { + function noop () {} + multerReadCh.subscribe(noop) + + try { + const res = await axios.post(`http://localhost:${port}/`, formData) + + expect(middlewareProcessBodyStub).to.be.calledOnceWithExactly(formData.get('key')) + expect(res.data).to.be.equal('DONE') + } finally { + multerReadCh.unsubscribe(noop) + } + }) + + it('should abort the request when abortController.abort() is called', async () => { + function blockRequest ({ res, abortController }) { + res.end('BLOCKED') + abortController.abort() + } + multerReadCh.subscribe(blockRequest) + + try { + const res = await axios.post(`http://localhost:${port}/`, formData) + + expect(middlewareProcessBodyStub).not.to.be.called + expect(res.data).to.be.equal('BLOCKED') + } finally { + multerReadCh.unsubscribe(blockRequest) + } + }) + + it('should not lose the http async context', async () => { + let store + let payload + + function handler (data) { + store = storage.getStore() + payload = data + } + multerReadCh.subscribe(handler) + + try { + const res = await axios.post(`http://localhost:${port}/`, formData) + + expect(store).to.have.property('req', payload.req) + expect(store).to.have.property('res', payload.res) + expect(store).to.have.property('span') + + expect(middlewareProcessBodyStub).to.be.calledOnceWithExactly(formData.get('key')) + expect(res.data).to.be.equal('DONE') + } finally { + multerReadCh.unsubscribe(handler) + } + }) + }) +}) diff --git a/packages/datadog-plugin-azure-functions/test/integration-test/fixtures/package.json b/packages/datadog-plugin-azure-functions/test/integration-test/fixtures/package.json index 07b0ac311ee..f17f97669ab 100644 --- a/packages/datadog-plugin-azure-functions/test/integration-test/fixtures/package.json +++ b/packages/datadog-plugin-azure-functions/test/integration-test/fixtures/package.json @@ -7,7 +7,7 @@ "start": "func start" }, "dependencies": { - "@azure/functions": "^4.0.0" + "@azure/functions": "^4.6.0" }, "devDependencies": { "azure-functions-core-tools": "^4.x" diff --git a/packages/datadog-plugin-azure-functions/test/integration-test/fixtures/yarn.lock b/packages/datadog-plugin-azure-functions/test/integration-test/fixtures/yarn.lock index 98c420c8953..bceddf8fcad 100644 --- a/packages/datadog-plugin-azure-functions/test/integration-test/fixtures/yarn.lock +++ b/packages/datadog-plugin-azure-functions/test/integration-test/fixtures/yarn.lock @@ -2,12 +2,12 @@ # yarn lockfile v1 -"@azure/functions@^4.0.0": - version "4.5.1" - resolved "https://registry.yarnpkg.com/@azure/functions/-/functions-4.5.1.tgz#70d1a99d335af87579a55d3c149ef1ae77da0a66" - integrity sha512-ikiw1IrM2W9NlQM3XazcX+4Sq3XAjZi4eeG22B5InKC2x5i7MatGF2S/Gn1ACZ+fEInwu+Ru9J8DlnBv1/hIvg== +"@azure/functions@^4.6.0": + version "4.6.0" + resolved "https://registry.yarnpkg.com/@azure/functions/-/functions-4.6.0.tgz#eee9ca945b8a2f2d0748c28006e057178cd5f8c9" + integrity sha512-vGq9jXlgrJ3KaI8bepgfpk26zVY8vFZsQukF85qjjKTAR90eFOOBNaa+mc/0ViDY2lcdrU2fL/o1pQyZUtTDsw== dependencies: - cookie "^0.6.0" + cookie "^0.7.0" long "^4.0.0" undici "^5.13.0" @@ -92,10 +92,10 @@ color-name@~1.1.4: resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== -cookie@^0.6.0: - version "0.6.0" - resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.6.0.tgz#2798b04b071b0ecbff0dbb62a505a8efa4e19051" - integrity sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw== +cookie@^0.7.0: + version "0.7.2" + resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.7.2.tgz#556369c472a2ba910f2979891b526b3436237ed7" + integrity sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w== debug@4, debug@^4.1.1: version "4.3.7" diff --git a/packages/datadog-plugin-fastify/src/code_origin.js b/packages/datadog-plugin-fastify/src/code_origin.js index 3e6f58d5624..6c9ddc7b028 100644 --- a/packages/datadog-plugin-fastify/src/code_origin.js +++ b/packages/datadog-plugin-fastify/src/code_origin.js @@ -1,6 +1,6 @@ 'use strict' -const { entryTag } = require('../../datadog-code-origin') +const { entryTags } = require('../../datadog-code-origin') const Plugin = require('../../dd-trace/src/plugins/plugin') const web = require('../../dd-trace/src/plugins/util/web') @@ -23,7 +23,7 @@ class FastifyCodeOriginForSpansPlugin extends Plugin { this.addSub('apm:fastify:route:added', ({ routeOptions, onRoute }) => { if (!routeOptions.config) routeOptions.config = {} - routeOptions.config[kCodeOriginForSpansTagsSym] = entryTag(onRoute) + routeOptions.config[kCodeOriginForSpansTagsSym] = entryTags(onRoute) }) } } diff --git a/packages/datadog-plugin-fastify/test/code_origin.spec.js b/packages/datadog-plugin-fastify/test/code_origin.spec.js index 711c2ffff6c..18f591dc6b9 100644 --- a/packages/datadog-plugin-fastify/test/code_origin.spec.js +++ b/packages/datadog-plugin-fastify/test/code_origin.spec.js @@ -3,6 +3,7 @@ const axios = require('axios') const semver = require('semver') const agent = require('../../dd-trace/test/plugins/agent') +const { getNextLineNumber } = require('../../dd-trace/test/plugins/helpers') const { NODE_MAJOR } = require('../../../version') const host = 'localhost' @@ -49,13 +50,13 @@ describe('Plugin', () => { // Wrap in a named function to have at least one frame with a function name function wrapperFunction () { - routeRegisterLine = getNextLineNumber() + routeRegisterLine = String(getNextLineNumber()) app.get('/user', function userHandler (request, reply) { reply.send() }) } - const callWrapperLine = getNextLineNumber() + const callWrapperLine = String(getNextLineNumber()) wrapperFunction() app.listen(() => { @@ -95,7 +96,7 @@ describe('Plugin', () => { let routeRegisterLine app.register(function v1Handler (app, opts, done) { - routeRegisterLine = getNextLineNumber() + routeRegisterLine = String(getNextLineNumber()) app.get('/user', function userHandler (request, reply) { reply.send() }) @@ -134,7 +135,7 @@ describe('Plugin', () => { next() }) - const routeRegisterLine = getNextLineNumber() + const routeRegisterLine = String(getNextLineNumber()) app.get('/user', function userHandler (request, reply) { reply.send() }) @@ -170,7 +171,7 @@ describe('Plugin', () => { // number of where the route handler is defined. However, this might not be the right choice and it might be // better to point to the middleware. it.skip('should point to middleware if middleware responds early', function testCase (done) { - const middlewareRegisterLine = getNextLineNumber() + const middlewareRegisterLine = String(getNextLineNumber()) app.use(function middleware (req, res, next) { res.end() }) @@ -210,7 +211,3 @@ describe('Plugin', () => { }) }) }) - -function getNextLineNumber () { - return String(Number(new Error().stack.split('\n')[2].match(/:(\d+):/)[1]) + 1) -} diff --git a/packages/datadog-plugin-fastify/test/suite.js b/packages/datadog-plugin-fastify/test/suite.js index 2033b6e6de1..bbb0218b894 100644 --- a/packages/datadog-plugin-fastify/test/suite.js +++ b/packages/datadog-plugin-fastify/test/suite.js @@ -1,9 +1,10 @@ 'use strict' -// const suiteTest = require('../../dd-trace/test/plugins/suite') -// suiteTest({ -// modName: 'fastify', -// repoUrl: 'fastify/fastify', -// commitish: 'latest', -// testCmd: 'node_modules/.bin/tap -J test/*.test.js test/*/*.test.js --no-coverage --no-check-coverage' -// }) +const suiteTest = require('../../dd-trace/test/plugins/suite') + +suiteTest({ + modName: 'fastify', + repoUrl: 'fastify/fastify', + commitish: 'latest', + testCmd: 'node_modules/.bin/tap -J test/*.test.js test/*/*.test.js --no-coverage --no-check-coverage' +}) diff --git a/packages/datadog-plugin-google-cloud-pubsub/src/consumer.js b/packages/datadog-plugin-google-cloud-pubsub/src/consumer.js index 3a330ad4c3a..84c4122ec57 100644 --- a/packages/datadog-plugin-google-cloud-pubsub/src/consumer.js +++ b/packages/datadog-plugin-google-cloud-pubsub/src/consumer.js @@ -1,5 +1,6 @@ 'use strict' +const { getMessageSize } = require('../../dd-trace/src/datastreams/processor') const ConsumerPlugin = require('../../dd-trace/src/plugins/consumer') class GoogleCloudPubsubConsumerPlugin extends ConsumerPlugin { @@ -11,7 +12,7 @@ class GoogleCloudPubsubConsumerPlugin extends ConsumerPlugin { const topic = subscription.metadata && subscription.metadata.topic const childOf = this.tracer.extract('text_map', message.attributes) || null - this.startSpan({ + const span = this.startSpan({ childOf, resource: topic, type: 'worker', @@ -23,6 +24,12 @@ class GoogleCloudPubsubConsumerPlugin extends ConsumerPlugin { 'pubsub.ack': 0 } }) + if (this.config.dsmEnabled && message?.attributes) { + const payloadSize = getMessageSize(message) + this.tracer.decodeDataStreamsContext(message.attributes) + this.tracer + .setCheckpoint(['direction:in', `topic:${topic}`, 'type:google-pubsub'], span, payloadSize) + } } finish (message) { diff --git a/packages/datadog-plugin-google-cloud-pubsub/src/producer.js b/packages/datadog-plugin-google-cloud-pubsub/src/producer.js index a34d6bfacd8..b6261ee85b6 100644 --- a/packages/datadog-plugin-google-cloud-pubsub/src/producer.js +++ b/packages/datadog-plugin-google-cloud-pubsub/src/producer.js @@ -1,6 +1,8 @@ 'use strict' const ProducerPlugin = require('../../dd-trace/src/plugins/producer') +const { DsmPathwayCodec } = require('../../dd-trace/src/datastreams/pathway') +const { getHeadersSize } = require('../../dd-trace/src/datastreams/processor') class GoogleCloudPubsubProducerPlugin extends ProducerPlugin { static get id () { return 'google-cloud-pubsub' } @@ -25,6 +27,12 @@ class GoogleCloudPubsubProducerPlugin extends ProducerPlugin { msg.attributes = {} } this.tracer.inject(span, 'text_map', msg.attributes) + if (this.config.dsmEnabled) { + const payloadSize = getHeadersSize(msg) + const dataStreamsContext = this.tracer + .setCheckpoint(['direction:out', `topic:${topic}`, 'type:google-pubsub'], span, payloadSize) + DsmPathwayCodec.encode(dataStreamsContext, msg.attributes) + } } } } diff --git a/packages/datadog-plugin-google-cloud-pubsub/test/index.spec.js b/packages/datadog-plugin-google-cloud-pubsub/test/index.spec.js index 89a0c5f03b8..80bc5f9509d 100644 --- a/packages/datadog-plugin-google-cloud-pubsub/test/index.spec.js +++ b/packages/datadog-plugin-google-cloud-pubsub/test/index.spec.js @@ -6,9 +6,12 @@ const id = require('../../dd-trace/src/id') const { ERROR_MESSAGE, ERROR_TYPE, ERROR_STACK } = require('../../dd-trace/src/constants') const { expectedSchema, rawExpectedSchema } = require('./naming') +const { computePathwayHash } = require('../../dd-trace/src/datastreams/pathway') +const { ENTRY_PARENT_HASH, DataStreamsProcessor } = require('../../dd-trace/src/datastreams/processor') // The roundtrip to the pubsub emulator takes time. Sometimes a *long* time. const TIMEOUT = 30000 +const dsmTopicName = 'dsm-topic' describe('Plugin', () => { let tracer @@ -18,6 +21,7 @@ describe('Plugin', () => { before(() => { process.env.PUBSUB_EMULATOR_HOST = 'localhost:8081' + process.env.DD_DATA_STREAMS_ENABLED = true }) after(() => { @@ -34,10 +38,12 @@ describe('Plugin', () => { let resource let v1 let gax + let expectedProducerHash + let expectedConsumerHash describe('without configuration', () => { beforeEach(() => { - return agent.load('google-cloud-pubsub') + return agent.load('google-cloud-pubsub', { dsmEnabled: false }) }) beforeEach(() => { @@ -296,7 +302,8 @@ describe('Plugin', () => { describe('with configuration', () => { beforeEach(() => { return agent.load('google-cloud-pubsub', { - service: 'a_test_service' + service: 'a_test_service', + dsmEnabled: false }) }) @@ -322,6 +329,113 @@ describe('Plugin', () => { }) }) + describe('data stream monitoring', () => { + let dsmTopic + let sub + let consume + + beforeEach(() => { + return agent.load('google-cloud-pubsub', { + dsmEnabled: true + }) + }) + + before(async () => { + const { PubSub } = require(`../../../versions/@google-cloud/pubsub@${version}`).get() + project = getProjectId() + resource = `projects/${project}/topics/${dsmTopicName}` + pubsub = new PubSub({ projectId: project }) + tracer.use('google-cloud-pubsub', { dsmEnabled: true }) + + dsmTopic = await pubsub.createTopic(dsmTopicName) + dsmTopic = dsmTopic[0] + sub = await dsmTopic.createSubscription('DSM') + sub = sub[0] + consume = function (cb) { + sub.on('message', cb) + } + + const dsmFullTopic = `projects/${project}/topics/${dsmTopicName}` + + expectedProducerHash = computePathwayHash( + 'test', + 'tester', + ['direction:out', 'topic:' + dsmFullTopic, 'type:google-pubsub'], + ENTRY_PARENT_HASH + ) + expectedConsumerHash = computePathwayHash( + 'test', + 'tester', + ['direction:in', 'topic:' + dsmFullTopic, 'type:google-pubsub'], + expectedProducerHash + ) + }) + + describe('should set a DSM checkpoint', () => { + it('on produce', async () => { + await publish(dsmTopic, { data: Buffer.from('DSM produce checkpoint') }) + + agent.expectPipelineStats(dsmStats => { + let statsPointsReceived = 0 + // we should have 1 dsm stats points + dsmStats.forEach((timeStatsBucket) => { + if (timeStatsBucket && timeStatsBucket.Stats) { + timeStatsBucket.Stats.forEach((statsBuckets) => { + statsPointsReceived += statsBuckets.Stats.length + }) + } + }) + expect(statsPointsReceived).to.be.at.least(1) + expect(agent.dsmStatsExist(agent, expectedProducerHash.readBigUInt64BE(0).toString())).to.equal(true) + }, { timeoutMs: TIMEOUT }) + }) + + it('on consume', async () => { + await publish(dsmTopic, { data: Buffer.from('DSM consume checkpoint') }) + await consume(async () => { + agent.expectPipelineStats(dsmStats => { + let statsPointsReceived = 0 + // we should have 2 dsm stats points + dsmStats.forEach((timeStatsBucket) => { + if (timeStatsBucket && timeStatsBucket.Stats) { + timeStatsBucket.Stats.forEach((statsBuckets) => { + statsPointsReceived += statsBuckets.Stats.length + }) + } + }) + expect(statsPointsReceived).to.be.at.least(2) + expect(agent.dsmStatsExist(agent, expectedConsumerHash.readBigUInt64BE(0).toString())).to.equal(true) + }, { timeoutMs: TIMEOUT }) + }) + }) + }) + + describe('it should set a message payload size', () => { + let recordCheckpointSpy + + beforeEach(() => { + recordCheckpointSpy = sinon.spy(DataStreamsProcessor.prototype, 'recordCheckpoint') + }) + + afterEach(() => { + DataStreamsProcessor.prototype.recordCheckpoint.restore() + }) + + it('when producing a message', async () => { + await publish(dsmTopic, { data: Buffer.from('DSM produce payload size') }) + expect(recordCheckpointSpy.args[0][0].hasOwnProperty('payloadSize')) + }) + + it('when consuming a message', async () => { + await publish(dsmTopic, { data: Buffer.from('DSM consume payload size') }) + + await consume(async () => { + expect(recordCheckpointSpy.args[0][0].hasOwnProperty('payloadSize')) + }) + }) + }) + }) + function expectSpanWithDefaults (expected) { const prefixedResource = [expected.meta['pubsub.method'], resource].filter(x => x).join(' ') const service = expected.meta['pubsub.method'] ? 'test-pubsub' : 'test' diff --git a/packages/datadog-plugin-grpc/src/client.js b/packages/datadog-plugin-grpc/src/client.js index ad841aab197..1b130a1f93e 100644 --- a/packages/datadog-plugin-grpc/src/client.js +++ b/packages/datadog-plugin-grpc/src/client.js @@ -64,6 +64,9 @@ class GrpcClientPlugin extends ClientPlugin { error ({ span, error }) { this.addCode(span, error.code) + if (error.code && !this._tracerConfig.grpc.client.error.statuses.includes(error.code)) { + return + } this.addError(error, span) } diff --git a/packages/datadog-plugin-grpc/src/server.js b/packages/datadog-plugin-grpc/src/server.js index d63164e31c1..0b599a1283d 100644 --- a/packages/datadog-plugin-grpc/src/server.js +++ b/packages/datadog-plugin-grpc/src/server.js @@ -70,6 +70,9 @@ class GrpcServerPlugin extends ServerPlugin { if (!span) return this.addCode(span, error.code) + if (error.code && !this._tracerConfig.grpc.server.error.statuses.includes(error.code)) { + return + } this.addError(error) } diff --git a/packages/datadog-plugin-grpc/test/client.spec.js b/packages/datadog-plugin-grpc/test/client.spec.js index 38205f1db38..4628fb8a5f6 100644 --- a/packages/datadog-plugin-grpc/test/client.spec.js +++ b/packages/datadog-plugin-grpc/test/client.spec.js @@ -7,7 +7,7 @@ const semver = require('semver') const Readable = require('stream').Readable const getService = require('./service') const loader = require('../../../versions/@grpc/proto-loader').get() -const { ERROR_MESSAGE, ERROR_TYPE, ERROR_STACK } = require('../../dd-trace/src/constants') +const { ERROR_MESSAGE, ERROR_TYPE, ERROR_STACK, GRPC_CLIENT_ERROR_STATUSES } = require('../../dd-trace/src/constants') const { DD_MAJOR } = require('../../../version') const nodeMajor = parseInt(process.versions.node.split('.')[0]) @@ -353,6 +353,23 @@ describe('Plugin', () => { }) }) + it('should ignore errors not set by DD_GRPC_CLIENT_ERROR_STATUSES', async () => { + tracer._tracer._config.grpc.client.error.statuses = [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13] + const client = await buildClient({ + getUnary: (_, callback) => callback(new Error('foobar')) + }) + + client.getUnary({ first: 'foobar' }, () => {}) + + return agent + .use(traces => { + expect(traces[0][0]).to.have.property('error', 0) + expect(traces[0][0].metrics).to.have.property('grpc.status.code', 2) + tracer._tracer._config.grpc.client.error.statuses = + GRPC_CLIENT_ERROR_STATUSES + }) + }) + it('should handle protocol errors', async () => { const definition = loader.loadSync(path.join(__dirname, 'invalid.proto')) const test = grpc.loadPackageDefinition(definition).test diff --git a/packages/datadog-plugin-grpc/test/server.spec.js b/packages/datadog-plugin-grpc/test/server.spec.js index 2406d087884..cf695840303 100644 --- a/packages/datadog-plugin-grpc/test/server.spec.js +++ b/packages/datadog-plugin-grpc/test/server.spec.js @@ -5,7 +5,7 @@ const agent = require('../../dd-trace/test/plugins/agent') const getPort = require('get-port') const Readable = require('stream').Readable -const { ERROR_MESSAGE, ERROR_TYPE, ERROR_STACK } = require('../../dd-trace/src/constants') +const { ERROR_MESSAGE, ERROR_TYPE, ERROR_STACK, GRPC_SERVER_ERROR_STATUSES } = require('../../dd-trace/src/constants') const nodeMajor = parseInt(process.versions.node.split('.')[0]) const pkgs = nodeMajor > 14 ? ['@grpc/grpc-js'] : ['grpc', '@grpc/grpc-js'] @@ -276,6 +276,38 @@ describe('Plugin', () => { }) }) + it('should ignore errors not set by DD_GRPC_SERVER_ERROR_STATUSES', async () => { + tracer._tracer._config.grpc.server.error.statuses = [6, 7, 8, 9, 10, 11, 12, 13] + const client = await buildClient({ + getUnary: (_, callback) => { + const metadata = new grpc.Metadata() + + metadata.set('extra', 'information') + + const error = new Error('foobar') + + error.code = grpc.status.NOT_FOUND + + const childOf = tracer.scope().active() + const child = tracer.startSpan('child', { childOf }) + + // Delay trace to ensure auto-cancellation doesn't override the status code. + setTimeout(() => child.finish()) + + callback(error, {}, metadata) + } + }) + + client.getUnary({ first: 'foobar' }, () => {}) + + return agent + .use(traces => { + expect(traces[0][0]).to.have.property('error', 0) + expect(traces[0][0].metrics).to.have.property('grpc.status.code', 5) + tracer._tracer._config.grpc.server.error.statuses = GRPC_SERVER_ERROR_STATUSES + }) + }) + it('should handle custom errors', async () => { const client = await buildClient({ getUnary: (_, callback) => { diff --git a/packages/datadog-plugin-http/test/code_origin.spec.js b/packages/datadog-plugin-http/test/code_origin.spec.js new file mode 100644 index 00000000000..4bb1a9003e0 --- /dev/null +++ b/packages/datadog-plugin-http/test/code_origin.spec.js @@ -0,0 +1,63 @@ +'use strict' + +const agent = require('../../dd-trace/test/plugins/agent') + +describe('Plugin', () => { + describe('http', () => { + describe('Code Origin for Spans', () => { + before(() => { + // Needed when this spec file run together with other spec files, in which case the agent config is not + // re-loaded unless the existing agent is wiped first. And we need the agent config to be re-loaded in order to + // enable Code Origin for Spans. + agent.wipe() + }) + + beforeEach(async () => { + return agent.load('http', { server: false }, { codeOriginForSpans: { enabled: true } }) + }) + + afterEach(() => { + return agent.close({ ritmReset: false }) + }) + + it('should add code_origin tags for outbound requests', done => { + server((port) => { + const http = require('http') + + agent + .use(traces => { + const span = traces[0][0] + expect(span.meta).to.have.property('_dd.code_origin.type', 'exit') + + // Just validate that frame 0 tags are present. The detailed validation is performed in a different test. + expect(span.meta).to.have.property('_dd.code_origin.frames.0.file') + expect(span.meta).to.have.property('_dd.code_origin.frames.0.line') + expect(span.meta).to.have.property('_dd.code_origin.frames.0.column') + expect(span.meta).to.have.property('_dd.code_origin.frames.0.method') + expect(span.meta).to.have.property('_dd.code_origin.frames.0.type') + }) + .then(done) + .catch(done) + + const req = http.request(`http://localhost:${port}/`, res => { + res.resume() + }) + + req.end() + }) + }) + }) + }) +}) + +function server (callback) { + const http = require('http') + + const server = http.createServer((req, res) => { + res.end() + }) + + server.listen(() => { + callback(server.address().port) + }) +} diff --git a/packages/datadog-plugin-kafkajs/src/batch-consumer.js b/packages/datadog-plugin-kafkajs/src/batch-consumer.js index 8415b037644..e0228a018c2 100644 --- a/packages/datadog-plugin-kafkajs/src/batch-consumer.js +++ b/packages/datadog-plugin-kafkajs/src/batch-consumer.js @@ -5,14 +5,17 @@ class KafkajsBatchConsumerPlugin extends ConsumerPlugin { static get id () { return 'kafkajs' } static get operation () { return 'consume-batch' } - start ({ topic, partition, messages, groupId }) { + start ({ topic, partition, messages, groupId, clusterId }) { if (!this.config.dsmEnabled) return for (const message of messages) { if (!message || !message.headers) continue const payloadSize = getMessageSize(message) this.tracer.decodeDataStreamsContext(message.headers) - this.tracer - .setCheckpoint(['direction:in', `group:${groupId}`, `topic:${topic}`, 'type:kafka'], null, payloadSize) + const edgeTags = ['direction:in', `group:${groupId}`, `topic:${topic}`, 'type:kafka'] + if (clusterId) { + edgeTags.push(`kafka_cluster_id:${clusterId}`) + } + this.tracer.setCheckpoint(edgeTags, null, payloadSize) } } } diff --git a/packages/datadog-plugin-kafkajs/src/consumer.js b/packages/datadog-plugin-kafkajs/src/consumer.js index 84b6a02fdda..ee04c5eb60c 100644 --- a/packages/datadog-plugin-kafkajs/src/consumer.js +++ b/packages/datadog-plugin-kafkajs/src/consumer.js @@ -62,7 +62,7 @@ class KafkajsConsumerPlugin extends ConsumerPlugin { } } - start ({ topic, partition, message, groupId }) { + start ({ topic, partition, message, groupId, clusterId }) { const childOf = extract(this.tracer, message.headers) const span = this.startSpan({ childOf, @@ -71,7 +71,8 @@ class KafkajsConsumerPlugin extends ConsumerPlugin { meta: { component: 'kafkajs', 'kafka.topic': topic, - 'kafka.message.offset': message.offset + 'kafka.message.offset': message.offset, + 'kafka.cluster_id': clusterId }, metrics: { 'kafka.partition': partition @@ -80,8 +81,11 @@ class KafkajsConsumerPlugin extends ConsumerPlugin { if (this.config.dsmEnabled && message?.headers) { const payloadSize = getMessageSize(message) this.tracer.decodeDataStreamsContext(message.headers) - this.tracer - .setCheckpoint(['direction:in', `group:${groupId}`, `topic:${topic}`, 'type:kafka'], span, payloadSize) + const edgeTags = ['direction:in', `group:${groupId}`, `topic:${topic}`, 'type:kafka'] + if (clusterId) { + edgeTags.push(`kafka_cluster_id:${clusterId}`) + } + this.tracer.setCheckpoint(edgeTags, span, payloadSize) } if (afterStartCh.hasSubscribers) { diff --git a/packages/datadog-plugin-kafkajs/src/producer.js b/packages/datadog-plugin-kafkajs/src/producer.js index 7b9aff95310..aa12357b4cf 100644 --- a/packages/datadog-plugin-kafkajs/src/producer.js +++ b/packages/datadog-plugin-kafkajs/src/producer.js @@ -66,12 +66,13 @@ class KafkajsProducerPlugin extends ProducerPlugin { } } - start ({ topic, messages, bootstrapServers }) { + start ({ topic, messages, bootstrapServers, clusterId }) { const span = this.startSpan({ resource: topic, meta: { component: 'kafkajs', - 'kafka.topic': topic + 'kafka.topic': topic, + 'kafka.cluster_id': clusterId }, metrics: { 'kafka.batch_size': messages.length @@ -85,8 +86,13 @@ class KafkajsProducerPlugin extends ProducerPlugin { this.tracer.inject(span, 'text_map', message.headers) if (this.config.dsmEnabled) { const payloadSize = getMessageSize(message) - const dataStreamsContext = this.tracer - .setCheckpoint(['direction:out', `topic:${topic}`, 'type:kafka'], span, payloadSize) + const edgeTags = ['direction:out', `topic:${topic}`, 'type:kafka'] + + if (clusterId) { + edgeTags.push(`kafka_cluster_id:${clusterId}`) + } + + const dataStreamsContext = this.tracer.setCheckpoint(edgeTags, span, payloadSize) DsmPathwayCodec.encode(dataStreamsContext, message.headers) } } diff --git a/packages/datadog-plugin-kafkajs/test/index.spec.js b/packages/datadog-plugin-kafkajs/test/index.spec.js index 3df303a95cf..f67279bdd9f 100644 --- a/packages/datadog-plugin-kafkajs/test/index.spec.js +++ b/packages/datadog-plugin-kafkajs/test/index.spec.js @@ -13,18 +13,22 @@ const { computePathwayHash } = require('../../dd-trace/src/datastreams/pathway') const { ENTRY_PARENT_HASH, DataStreamsProcessor } = require('../../dd-trace/src/datastreams/processor') const testTopic = 'test-topic' -const expectedProducerHash = computePathwayHash( - 'test', - 'tester', - ['direction:out', 'topic:' + testTopic, 'type:kafka'], - ENTRY_PARENT_HASH -) -const expectedConsumerHash = computePathwayHash( - 'test', - 'tester', - ['direction:in', 'group:test-group', 'topic:' + testTopic, 'type:kafka'], - expectedProducerHash -) +const testKafkaClusterId = '5L6g3nShT-eMCtK--X86sw' + +const getDsmPathwayHash = (clusterIdAvailable, isProducer, parentHash) => { + let edgeTags + if (isProducer) { + edgeTags = ['direction:out', 'topic:' + testTopic, 'type:kafka'] + } else { + edgeTags = ['direction:in', 'group:test-group', 'topic:' + testTopic, 'type:kafka'] + } + + if (clusterIdAvailable) { + edgeTags.push(`kafka_cluster_id:${testKafkaClusterId}`) + } + edgeTags.sort() + return computePathwayHash('test', 'tester', edgeTags, parentHash) +} describe('Plugin', () => { describe('kafkajs', function () { @@ -38,6 +42,16 @@ describe('Plugin', () => { let kafka let tracer let Kafka + let clusterIdAvailable + let expectedProducerHash + let expectedConsumerHash + + before(() => { + clusterIdAvailable = semver.intersects(version, '>=1.13') + expectedProducerHash = getDsmPathwayHash(clusterIdAvailable, true, ENTRY_PARENT_HASH) + expectedConsumerHash = getDsmPathwayHash(clusterIdAvailable, false, expectedProducerHash) + }) + describe('without configuration', () => { const messages = [{ key: 'key1', value: 'test2' }] @@ -56,14 +70,17 @@ describe('Plugin', () => { describe('producer', () => { it('should be instrumented', async () => { + const meta = { + 'span.kind': 'producer', + component: 'kafkajs', + 'pathway.hash': expectedProducerHash.readBigUInt64BE(0).toString() + } + if (clusterIdAvailable) meta['kafka.cluster_id'] = testKafkaClusterId + const expectedSpanPromise = expectSpanWithDefaults({ name: expectedSchema.send.opName, service: expectedSchema.send.serviceName, - meta: { - 'span.kind': 'producer', - component: 'kafkajs', - 'pathway.hash': expectedProducerHash.readBigUInt64BE(0).toString() - }, + meta, metrics: { 'kafka.batch_size': messages.length }, @@ -353,6 +370,12 @@ describe('Plugin', () => { await consumer.subscribe({ topic: testTopic }) }) + before(() => { + clusterIdAvailable = semver.intersects(version, '>=1.13') + expectedProducerHash = getDsmPathwayHash(clusterIdAvailable, true, ENTRY_PARENT_HASH) + expectedConsumerHash = getDsmPathwayHash(clusterIdAvailable, false, expectedProducerHash) + }) + afterEach(async () => { await consumer.disconnect() }) @@ -368,19 +391,6 @@ describe('Plugin', () => { setDataStreamsContextSpy.restore() }) - const expectedProducerHash = computePathwayHash( - 'test', - 'tester', - ['direction:out', 'topic:' + testTopic, 'type:kafka'], - ENTRY_PARENT_HASH - ) - const expectedConsumerHash = computePathwayHash( - 'test', - 'tester', - ['direction:in', 'group:test-group', 'topic:' + testTopic, 'type:kafka'], - expectedProducerHash - ) - it('Should set a checkpoint on produce', async () => { const messages = [{ key: 'consumerDSM1', value: 'test2' }] await sendMessages(kafka, testTopic, messages) @@ -476,9 +486,9 @@ describe('Plugin', () => { } /** - * No choice but to reinitialize everything, because the only way to flush eachMessage - * calls is to disconnect. - */ + * No choice but to reinitialize everything, because the only way to flush eachMessage + * calls is to disconnect. + */ consumer.connect() await sendMessages(kafka, testTopic, messages) await consumer.run({ eachMessage: async () => {}, autoCommit: false }) diff --git a/packages/datadog-plugin-mocha/src/index.js b/packages/datadog-plugin-mocha/src/index.js index 30f6e88a9fc..0513a4a95d6 100644 --- a/packages/datadog-plugin-mocha/src/index.js +++ b/packages/datadog-plugin-mocha/src/index.js @@ -242,7 +242,7 @@ class MochaPlugin extends CiPlugin { } }) - this.addSub('ci:mocha:test:retry', (isFirstAttempt) => { + this.addSub('ci:mocha:test:retry', ({ isFirstAttempt, err }) => { const store = storage.getStore() const span = store?.span if (span) { @@ -250,6 +250,9 @@ class MochaPlugin extends CiPlugin { if (!isFirstAttempt) { span.setTag(TEST_IS_RETRY, 'true') } + if (err) { + span.setTag('error', err) + } const spanTags = span.context()._tags this.telemetry.ciVisEvent( diff --git a/packages/datadog-plugin-openai/src/index.js b/packages/datadog-plugin-openai/src/index.js index f96b44543d2..c76f7333910 100644 --- a/packages/datadog-plugin-openai/src/index.js +++ b/packages/datadog-plugin-openai/src/index.js @@ -1,1023 +1,17 @@ 'use strict' -const path = require('path') +const CompositePlugin = require('../../dd-trace/src/plugins/composite') +const OpenAiTracingPlugin = require('./tracing') +const OpenAiLLMObsPlugin = require('../../dd-trace/src/llmobs/plugins/openai') -const TracingPlugin = require('../../dd-trace/src/plugins/tracing') -const { storage } = require('../../datadog-core') -const services = require('./services') -const Sampler = require('../../dd-trace/src/sampler') -const { MEASURED } = require('../../../ext/tags') -const { estimateTokens } = require('./token-estimator') - -// String#replaceAll unavailable on Node.js@v14 (dd-trace@<=v3) -const RE_NEWLINE = /\n/g -const RE_TAB = /\t/g - -// TODO: In the future we should refactor config.js to make it requirable -let MAX_TEXT_LEN = 128 - -function safeRequire (path) { - try { - // eslint-disable-next-line import/no-extraneous-dependencies - return require(path) - } catch { - return null - } -} - -const encodingForModel = safeRequire('tiktoken')?.encoding_for_model - -class OpenApiPlugin extends TracingPlugin { +class OpenAiPlugin extends CompositePlugin { static get id () { return 'openai' } - static get operation () { return 'request' } - static get system () { return 'openai' } - static get prefix () { - return 'tracing:apm:openai:request' - } - - constructor (...args) { - super(...args) - - const { metrics, logger } = services.init(this._tracerConfig) - this.metrics = metrics - this.logger = logger - - this.sampler = new Sampler(0.1) // default 10% log sampling - - // hoist the max length env var to avoid making all of these functions a class method - if (this._tracerConfig) { - MAX_TEXT_LEN = this._tracerConfig.openaiSpanCharLimit - } - } - - configure (config) { - if (config.enabled === false) { - services.shutdown() - } - - super.configure(config) - } - - bindStart (ctx) { - const { methodName, args, basePath, apiKey } = ctx - const payload = normalizeRequestPayload(methodName, args) - const store = storage.getStore() || {} - - const span = this.startSpan('openai.request', { - service: this.config.service, - resource: methodName, - type: 'openai', - kind: 'client', - meta: { - [MEASURED]: 1, - // Data that is always available with a request - 'openai.user.api_key': truncateApiKey(apiKey), - 'openai.api_base': basePath, - // The openai.api_type (openai|azure) is present in Python but not in Node.js - // Add support once https://github.com/openai/openai-node/issues/53 is closed - - // Data that is common across many requests - 'openai.request.best_of': payload.best_of, - 'openai.request.echo': payload.echo, - 'openai.request.logprobs': payload.logprobs, - 'openai.request.max_tokens': payload.max_tokens, - 'openai.request.model': payload.model, // vague model - 'openai.request.n': payload.n, - 'openai.request.presence_penalty': payload.presence_penalty, - 'openai.request.frequency_penalty': payload.frequency_penalty, - 'openai.request.stop': payload.stop, - 'openai.request.suffix': payload.suffix, - 'openai.request.temperature': payload.temperature, - 'openai.request.top_p': payload.top_p, - 'openai.request.user': payload.user, - 'openai.request.file_id': payload.file_id // deleteFile, retrieveFile, downloadFile - } - }, false) - - const openaiStore = Object.create(null) - - const tags = {} // The remaining tags are added one at a time - - // createChatCompletion, createCompletion, createImage, createImageEdit, createTranscription, createTranslation - if (payload.prompt) { - const prompt = payload.prompt - openaiStore.prompt = prompt - if (typeof prompt === 'string' || (Array.isArray(prompt) && typeof prompt[0] === 'number')) { - // This is a single prompt, either String or [Number] - tags['openai.request.prompt'] = normalizeStringOrTokenArray(prompt, true) - } else if (Array.isArray(prompt)) { - // This is multiple prompts, either [String] or [[Number]] - for (let i = 0; i < prompt.length; i++) { - tags[`openai.request.prompt.${i}`] = normalizeStringOrTokenArray(prompt[i], true) - } - } - } - - // createEdit, createEmbedding, createModeration - if (payload.input) { - const normalized = normalizeStringOrTokenArray(payload.input, false) - tags['openai.request.input'] = truncateText(normalized) - openaiStore.input = normalized - } - - // createChatCompletion, createCompletion - if (payload.logit_bias !== null && typeof payload.logit_bias === 'object') { - for (const [tokenId, bias] of Object.entries(payload.logit_bias)) { - tags[`openai.request.logit_bias.${tokenId}`] = bias - } - } - - if (payload.stream) { - tags['openai.request.stream'] = payload.stream - } - - switch (methodName) { - case 'createFineTune': - case 'fine_tuning.jobs.create': - case 'fine-tune.create': - createFineTuneRequestExtraction(tags, payload) - break - - case 'createImage': - case 'images.generate': - case 'createImageEdit': - case 'images.edit': - case 'createImageVariation': - case 'images.createVariation': - commonCreateImageRequestExtraction(tags, payload, openaiStore) - break - - case 'createChatCompletion': - case 'chat.completions.create': - createChatCompletionRequestExtraction(tags, payload, openaiStore) - break - - case 'createFile': - case 'files.create': - case 'retrieveFile': - case 'files.retrieve': - commonFileRequestExtraction(tags, payload) - break - - case 'createTranscription': - case 'audio.transcriptions.create': - case 'createTranslation': - case 'audio.translations.create': - commonCreateAudioRequestExtraction(tags, payload, openaiStore) - break - - case 'retrieveModel': - case 'models.retrieve': - retrieveModelRequestExtraction(tags, payload) - break - - case 'listFineTuneEvents': - case 'fine_tuning.jobs.listEvents': - case 'fine-tune.listEvents': - case 'retrieveFineTune': - case 'fine_tuning.jobs.retrieve': - case 'fine-tune.retrieve': - case 'deleteModel': - case 'models.del': - case 'cancelFineTune': - case 'fine_tuning.jobs.cancel': - case 'fine-tune.cancel': - commonLookupFineTuneRequestExtraction(tags, payload) - break - - case 'createEdit': - case 'edits.create': - createEditRequestExtraction(tags, payload, openaiStore) - break - } - - span.addTags(tags) - - ctx.currentStore = { ...store, span, openai: openaiStore } - - return ctx.currentStore - } - - asyncEnd (ctx) { - const { result } = ctx - const store = ctx.currentStore - - const span = store?.span - if (!span) return - - const error = !!span.context()._tags.error - - let headers, body, method, path - if (!error) { - headers = result.headers - body = result.data - method = result.request.method - path = result.request.path - } - - if (!error && headers?.constructor.name === 'Headers') { - headers = Object.fromEntries(headers) - } - const methodName = span._spanContext._tags['resource.name'] - - body = coerceResponseBody(body, methodName) - - const openaiStore = store.openai - - if (!error && (path?.startsWith('https://') || path?.startsWith('http://'))) { - // basic checking for if the path was set as a full URL - // not using a full regex as it will likely be "https://api.openai.com/..." - path = new URL(path).pathname - } - const endpoint = lookupOperationEndpoint(methodName, path) - - const tags = error - ? {} - : { - 'openai.request.endpoint': endpoint, - 'openai.request.method': method.toUpperCase(), - - 'openai.organization.id': body.organization_id, // only available in fine-tunes endpoints - 'openai.organization.name': headers['openai-organization'], - - 'openai.response.model': headers['openai-model'] || body.model, // specific model, often undefined - 'openai.response.id': body.id, // common creation value, numeric epoch - 'openai.response.deleted': body.deleted, // common boolean field in delete responses - - // The OpenAI API appears to use both created and created_at in different places - // Here we're conciously choosing to surface this inconsistency instead of normalizing - 'openai.response.created': body.created, - 'openai.response.created_at': body.created_at - } - - responseDataExtractionByMethod(methodName, tags, body, openaiStore) - span.addTags(tags) - - span.finish() - this.sendLog(methodName, span, tags, openaiStore, error) - this.sendMetrics(headers, body, endpoint, span._duration, error, tags) - } - - sendMetrics (headers, body, endpoint, duration, error, spanTags) { - const tags = [`error:${Number(!!error)}`] - if (error) { - this.metrics.increment('openai.request.error', 1, tags) - } else { - tags.push(`org:${headers['openai-organization']}`) - tags.push(`endpoint:${endpoint}`) // just "/v1/models", no method - tags.push(`model:${headers['openai-model'] || body.model}`) - } - - this.metrics.distribution('openai.request.duration', duration * 1000, tags) - - const promptTokens = spanTags['openai.response.usage.prompt_tokens'] - const promptTokensEstimated = spanTags['openai.response.usage.prompt_tokens_estimated'] - - const completionTokens = spanTags['openai.response.usage.completion_tokens'] - const completionTokensEstimated = spanTags['openai.response.usage.completion_tokens_estimated'] - - const totalTokens = spanTags['openai.response.usage.total_tokens'] - - if (!error) { - if (promptTokens != null) { - if (promptTokensEstimated) { - this.metrics.distribution( - 'openai.tokens.prompt', promptTokens, [...tags, 'openai.estimated:true']) - } else { - this.metrics.distribution('openai.tokens.prompt', promptTokens, tags) - } - } - - if (completionTokens != null) { - if (completionTokensEstimated) { - this.metrics.distribution( - 'openai.tokens.completion', completionTokens, [...tags, 'openai.estimated:true']) - } else { - this.metrics.distribution('openai.tokens.completion', completionTokens, tags) - } - } - - if (totalTokens != null) { - if (promptTokensEstimated || completionTokensEstimated) { - this.metrics.distribution( - 'openai.tokens.total', totalTokens, [...tags, 'openai.estimated:true']) - } else { - this.metrics.distribution('openai.tokens.total', totalTokens, tags) - } - } - } - - if (headers) { - if (headers['x-ratelimit-limit-requests']) { - this.metrics.gauge('openai.ratelimit.requests', Number(headers['x-ratelimit-limit-requests']), tags) - } - - if (headers['x-ratelimit-remaining-requests']) { - this.metrics.gauge( - 'openai.ratelimit.remaining.requests', Number(headers['x-ratelimit-remaining-requests']), tags - ) - } - - if (headers['x-ratelimit-limit-tokens']) { - this.metrics.gauge('openai.ratelimit.tokens', Number(headers['x-ratelimit-limit-tokens']), tags) - } - - if (headers['x-ratelimit-remaining-tokens']) { - this.metrics.gauge('openai.ratelimit.remaining.tokens', Number(headers['x-ratelimit-remaining-tokens']), tags) - } - } - } - - sendLog (methodName, span, tags, openaiStore, error) { - if (!openaiStore) return - if (!Object.keys(openaiStore).length) return - if (!this.sampler.isSampled()) return - - const log = { - status: error ? 'error' : 'info', - message: `sampled ${methodName}`, - ...openaiStore - } - - this.logger.log(log, span, tags) - } -} - -function countPromptTokens (methodName, payload, model) { - let promptTokens = 0 - let promptEstimated = false - if (methodName === 'chat.completions.create') { - const messages = payload.messages - for (const message of messages) { - const content = message.content - if (typeof content === 'string') { - const { tokens, estimated } = countTokens(content, model) - promptTokens += tokens - promptEstimated = estimated - } else if (Array.isArray(content)) { - for (const c of content) { - if (c.type === 'text') { - const { tokens, estimated } = countTokens(c.text, model) - promptTokens += tokens - promptEstimated = estimated - } - // unsupported token computation for image_url - // as even though URL is a string, its true token count - // is based on the image itself, something onerous to do client-side - } - } - } - } else if (methodName === 'completions.create') { - let prompt = payload.prompt - if (!Array.isArray(prompt)) prompt = [prompt] - - for (const p of prompt) { - const { tokens, estimated } = countTokens(p, model) - promptTokens += tokens - promptEstimated = estimated - } - } - - return { promptTokens, promptEstimated } -} - -function countCompletionTokens (body, model) { - let completionTokens = 0 - let completionEstimated = false - if (body?.choices) { - for (const choice of body.choices) { - const message = choice.message || choice.delta // delta for streamed responses - const text = choice.text - const content = text || message?.content - - const { tokens, estimated } = countTokens(content, model) - completionTokens += tokens - completionEstimated = estimated - } - } - - return { completionTokens, completionEstimated } -} - -function countTokens (content, model) { - if (encodingForModel) { - try { - // try using tiktoken if it was available - const encoder = encodingForModel(model) - const tokens = encoder.encode(content).length - encoder.free() - return { tokens, estimated: false } - } catch { - // possible errors from tiktoken: - // * model not available for token counts - // * issue encoding content - } - } - - return { - tokens: estimateTokens(content), - estimated: true - } -} - -function createEditRequestExtraction (tags, payload, openaiStore) { - const instruction = payload.instruction - tags['openai.request.instruction'] = instruction - openaiStore.instruction = instruction -} - -function retrieveModelRequestExtraction (tags, payload) { - tags['openai.request.id'] = payload.id -} - -function createChatCompletionRequestExtraction (tags, payload, openaiStore) { - const messages = payload.messages - if (!defensiveArrayLength(messages)) return - - openaiStore.messages = payload.messages - for (let i = 0; i < payload.messages.length; i++) { - const message = payload.messages[i] - tagChatCompletionRequestContent(message.content, i, tags) - tags[`openai.request.messages.${i}.role`] = message.role - tags[`openai.request.messages.${i}.name`] = message.name - tags[`openai.request.messages.${i}.finish_reason`] = message.finish_reason - } -} - -function commonCreateImageRequestExtraction (tags, payload, openaiStore) { - // createImageEdit, createImageVariation - const img = payload.file || payload.image - if (img !== null && typeof img === 'object' && img.path) { - const file = path.basename(img.path) - tags['openai.request.image'] = file - openaiStore.file = file - } - - // createImageEdit - if (payload.mask !== null && typeof payload.mask === 'object' && payload.mask.path) { - const mask = path.basename(payload.mask.path) - tags['openai.request.mask'] = mask - openaiStore.mask = mask - } - - tags['openai.request.size'] = payload.size - tags['openai.request.response_format'] = payload.response_format - tags['openai.request.language'] = payload.language -} - -function responseDataExtractionByMethod (methodName, tags, body, openaiStore) { - switch (methodName) { - case 'createModeration': - case 'moderations.create': - createModerationResponseExtraction(tags, body) - break - - case 'createCompletion': - case 'completions.create': - case 'createChatCompletion': - case 'chat.completions.create': - case 'createEdit': - case 'edits.create': - commonCreateResponseExtraction(tags, body, openaiStore, methodName) - break - - case 'listFiles': - case 'files.list': - case 'listFineTunes': - case 'fine_tuning.jobs.list': - case 'fine-tune.list': - case 'listFineTuneEvents': - case 'fine_tuning.jobs.listEvents': - case 'fine-tune.listEvents': - commonListCountResponseExtraction(tags, body) - break - - case 'createEmbedding': - case 'embeddings.create': - createEmbeddingResponseExtraction(tags, body, openaiStore) - break - - case 'createFile': - case 'files.create': - case 'retrieveFile': - case 'files.retrieve': - createRetrieveFileResponseExtraction(tags, body) - break - - case 'deleteFile': - case 'files.del': - deleteFileResponseExtraction(tags, body) - break - - case 'downloadFile': - case 'files.retrieveContent': - case 'files.content': - downloadFileResponseExtraction(tags, body) - break - - case 'createFineTune': - case 'fine_tuning.jobs.create': - case 'fine-tune.create': - case 'retrieveFineTune': - case 'fine_tuning.jobs.retrieve': - case 'fine-tune.retrieve': - case 'cancelFineTune': - case 'fine_tuning.jobs.cancel': - case 'fine-tune.cancel': - commonFineTuneResponseExtraction(tags, body) - break - - case 'createTranscription': - case 'audio.transcriptions.create': - case 'createTranslation': - case 'audio.translations.create': - createAudioResponseExtraction(tags, body) - break - - case 'createImage': - case 'images.generate': - case 'createImageEdit': - case 'images.edit': - case 'createImageVariation': - case 'images.createVariation': - commonImageResponseExtraction(tags, body) - break - - case 'listModels': - case 'models.list': - listModelsResponseExtraction(tags, body) - break - - case 'retrieveModel': - case 'models.retrieve': - retrieveModelResponseExtraction(tags, body) - break - } -} - -function retrieveModelResponseExtraction (tags, body) { - tags['openai.response.owned_by'] = body.owned_by - tags['openai.response.parent'] = body.parent - tags['openai.response.root'] = body.root - - if (!body.permission) return - - tags['openai.response.permission.id'] = body.permission[0].id - tags['openai.response.permission.created'] = body.permission[0].created - tags['openai.response.permission.allow_create_engine'] = body.permission[0].allow_create_engine - tags['openai.response.permission.allow_sampling'] = body.permission[0].allow_sampling - tags['openai.response.permission.allow_logprobs'] = body.permission[0].allow_logprobs - tags['openai.response.permission.allow_search_indices'] = body.permission[0].allow_search_indices - tags['openai.response.permission.allow_view'] = body.permission[0].allow_view - tags['openai.response.permission.allow_fine_tuning'] = body.permission[0].allow_fine_tuning - tags['openai.response.permission.organization'] = body.permission[0].organization - tags['openai.response.permission.group'] = body.permission[0].group - tags['openai.response.permission.is_blocking'] = body.permission[0].is_blocking -} - -function commonLookupFineTuneRequestExtraction (tags, body) { - tags['openai.request.fine_tune_id'] = body.fine_tune_id - tags['openai.request.stream'] = !!body.stream // listFineTuneEvents -} - -function listModelsResponseExtraction (tags, body) { - if (!body.data) return - - tags['openai.response.count'] = body.data.length -} - -function commonImageResponseExtraction (tags, body) { - if (!body.data) return - - tags['openai.response.images_count'] = body.data.length - - for (let i = 0; i < body.data.length; i++) { - const image = body.data[i] - // exactly one of these two options is provided - tags[`openai.response.images.${i}.url`] = truncateText(image.url) - tags[`openai.response.images.${i}.b64_json`] = image.b64_json && 'returned' - } -} - -function createAudioResponseExtraction (tags, body) { - tags['openai.response.text'] = body.text - tags['openai.response.language'] = body.language - tags['openai.response.duration'] = body.duration - tags['openai.response.segments_count'] = defensiveArrayLength(body.segments) -} - -function createFineTuneRequestExtraction (tags, body) { - tags['openai.request.training_file'] = body.training_file - tags['openai.request.validation_file'] = body.validation_file - tags['openai.request.n_epochs'] = body.n_epochs - tags['openai.request.batch_size'] = body.batch_size - tags['openai.request.learning_rate_multiplier'] = body.learning_rate_multiplier - tags['openai.request.prompt_loss_weight'] = body.prompt_loss_weight - tags['openai.request.compute_classification_metrics'] = body.compute_classification_metrics - tags['openai.request.classification_n_classes'] = body.classification_n_classes - tags['openai.request.classification_positive_class'] = body.classification_positive_class - tags['openai.request.classification_betas_count'] = defensiveArrayLength(body.classification_betas) -} - -function commonFineTuneResponseExtraction (tags, body) { - tags['openai.response.events_count'] = defensiveArrayLength(body.events) - tags['openai.response.fine_tuned_model'] = body.fine_tuned_model - - const hyperparams = body.hyperparams || body.hyperparameters - const hyperparamsKey = body.hyperparams ? 'hyperparams' : 'hyperparameters' - - if (hyperparams) { - tags[`openai.response.${hyperparamsKey}.n_epochs`] = hyperparams.n_epochs - tags[`openai.response.${hyperparamsKey}.batch_size`] = hyperparams.batch_size - tags[`openai.response.${hyperparamsKey}.prompt_loss_weight`] = hyperparams.prompt_loss_weight - tags[`openai.response.${hyperparamsKey}.learning_rate_multiplier`] = hyperparams.learning_rate_multiplier - } - tags['openai.response.training_files_count'] = defensiveArrayLength(body.training_files || body.training_file) - tags['openai.response.result_files_count'] = defensiveArrayLength(body.result_files) - tags['openai.response.validation_files_count'] = defensiveArrayLength(body.validation_files || body.validation_file) - tags['openai.response.updated_at'] = body.updated_at - tags['openai.response.status'] = body.status -} - -// the OpenAI package appears to stream the content download then provide it all as a singular string -function downloadFileResponseExtraction (tags, body) { - if (!body.file) return - tags['openai.response.total_bytes'] = body.file.length -} - -function deleteFileResponseExtraction (tags, body) { - tags['openai.response.id'] = body.id -} - -function commonCreateAudioRequestExtraction (tags, body, openaiStore) { - tags['openai.request.response_format'] = body.response_format - tags['openai.request.language'] = body.language - - if (body.file !== null && typeof body.file === 'object' && body.file.path) { - const filename = path.basename(body.file.path) - tags['openai.request.filename'] = filename - openaiStore.file = filename - } -} - -function commonFileRequestExtraction (tags, body) { - tags['openai.request.purpose'] = body.purpose - - // User can provider either exact file contents or a file read stream - // With the stream we extract the filepath - // This is a best effort attempt to extract the filename during the request - if (body.file !== null && typeof body.file === 'object' && body.file.path) { - tags['openai.request.filename'] = path.basename(body.file.path) - } -} - -function createRetrieveFileResponseExtraction (tags, body) { - tags['openai.response.filename'] = body.filename - tags['openai.response.purpose'] = body.purpose - tags['openai.response.bytes'] = body.bytes - tags['openai.response.status'] = body.status - tags['openai.response.status_details'] = body.status_details -} - -function createEmbeddingResponseExtraction (tags, body, openaiStore) { - usageExtraction(tags, body, openaiStore) - - if (!body.data) return - - tags['openai.response.embeddings_count'] = body.data.length - for (let i = 0; i < body.data.length; i++) { - tags[`openai.response.embedding.${i}.embedding_length`] = body.data[i].embedding.length - } -} - -function commonListCountResponseExtraction (tags, body) { - if (!body.data) return - tags['openai.response.count'] = body.data.length -} - -// TODO: Is there ever more than one entry in body.results? -function createModerationResponseExtraction (tags, body) { - tags['openai.response.id'] = body.id - // tags[`openai.response.model`] = body.model // redundant, already extracted globally - - if (!body.results) return - - tags['openai.response.flagged'] = body.results[0].flagged - - for (const [category, match] of Object.entries(body.results[0].categories)) { - tags[`openai.response.categories.${category}`] = match - } - - for (const [category, score] of Object.entries(body.results[0].category_scores)) { - tags[`openai.response.category_scores.${category}`] = score - } -} - -// createCompletion, createChatCompletion, createEdit -function commonCreateResponseExtraction (tags, body, openaiStore, methodName) { - usageExtraction(tags, body, methodName, openaiStore) - - if (!body.choices) return - - tags['openai.response.choices_count'] = body.choices.length - - openaiStore.choices = body.choices - - for (let choiceIdx = 0; choiceIdx < body.choices.length; choiceIdx++) { - const choice = body.choices[choiceIdx] - - // logprobs can be null and we still want to tag it as 'returned' even when set to 'null' - const specifiesLogProb = Object.keys(choice).indexOf('logprobs') !== -1 - - tags[`openai.response.choices.${choiceIdx}.finish_reason`] = choice.finish_reason - tags[`openai.response.choices.${choiceIdx}.logprobs`] = specifiesLogProb ? 'returned' : undefined - tags[`openai.response.choices.${choiceIdx}.text`] = truncateText(choice.text) - - // createChatCompletion only - const message = choice.message || choice.delta // delta for streamed responses - if (message) { - tags[`openai.response.choices.${choiceIdx}.message.role`] = message.role - tags[`openai.response.choices.${choiceIdx}.message.content`] = truncateText(message.content) - tags[`openai.response.choices.${choiceIdx}.message.name`] = truncateText(message.name) - if (message.tool_calls) { - const toolCalls = message.tool_calls - for (let toolIdx = 0; toolIdx < toolCalls.length; toolIdx++) { - tags[`openai.response.choices.${choiceIdx}.message.tool_calls.${toolIdx}.function.name`] = - toolCalls[toolIdx].function.name - tags[`openai.response.choices.${choiceIdx}.message.tool_calls.${toolIdx}.function.arguments`] = - toolCalls[toolIdx].function.arguments - tags[`openai.response.choices.${choiceIdx}.message.tool_calls.${toolIdx}.id`] = - toolCalls[toolIdx].id - } - } - } - } -} - -// createCompletion, createChatCompletion, createEdit, createEmbedding -function usageExtraction (tags, body, methodName, openaiStore) { - let promptTokens = 0 - let completionTokens = 0 - let totalTokens = 0 - if (body && body.usage) { - promptTokens = body.usage.prompt_tokens - completionTokens = body.usage.completion_tokens - totalTokens = body.usage.total_tokens - } else if (body.model && ['chat.completions.create', 'completions.create'].includes(methodName)) { - // estimate tokens based on method name for completions and chat completions - const { model } = body - let promptEstimated = false - let completionEstimated = false - - // prompt tokens - const payload = openaiStore - const promptTokensCount = countPromptTokens(methodName, payload, model) - promptTokens = promptTokensCount.promptTokens - promptEstimated = promptTokensCount.promptEstimated - - // completion tokens - const completionTokensCount = countCompletionTokens(body, model) - completionTokens = completionTokensCount.completionTokens - completionEstimated = completionTokensCount.completionEstimated - - // total tokens - totalTokens = promptTokens + completionTokens - if (promptEstimated) tags['openai.response.usage.prompt_tokens_estimated'] = true - if (completionEstimated) tags['openai.response.usage.completion_tokens_estimated'] = true - } - - if (promptTokens != null) tags['openai.response.usage.prompt_tokens'] = promptTokens - if (completionTokens != null) tags['openai.response.usage.completion_tokens'] = completionTokens - if (totalTokens != null) tags['openai.response.usage.total_tokens'] = totalTokens -} - -function truncateApiKey (apiKey) { - return apiKey && `sk-...${apiKey.substr(apiKey.length - 4)}` -} - -/** - * for cleaning up prompt and response - */ -function truncateText (text) { - if (!text) return - if (typeof text !== 'string' || !text || (typeof text === 'string' && text.length === 0)) return - - text = text - .replace(RE_NEWLINE, '\\n') - .replace(RE_TAB, '\\t') - - if (text.length > MAX_TEXT_LEN) { - return text.substring(0, MAX_TEXT_LEN) + '...' - } - - return text -} - -function tagChatCompletionRequestContent (contents, messageIdx, tags) { - if (typeof contents === 'string') { - tags[`openai.request.messages.${messageIdx}.content`] = contents - } else if (Array.isArray(contents)) { - // content can also be an array of objects - // which represent text input or image url - for (const contentIdx in contents) { - const content = contents[contentIdx] - const type = content.type - tags[`openai.request.messages.${messageIdx}.content.${contentIdx}.type`] = content.type - if (type === 'text') { - tags[`openai.request.messages.${messageIdx}.content.${contentIdx}.text`] = truncateText(content.text) - } else if (type === 'image_url') { - tags[`openai.request.messages.${messageIdx}.content.${contentIdx}.image_url.url`] = - truncateText(content.image_url.url) - } - // unsupported type otherwise, won't be tagged - } - } - // unsupported type otherwise, won't be tagged -} - -// The server almost always responds with JSON -function coerceResponseBody (body, methodName) { - switch (methodName) { - case 'downloadFile': - case 'files.retrieveContent': - case 'files.content': - return { file: body } - } - - const type = typeof body - if (type === 'string') { - try { - return JSON.parse(body) - } catch { - return body + static get plugins () { + return { + llmobs: OpenAiLLMObsPlugin, + tracing: OpenAiTracingPlugin } - } else if (type === 'object') { - return body - } else { - return {} } } -// This method is used to replace a dynamic URL segment with an asterisk -function lookupOperationEndpoint (operationId, url) { - switch (operationId) { - case 'deleteModel': - case 'models.del': - case 'retrieveModel': - case 'models.retrieve': - return '/v1/models/*' - - case 'deleteFile': - case 'files.del': - case 'retrieveFile': - case 'files.retrieve': - return '/v1/files/*' - - case 'downloadFile': - case 'files.retrieveContent': - case 'files.content': - return '/v1/files/*/content' - - case 'retrieveFineTune': - case 'fine-tune.retrieve': - return '/v1/fine-tunes/*' - case 'fine_tuning.jobs.retrieve': - return '/v1/fine_tuning/jobs/*' - - case 'listFineTuneEvents': - case 'fine-tune.listEvents': - return '/v1/fine-tunes/*/events' - case 'fine_tuning.jobs.listEvents': - return '/v1/fine_tuning/jobs/*/events' - - case 'cancelFineTune': - case 'fine-tune.cancel': - return '/v1/fine-tunes/*/cancel' - case 'fine_tuning.jobs.cancel': - return '/v1/fine_tuning/jobs/*/cancel' - } - - return url -} - -/** - * This function essentially normalizes the OpenAI method interface. Many methods accept - * a single object argument. The remaining ones take individual arguments. This function - * turns the individual arguments into an object to make extracting properties consistent. - */ -function normalizeRequestPayload (methodName, args) { - switch (methodName) { - case 'listModels': - case 'models.list': - case 'listFiles': - case 'files.list': - case 'listFineTunes': - case 'fine_tuning.jobs.list': - case 'fine-tune.list': - // no argument - return {} - - case 'retrieveModel': - case 'models.retrieve': - return { id: args[0] } - - case 'createFile': - return { - file: args[0], - purpose: args[1] - } - - case 'deleteFile': - case 'files.del': - case 'retrieveFile': - case 'files.retrieve': - case 'downloadFile': - case 'files.retrieveContent': - case 'files.content': - return { file_id: args[0] } - - case 'listFineTuneEvents': - case 'fine_tuning.jobs.listEvents': - case 'fine-tune.listEvents': - return { - fine_tune_id: args[0], - stream: args[1] // undocumented - } - - case 'retrieveFineTune': - case 'fine_tuning.jobs.retrieve': - case 'fine-tune.retrieve': - case 'deleteModel': - case 'models.del': - case 'cancelFineTune': - case 'fine_tuning.jobs.cancel': - case 'fine-tune.cancel': - return { fine_tune_id: args[0] } - - case 'createImageEdit': - return { - file: args[0], - prompt: args[1], // Note: order of prompt/mask in Node.js lib differs from public docs - mask: args[2], - n: args[3], - size: args[4], - response_format: args[5], - user: args[6] - } - - case 'createImageVariation': - return { - file: args[0], - n: args[1], - size: args[2], - response_format: args[3], - user: args[4] - } - - case 'createTranscription': - case 'createTranslation': - return { - file: args[0], - model: args[1], - prompt: args[2], - response_format: args[3], - temperature: args[4], - language: args[5] // only used for createTranscription - } - } - - // Remaining OpenAI methods take a single object argument - return args[0] -} - -/** - * Converts an array of tokens to a string - * If input is already a string it's returned - * In either case the value is truncated - - * It's intentional that the array be truncated arbitrarily, e.g. "[999, 888, 77..." - - * "foo" -> "foo" - * [1,2,3] -> "[1, 2, 3]" - */ -function normalizeStringOrTokenArray (input, truncate) { - const normalized = Array.isArray(input) - ? `[${input.join(', ')}]` // "[1, 2, 999]" - : input // "foo" - return truncate ? truncateText(normalized) : normalized -} - -function defensiveArrayLength (maybeArray) { - if (maybeArray) { - if (Array.isArray(maybeArray)) { - return maybeArray.length - } else { - // case of a singular item (ie body.training_file vs body.training_files) - return 1 - } - } - - return undefined -} - -module.exports = OpenApiPlugin +module.exports = OpenAiPlugin diff --git a/packages/datadog-plugin-openai/src/tracing.js b/packages/datadog-plugin-openai/src/tracing.js new file mode 100644 index 00000000000..a92f66a6df6 --- /dev/null +++ b/packages/datadog-plugin-openai/src/tracing.js @@ -0,0 +1,1023 @@ +'use strict' + +const path = require('path') + +const TracingPlugin = require('../../dd-trace/src/plugins/tracing') +const { storage } = require('../../datadog-core') +const services = require('./services') +const Sampler = require('../../dd-trace/src/sampler') +const { MEASURED } = require('../../../ext/tags') +const { estimateTokens } = require('./token-estimator') + +// String#replaceAll unavailable on Node.js@v14 (dd-trace@<=v3) +const RE_NEWLINE = /\n/g +const RE_TAB = /\t/g + +// TODO: In the future we should refactor config.js to make it requirable +let MAX_TEXT_LEN = 128 + +function safeRequire (path) { + try { + // eslint-disable-next-line import/no-extraneous-dependencies + return require(path) + } catch { + return null + } +} + +const encodingForModel = safeRequire('tiktoken')?.encoding_for_model + +class OpenAiTracingPlugin extends TracingPlugin { + static get id () { return 'openai' } + static get operation () { return 'request' } + static get system () { return 'openai' } + static get prefix () { + return 'tracing:apm:openai:request' + } + + constructor (...args) { + super(...args) + + const { metrics, logger } = services.init(this._tracerConfig) + this.metrics = metrics + this.logger = logger + + this.sampler = new Sampler(0.1) // default 10% log sampling + + // hoist the max length env var to avoid making all of these functions a class method + if (this._tracerConfig) { + MAX_TEXT_LEN = this._tracerConfig.openaiSpanCharLimit + } + } + + configure (config) { + if (config.enabled === false) { + services.shutdown() + } + + super.configure(config) + } + + bindStart (ctx) { + const { methodName, args, basePath, apiKey } = ctx + const payload = normalizeRequestPayload(methodName, args) + const store = storage.getStore() || {} + + const span = this.startSpan('openai.request', { + service: this.config.service, + resource: methodName, + type: 'openai', + kind: 'client', + meta: { + [MEASURED]: 1, + // Data that is always available with a request + 'openai.user.api_key': truncateApiKey(apiKey), + 'openai.api_base': basePath, + // The openai.api_type (openai|azure) is present in Python but not in Node.js + // Add support once https://github.com/openai/openai-node/issues/53 is closed + + // Data that is common across many requests + 'openai.request.best_of': payload.best_of, + 'openai.request.echo': payload.echo, + 'openai.request.logprobs': payload.logprobs, + 'openai.request.max_tokens': payload.max_tokens, + 'openai.request.model': payload.model, // vague model + 'openai.request.n': payload.n, + 'openai.request.presence_penalty': payload.presence_penalty, + 'openai.request.frequency_penalty': payload.frequency_penalty, + 'openai.request.stop': payload.stop, + 'openai.request.suffix': payload.suffix, + 'openai.request.temperature': payload.temperature, + 'openai.request.top_p': payload.top_p, + 'openai.request.user': payload.user, + 'openai.request.file_id': payload.file_id // deleteFile, retrieveFile, downloadFile + } + }, false) + + const openaiStore = Object.create(null) + + const tags = {} // The remaining tags are added one at a time + + // createChatCompletion, createCompletion, createImage, createImageEdit, createTranscription, createTranslation + if (payload.prompt) { + const prompt = payload.prompt + openaiStore.prompt = prompt + if (typeof prompt === 'string' || (Array.isArray(prompt) && typeof prompt[0] === 'number')) { + // This is a single prompt, either String or [Number] + tags['openai.request.prompt'] = normalizeStringOrTokenArray(prompt, true) + } else if (Array.isArray(prompt)) { + // This is multiple prompts, either [String] or [[Number]] + for (let i = 0; i < prompt.length; i++) { + tags[`openai.request.prompt.${i}`] = normalizeStringOrTokenArray(prompt[i], true) + } + } + } + + // createEdit, createEmbedding, createModeration + if (payload.input) { + const normalized = normalizeStringOrTokenArray(payload.input, false) + tags['openai.request.input'] = truncateText(normalized) + openaiStore.input = normalized + } + + // createChatCompletion, createCompletion + if (payload.logit_bias !== null && typeof payload.logit_bias === 'object') { + for (const [tokenId, bias] of Object.entries(payload.logit_bias)) { + tags[`openai.request.logit_bias.${tokenId}`] = bias + } + } + + if (payload.stream) { + tags['openai.request.stream'] = payload.stream + } + + switch (methodName) { + case 'createFineTune': + case 'fine_tuning.jobs.create': + case 'fine-tune.create': + createFineTuneRequestExtraction(tags, payload) + break + + case 'createImage': + case 'images.generate': + case 'createImageEdit': + case 'images.edit': + case 'createImageVariation': + case 'images.createVariation': + commonCreateImageRequestExtraction(tags, payload, openaiStore) + break + + case 'createChatCompletion': + case 'chat.completions.create': + createChatCompletionRequestExtraction(tags, payload, openaiStore) + break + + case 'createFile': + case 'files.create': + case 'retrieveFile': + case 'files.retrieve': + commonFileRequestExtraction(tags, payload) + break + + case 'createTranscription': + case 'audio.transcriptions.create': + case 'createTranslation': + case 'audio.translations.create': + commonCreateAudioRequestExtraction(tags, payload, openaiStore) + break + + case 'retrieveModel': + case 'models.retrieve': + retrieveModelRequestExtraction(tags, payload) + break + + case 'listFineTuneEvents': + case 'fine_tuning.jobs.listEvents': + case 'fine-tune.listEvents': + case 'retrieveFineTune': + case 'fine_tuning.jobs.retrieve': + case 'fine-tune.retrieve': + case 'deleteModel': + case 'models.del': + case 'cancelFineTune': + case 'fine_tuning.jobs.cancel': + case 'fine-tune.cancel': + commonLookupFineTuneRequestExtraction(tags, payload) + break + + case 'createEdit': + case 'edits.create': + createEditRequestExtraction(tags, payload, openaiStore) + break + } + + span.addTags(tags) + + ctx.currentStore = { ...store, span, openai: openaiStore } + + return ctx.currentStore + } + + asyncEnd (ctx) { + const { result } = ctx + const store = ctx.currentStore + + const span = store?.span + if (!span) return + + const error = !!span.context()._tags.error + + let headers, body, method, path + if (!error) { + headers = result.headers + body = result.data + method = result.request.method + path = result.request.path + } + + if (!error && headers?.constructor.name === 'Headers') { + headers = Object.fromEntries(headers) + } + const methodName = span._spanContext._tags['resource.name'] + + body = coerceResponseBody(body, methodName) + + const openaiStore = store.openai + + if (!error && (path?.startsWith('https://') || path?.startsWith('http://'))) { + // basic checking for if the path was set as a full URL + // not using a full regex as it will likely be "https://api.openai.com/..." + path = new URL(path).pathname + } + const endpoint = lookupOperationEndpoint(methodName, path) + + const tags = error + ? {} + : { + 'openai.request.endpoint': endpoint, + 'openai.request.method': method.toUpperCase(), + + 'openai.organization.id': body.organization_id, // only available in fine-tunes endpoints + 'openai.organization.name': headers['openai-organization'], + + 'openai.response.model': headers['openai-model'] || body.model, // specific model, often undefined + 'openai.response.id': body.id, // common creation value, numeric epoch + 'openai.response.deleted': body.deleted, // common boolean field in delete responses + + // The OpenAI API appears to use both created and created_at in different places + // Here we're conciously choosing to surface this inconsistency instead of normalizing + 'openai.response.created': body.created, + 'openai.response.created_at': body.created_at + } + + responseDataExtractionByMethod(methodName, tags, body, openaiStore) + span.addTags(tags) + + span.finish() + this.sendLog(methodName, span, tags, openaiStore, error) + this.sendMetrics(headers, body, endpoint, span._duration, error, tags) + } + + sendMetrics (headers, body, endpoint, duration, error, spanTags) { + const tags = [`error:${Number(!!error)}`] + if (error) { + this.metrics.increment('openai.request.error', 1, tags) + } else { + tags.push(`org:${headers['openai-organization']}`) + tags.push(`endpoint:${endpoint}`) // just "/v1/models", no method + tags.push(`model:${headers['openai-model'] || body.model}`) + } + + this.metrics.distribution('openai.request.duration', duration * 1000, tags) + + const promptTokens = spanTags['openai.response.usage.prompt_tokens'] + const promptTokensEstimated = spanTags['openai.response.usage.prompt_tokens_estimated'] + + const completionTokens = spanTags['openai.response.usage.completion_tokens'] + const completionTokensEstimated = spanTags['openai.response.usage.completion_tokens_estimated'] + + const totalTokens = spanTags['openai.response.usage.total_tokens'] + + if (!error) { + if (promptTokens != null) { + if (promptTokensEstimated) { + this.metrics.distribution( + 'openai.tokens.prompt', promptTokens, [...tags, 'openai.estimated:true']) + } else { + this.metrics.distribution('openai.tokens.prompt', promptTokens, tags) + } + } + + if (completionTokens != null) { + if (completionTokensEstimated) { + this.metrics.distribution( + 'openai.tokens.completion', completionTokens, [...tags, 'openai.estimated:true']) + } else { + this.metrics.distribution('openai.tokens.completion', completionTokens, tags) + } + } + + if (totalTokens != null) { + if (promptTokensEstimated || completionTokensEstimated) { + this.metrics.distribution( + 'openai.tokens.total', totalTokens, [...tags, 'openai.estimated:true']) + } else { + this.metrics.distribution('openai.tokens.total', totalTokens, tags) + } + } + } + + if (headers) { + if (headers['x-ratelimit-limit-requests']) { + this.metrics.gauge('openai.ratelimit.requests', Number(headers['x-ratelimit-limit-requests']), tags) + } + + if (headers['x-ratelimit-remaining-requests']) { + this.metrics.gauge( + 'openai.ratelimit.remaining.requests', Number(headers['x-ratelimit-remaining-requests']), tags + ) + } + + if (headers['x-ratelimit-limit-tokens']) { + this.metrics.gauge('openai.ratelimit.tokens', Number(headers['x-ratelimit-limit-tokens']), tags) + } + + if (headers['x-ratelimit-remaining-tokens']) { + this.metrics.gauge('openai.ratelimit.remaining.tokens', Number(headers['x-ratelimit-remaining-tokens']), tags) + } + } + } + + sendLog (methodName, span, tags, openaiStore, error) { + if (!openaiStore) return + if (!Object.keys(openaiStore).length) return + if (!this.sampler.isSampled()) return + + const log = { + status: error ? 'error' : 'info', + message: `sampled ${methodName}`, + ...openaiStore + } + + this.logger.log(log, span, tags) + } +} + +function countPromptTokens (methodName, payload, model) { + let promptTokens = 0 + let promptEstimated = false + if (methodName === 'chat.completions.create') { + const messages = payload.messages + for (const message of messages) { + const content = message.content + if (typeof content === 'string') { + const { tokens, estimated } = countTokens(content, model) + promptTokens += tokens + promptEstimated = estimated + } else if (Array.isArray(content)) { + for (const c of content) { + if (c.type === 'text') { + const { tokens, estimated } = countTokens(c.text, model) + promptTokens += tokens + promptEstimated = estimated + } + // unsupported token computation for image_url + // as even though URL is a string, its true token count + // is based on the image itself, something onerous to do client-side + } + } + } + } else if (methodName === 'completions.create') { + let prompt = payload.prompt + if (!Array.isArray(prompt)) prompt = [prompt] + + for (const p of prompt) { + const { tokens, estimated } = countTokens(p, model) + promptTokens += tokens + promptEstimated = estimated + } + } + + return { promptTokens, promptEstimated } +} + +function countCompletionTokens (body, model) { + let completionTokens = 0 + let completionEstimated = false + if (body?.choices) { + for (const choice of body.choices) { + const message = choice.message || choice.delta // delta for streamed responses + const text = choice.text + const content = text || message?.content + + const { tokens, estimated } = countTokens(content, model) + completionTokens += tokens + completionEstimated = estimated + } + } + + return { completionTokens, completionEstimated } +} + +function countTokens (content, model) { + if (encodingForModel) { + try { + // try using tiktoken if it was available + const encoder = encodingForModel(model) + const tokens = encoder.encode(content).length + encoder.free() + return { tokens, estimated: false } + } catch { + // possible errors from tiktoken: + // * model not available for token counts + // * issue encoding content + } + } + + return { + tokens: estimateTokens(content), + estimated: true + } +} + +function createEditRequestExtraction (tags, payload, openaiStore) { + const instruction = payload.instruction + tags['openai.request.instruction'] = instruction + openaiStore.instruction = instruction +} + +function retrieveModelRequestExtraction (tags, payload) { + tags['openai.request.id'] = payload.id +} + +function createChatCompletionRequestExtraction (tags, payload, openaiStore) { + const messages = payload.messages + if (!defensiveArrayLength(messages)) return + + openaiStore.messages = payload.messages + for (let i = 0; i < payload.messages.length; i++) { + const message = payload.messages[i] + tagChatCompletionRequestContent(message.content, i, tags) + tags[`openai.request.messages.${i}.role`] = message.role + tags[`openai.request.messages.${i}.name`] = message.name + tags[`openai.request.messages.${i}.finish_reason`] = message.finish_reason + } +} + +function commonCreateImageRequestExtraction (tags, payload, openaiStore) { + // createImageEdit, createImageVariation + const img = payload.file || payload.image + if (img !== null && typeof img === 'object' && img.path) { + const file = path.basename(img.path) + tags['openai.request.image'] = file + openaiStore.file = file + } + + // createImageEdit + if (payload.mask !== null && typeof payload.mask === 'object' && payload.mask.path) { + const mask = path.basename(payload.mask.path) + tags['openai.request.mask'] = mask + openaiStore.mask = mask + } + + tags['openai.request.size'] = payload.size + tags['openai.request.response_format'] = payload.response_format + tags['openai.request.language'] = payload.language +} + +function responseDataExtractionByMethod (methodName, tags, body, openaiStore) { + switch (methodName) { + case 'createModeration': + case 'moderations.create': + createModerationResponseExtraction(tags, body) + break + + case 'createCompletion': + case 'completions.create': + case 'createChatCompletion': + case 'chat.completions.create': + case 'createEdit': + case 'edits.create': + commonCreateResponseExtraction(tags, body, openaiStore, methodName) + break + + case 'listFiles': + case 'files.list': + case 'listFineTunes': + case 'fine_tuning.jobs.list': + case 'fine-tune.list': + case 'listFineTuneEvents': + case 'fine_tuning.jobs.listEvents': + case 'fine-tune.listEvents': + commonListCountResponseExtraction(tags, body) + break + + case 'createEmbedding': + case 'embeddings.create': + createEmbeddingResponseExtraction(tags, body, openaiStore) + break + + case 'createFile': + case 'files.create': + case 'retrieveFile': + case 'files.retrieve': + createRetrieveFileResponseExtraction(tags, body) + break + + case 'deleteFile': + case 'files.del': + deleteFileResponseExtraction(tags, body) + break + + case 'downloadFile': + case 'files.retrieveContent': + case 'files.content': + downloadFileResponseExtraction(tags, body) + break + + case 'createFineTune': + case 'fine_tuning.jobs.create': + case 'fine-tune.create': + case 'retrieveFineTune': + case 'fine_tuning.jobs.retrieve': + case 'fine-tune.retrieve': + case 'cancelFineTune': + case 'fine_tuning.jobs.cancel': + case 'fine-tune.cancel': + commonFineTuneResponseExtraction(tags, body) + break + + case 'createTranscription': + case 'audio.transcriptions.create': + case 'createTranslation': + case 'audio.translations.create': + createAudioResponseExtraction(tags, body) + break + + case 'createImage': + case 'images.generate': + case 'createImageEdit': + case 'images.edit': + case 'createImageVariation': + case 'images.createVariation': + commonImageResponseExtraction(tags, body) + break + + case 'listModels': + case 'models.list': + listModelsResponseExtraction(tags, body) + break + + case 'retrieveModel': + case 'models.retrieve': + retrieveModelResponseExtraction(tags, body) + break + } +} + +function retrieveModelResponseExtraction (tags, body) { + tags['openai.response.owned_by'] = body.owned_by + tags['openai.response.parent'] = body.parent + tags['openai.response.root'] = body.root + + if (!body.permission) return + + tags['openai.response.permission.id'] = body.permission[0].id + tags['openai.response.permission.created'] = body.permission[0].created + tags['openai.response.permission.allow_create_engine'] = body.permission[0].allow_create_engine + tags['openai.response.permission.allow_sampling'] = body.permission[0].allow_sampling + tags['openai.response.permission.allow_logprobs'] = body.permission[0].allow_logprobs + tags['openai.response.permission.allow_search_indices'] = body.permission[0].allow_search_indices + tags['openai.response.permission.allow_view'] = body.permission[0].allow_view + tags['openai.response.permission.allow_fine_tuning'] = body.permission[0].allow_fine_tuning + tags['openai.response.permission.organization'] = body.permission[0].organization + tags['openai.response.permission.group'] = body.permission[0].group + tags['openai.response.permission.is_blocking'] = body.permission[0].is_blocking +} + +function commonLookupFineTuneRequestExtraction (tags, body) { + tags['openai.request.fine_tune_id'] = body.fine_tune_id + tags['openai.request.stream'] = !!body.stream // listFineTuneEvents +} + +function listModelsResponseExtraction (tags, body) { + if (!body.data) return + + tags['openai.response.count'] = body.data.length +} + +function commonImageResponseExtraction (tags, body) { + if (!body.data) return + + tags['openai.response.images_count'] = body.data.length + + for (let i = 0; i < body.data.length; i++) { + const image = body.data[i] + // exactly one of these two options is provided + tags[`openai.response.images.${i}.url`] = truncateText(image.url) + tags[`openai.response.images.${i}.b64_json`] = image.b64_json && 'returned' + } +} + +function createAudioResponseExtraction (tags, body) { + tags['openai.response.text'] = body.text + tags['openai.response.language'] = body.language + tags['openai.response.duration'] = body.duration + tags['openai.response.segments_count'] = defensiveArrayLength(body.segments) +} + +function createFineTuneRequestExtraction (tags, body) { + tags['openai.request.training_file'] = body.training_file + tags['openai.request.validation_file'] = body.validation_file + tags['openai.request.n_epochs'] = body.n_epochs + tags['openai.request.batch_size'] = body.batch_size + tags['openai.request.learning_rate_multiplier'] = body.learning_rate_multiplier + tags['openai.request.prompt_loss_weight'] = body.prompt_loss_weight + tags['openai.request.compute_classification_metrics'] = body.compute_classification_metrics + tags['openai.request.classification_n_classes'] = body.classification_n_classes + tags['openai.request.classification_positive_class'] = body.classification_positive_class + tags['openai.request.classification_betas_count'] = defensiveArrayLength(body.classification_betas) +} + +function commonFineTuneResponseExtraction (tags, body) { + tags['openai.response.events_count'] = defensiveArrayLength(body.events) + tags['openai.response.fine_tuned_model'] = body.fine_tuned_model + + const hyperparams = body.hyperparams || body.hyperparameters + const hyperparamsKey = body.hyperparams ? 'hyperparams' : 'hyperparameters' + + if (hyperparams) { + tags[`openai.response.${hyperparamsKey}.n_epochs`] = hyperparams.n_epochs + tags[`openai.response.${hyperparamsKey}.batch_size`] = hyperparams.batch_size + tags[`openai.response.${hyperparamsKey}.prompt_loss_weight`] = hyperparams.prompt_loss_weight + tags[`openai.response.${hyperparamsKey}.learning_rate_multiplier`] = hyperparams.learning_rate_multiplier + } + tags['openai.response.training_files_count'] = defensiveArrayLength(body.training_files || body.training_file) + tags['openai.response.result_files_count'] = defensiveArrayLength(body.result_files) + tags['openai.response.validation_files_count'] = defensiveArrayLength(body.validation_files || body.validation_file) + tags['openai.response.updated_at'] = body.updated_at + tags['openai.response.status'] = body.status +} + +// the OpenAI package appears to stream the content download then provide it all as a singular string +function downloadFileResponseExtraction (tags, body) { + if (!body.file) return + tags['openai.response.total_bytes'] = body.file.length +} + +function deleteFileResponseExtraction (tags, body) { + tags['openai.response.id'] = body.id +} + +function commonCreateAudioRequestExtraction (tags, body, openaiStore) { + tags['openai.request.response_format'] = body.response_format + tags['openai.request.language'] = body.language + + if (body.file !== null && typeof body.file === 'object' && body.file.path) { + const filename = path.basename(body.file.path) + tags['openai.request.filename'] = filename + openaiStore.file = filename + } +} + +function commonFileRequestExtraction (tags, body) { + tags['openai.request.purpose'] = body.purpose + + // User can provider either exact file contents or a file read stream + // With the stream we extract the filepath + // This is a best effort attempt to extract the filename during the request + if (body.file !== null && typeof body.file === 'object' && body.file.path) { + tags['openai.request.filename'] = path.basename(body.file.path) + } +} + +function createRetrieveFileResponseExtraction (tags, body) { + tags['openai.response.filename'] = body.filename + tags['openai.response.purpose'] = body.purpose + tags['openai.response.bytes'] = body.bytes + tags['openai.response.status'] = body.status + tags['openai.response.status_details'] = body.status_details +} + +function createEmbeddingResponseExtraction (tags, body, openaiStore) { + usageExtraction(tags, body, openaiStore) + + if (!body.data) return + + tags['openai.response.embeddings_count'] = body.data.length + for (let i = 0; i < body.data.length; i++) { + tags[`openai.response.embedding.${i}.embedding_length`] = body.data[i].embedding.length + } +} + +function commonListCountResponseExtraction (tags, body) { + if (!body.data) return + tags['openai.response.count'] = body.data.length +} + +// TODO: Is there ever more than one entry in body.results? +function createModerationResponseExtraction (tags, body) { + tags['openai.response.id'] = body.id + // tags[`openai.response.model`] = body.model // redundant, already extracted globally + + if (!body.results) return + + tags['openai.response.flagged'] = body.results[0].flagged + + for (const [category, match] of Object.entries(body.results[0].categories)) { + tags[`openai.response.categories.${category}`] = match + } + + for (const [category, score] of Object.entries(body.results[0].category_scores)) { + tags[`openai.response.category_scores.${category}`] = score + } +} + +// createCompletion, createChatCompletion, createEdit +function commonCreateResponseExtraction (tags, body, openaiStore, methodName) { + usageExtraction(tags, body, methodName, openaiStore) + + if (!body.choices) return + + tags['openai.response.choices_count'] = body.choices.length + + openaiStore.choices = body.choices + + for (let choiceIdx = 0; choiceIdx < body.choices.length; choiceIdx++) { + const choice = body.choices[choiceIdx] + + // logprobs can be null and we still want to tag it as 'returned' even when set to 'null' + const specifiesLogProb = Object.keys(choice).indexOf('logprobs') !== -1 + + tags[`openai.response.choices.${choiceIdx}.finish_reason`] = choice.finish_reason + tags[`openai.response.choices.${choiceIdx}.logprobs`] = specifiesLogProb ? 'returned' : undefined + tags[`openai.response.choices.${choiceIdx}.text`] = truncateText(choice.text) + + // createChatCompletion only + const message = choice.message || choice.delta // delta for streamed responses + if (message) { + tags[`openai.response.choices.${choiceIdx}.message.role`] = message.role + tags[`openai.response.choices.${choiceIdx}.message.content`] = truncateText(message.content) + tags[`openai.response.choices.${choiceIdx}.message.name`] = truncateText(message.name) + if (message.tool_calls) { + const toolCalls = message.tool_calls + for (let toolIdx = 0; toolIdx < toolCalls.length; toolIdx++) { + tags[`openai.response.choices.${choiceIdx}.message.tool_calls.${toolIdx}.function.name`] = + toolCalls[toolIdx].function.name + tags[`openai.response.choices.${choiceIdx}.message.tool_calls.${toolIdx}.function.arguments`] = + toolCalls[toolIdx].function.arguments + tags[`openai.response.choices.${choiceIdx}.message.tool_calls.${toolIdx}.id`] = + toolCalls[toolIdx].id + } + } + } + } +} + +// createCompletion, createChatCompletion, createEdit, createEmbedding +function usageExtraction (tags, body, methodName, openaiStore) { + let promptTokens = 0 + let completionTokens = 0 + let totalTokens = 0 + if (body && body.usage) { + promptTokens = body.usage.prompt_tokens + completionTokens = body.usage.completion_tokens + totalTokens = body.usage.total_tokens + } else if (body.model && ['chat.completions.create', 'completions.create'].includes(methodName)) { + // estimate tokens based on method name for completions and chat completions + const { model } = body + let promptEstimated = false + let completionEstimated = false + + // prompt tokens + const payload = openaiStore + const promptTokensCount = countPromptTokens(methodName, payload, model) + promptTokens = promptTokensCount.promptTokens + promptEstimated = promptTokensCount.promptEstimated + + // completion tokens + const completionTokensCount = countCompletionTokens(body, model) + completionTokens = completionTokensCount.completionTokens + completionEstimated = completionTokensCount.completionEstimated + + // total tokens + totalTokens = promptTokens + completionTokens + if (promptEstimated) tags['openai.response.usage.prompt_tokens_estimated'] = true + if (completionEstimated) tags['openai.response.usage.completion_tokens_estimated'] = true + } + + if (promptTokens != null) tags['openai.response.usage.prompt_tokens'] = promptTokens + if (completionTokens != null) tags['openai.response.usage.completion_tokens'] = completionTokens + if (totalTokens != null) tags['openai.response.usage.total_tokens'] = totalTokens +} + +function truncateApiKey (apiKey) { + return apiKey && `sk-...${apiKey.substr(apiKey.length - 4)}` +} + +/** + * for cleaning up prompt and response + */ +function truncateText (text) { + if (!text) return + if (typeof text !== 'string' || !text || (typeof text === 'string' && text.length === 0)) return + + text = text + .replace(RE_NEWLINE, '\\n') + .replace(RE_TAB, '\\t') + + if (text.length > MAX_TEXT_LEN) { + return text.substring(0, MAX_TEXT_LEN) + '...' + } + + return text +} + +function tagChatCompletionRequestContent (contents, messageIdx, tags) { + if (typeof contents === 'string') { + tags[`openai.request.messages.${messageIdx}.content`] = contents + } else if (Array.isArray(contents)) { + // content can also be an array of objects + // which represent text input or image url + for (const contentIdx in contents) { + const content = contents[contentIdx] + const type = content.type + tags[`openai.request.messages.${messageIdx}.content.${contentIdx}.type`] = content.type + if (type === 'text') { + tags[`openai.request.messages.${messageIdx}.content.${contentIdx}.text`] = truncateText(content.text) + } else if (type === 'image_url') { + tags[`openai.request.messages.${messageIdx}.content.${contentIdx}.image_url.url`] = + truncateText(content.image_url.url) + } + // unsupported type otherwise, won't be tagged + } + } + // unsupported type otherwise, won't be tagged +} + +// The server almost always responds with JSON +function coerceResponseBody (body, methodName) { + switch (methodName) { + case 'downloadFile': + case 'files.retrieveContent': + case 'files.content': + return { file: body } + } + + const type = typeof body + if (type === 'string') { + try { + return JSON.parse(body) + } catch { + return body + } + } else if (type === 'object') { + return body + } else { + return {} + } +} + +// This method is used to replace a dynamic URL segment with an asterisk +function lookupOperationEndpoint (operationId, url) { + switch (operationId) { + case 'deleteModel': + case 'models.del': + case 'retrieveModel': + case 'models.retrieve': + return '/v1/models/*' + + case 'deleteFile': + case 'files.del': + case 'retrieveFile': + case 'files.retrieve': + return '/v1/files/*' + + case 'downloadFile': + case 'files.retrieveContent': + case 'files.content': + return '/v1/files/*/content' + + case 'retrieveFineTune': + case 'fine-tune.retrieve': + return '/v1/fine-tunes/*' + case 'fine_tuning.jobs.retrieve': + return '/v1/fine_tuning/jobs/*' + + case 'listFineTuneEvents': + case 'fine-tune.listEvents': + return '/v1/fine-tunes/*/events' + case 'fine_tuning.jobs.listEvents': + return '/v1/fine_tuning/jobs/*/events' + + case 'cancelFineTune': + case 'fine-tune.cancel': + return '/v1/fine-tunes/*/cancel' + case 'fine_tuning.jobs.cancel': + return '/v1/fine_tuning/jobs/*/cancel' + } + + return url +} + +/** + * This function essentially normalizes the OpenAI method interface. Many methods accept + * a single object argument. The remaining ones take individual arguments. This function + * turns the individual arguments into an object to make extracting properties consistent. + */ +function normalizeRequestPayload (methodName, args) { + switch (methodName) { + case 'listModels': + case 'models.list': + case 'listFiles': + case 'files.list': + case 'listFineTunes': + case 'fine_tuning.jobs.list': + case 'fine-tune.list': + // no argument + return {} + + case 'retrieveModel': + case 'models.retrieve': + return { id: args[0] } + + case 'createFile': + return { + file: args[0], + purpose: args[1] + } + + case 'deleteFile': + case 'files.del': + case 'retrieveFile': + case 'files.retrieve': + case 'downloadFile': + case 'files.retrieveContent': + case 'files.content': + return { file_id: args[0] } + + case 'listFineTuneEvents': + case 'fine_tuning.jobs.listEvents': + case 'fine-tune.listEvents': + return { + fine_tune_id: args[0], + stream: args[1] // undocumented + } + + case 'retrieveFineTune': + case 'fine_tuning.jobs.retrieve': + case 'fine-tune.retrieve': + case 'deleteModel': + case 'models.del': + case 'cancelFineTune': + case 'fine_tuning.jobs.cancel': + case 'fine-tune.cancel': + return { fine_tune_id: args[0] } + + case 'createImageEdit': + return { + file: args[0], + prompt: args[1], // Note: order of prompt/mask in Node.js lib differs from public docs + mask: args[2], + n: args[3], + size: args[4], + response_format: args[5], + user: args[6] + } + + case 'createImageVariation': + return { + file: args[0], + n: args[1], + size: args[2], + response_format: args[3], + user: args[4] + } + + case 'createTranscription': + case 'createTranslation': + return { + file: args[0], + model: args[1], + prompt: args[2], + response_format: args[3], + temperature: args[4], + language: args[5] // only used for createTranscription + } + } + + // Remaining OpenAI methods take a single object argument + return args[0] +} + +/** + * Converts an array of tokens to a string + * If input is already a string it's returned + * In either case the value is truncated + + * It's intentional that the array be truncated arbitrarily, e.g. "[999, 888, 77..." + + * "foo" -> "foo" + * [1,2,3] -> "[1, 2, 3]" + */ +function normalizeStringOrTokenArray (input, truncate) { + const normalized = Array.isArray(input) + ? `[${input.join(', ')}]` // "[1, 2, 999]" + : input // "foo" + return truncate ? truncateText(normalized) : normalized +} + +function defensiveArrayLength (maybeArray) { + if (maybeArray) { + if (Array.isArray(maybeArray)) { + return maybeArray.length + } else { + // case of a singular item (ie body.training_file vs body.training_files) + return 1 + } + } + + return undefined +} + +module.exports = OpenAiTracingPlugin diff --git a/packages/dd-trace/src/appsec/channels.js b/packages/dd-trace/src/appsec/channels.js index 3081ed9974a..10bd31c9fb5 100644 --- a/packages/dd-trace/src/appsec/channels.js +++ b/packages/dd-trace/src/appsec/channels.js @@ -6,6 +6,7 @@ const dc = require('dc-polyfill') module.exports = { bodyParser: dc.channel('datadog:body-parser:read:finish'), cookieParser: dc.channel('datadog:cookie-parser:read:finish'), + multerParser: dc.channel('datadog:multer:read:finish'), startGraphqlResolve: dc.channel('datadog:graphql:resolver:start'), graphqlMiddlewareChannel: dc.tracingChannel('datadog:apollo:middleware'), apolloChannel: dc.tracingChannel('datadog:apollo:request'), diff --git a/packages/dd-trace/src/appsec/iast/taint-tracking/plugin.js b/packages/dd-trace/src/appsec/iast/taint-tracking/plugin.js index 48902323bec..67e99ff7fb0 100644 --- a/packages/dd-trace/src/appsec/iast/taint-tracking/plugin.js +++ b/packages/dd-trace/src/appsec/iast/taint-tracking/plugin.js @@ -26,15 +26,22 @@ class TaintTrackingPlugin extends SourceIastPlugin { } onConfigure () { + const onRequestBody = ({ req }) => { + const iastContext = getIastContext(storage.getStore()) + if (iastContext && iastContext.body !== req.body) { + this._taintTrackingHandler(HTTP_REQUEST_BODY, req, 'body', iastContext) + iastContext.body = req.body + } + } + this.addSub( { channelName: 'datadog:body-parser:read:finish', tag: HTTP_REQUEST_BODY }, - ({ req }) => { - const iastContext = getIastContext(storage.getStore()) - if (iastContext && iastContext.body !== req.body) { - this._taintTrackingHandler(HTTP_REQUEST_BODY, req, 'body', iastContext) - iastContext.body = req.body - } - } + onRequestBody + ) + + this.addSub( + { channelName: 'datadog:multer:read:finish', tag: HTTP_REQUEST_BODY }, + onRequestBody ) this.addSub( diff --git a/packages/dd-trace/src/appsec/iast/vulnerability-reporter.js b/packages/dd-trace/src/appsec/iast/vulnerability-reporter.js index cc25d51b1e9..e2d1619b118 100644 --- a/packages/dd-trace/src/appsec/iast/vulnerability-reporter.js +++ b/packages/dd-trace/src/appsec/iast/vulnerability-reporter.js @@ -1,10 +1,11 @@ 'use strict' -const { MANUAL_KEEP } = require('../../../../../ext/tags') const LRU = require('lru-cache') const vulnerabilitiesFormatter = require('./vulnerabilities-formatter') const { IAST_ENABLED_TAG_KEY, IAST_JSON_TAG_KEY } = require('./tags') const standalone = require('../standalone') +const { SAMPLING_MECHANISM_APPSEC } = require('../../constants') +const { keepTrace } = require('../../priority_sampler') const VULNERABILITIES_KEY = 'vulnerabilities' const VULNERABILITY_HASHES_MAX_SIZE = 1000 @@ -56,9 +57,10 @@ function sendVulnerabilities (vulnerabilities, rootSpan) { const tags = {} // TODO: Store this outside of the span and set the tag in the exporter. tags[IAST_JSON_TAG_KEY] = JSON.stringify(jsonToSend) - tags[MANUAL_KEEP] = 'true' span.addTags(tags) + keepTrace(span, SAMPLING_MECHANISM_APPSEC) + standalone.sample(span) if (!rootSpan) span.finish() diff --git a/packages/dd-trace/src/appsec/index.js b/packages/dd-trace/src/appsec/index.js index f3656e459e8..f4f9a4db036 100644 --- a/packages/dd-trace/src/appsec/index.js +++ b/packages/dd-trace/src/appsec/index.js @@ -6,6 +6,7 @@ const remoteConfig = require('./remote_config') const { bodyParser, cookieParser, + multerParser, incomingHttpRequestStart, incomingHttpRequestEnd, passportVerify, @@ -58,6 +59,7 @@ function enable (_config) { apiSecuritySampler.configure(_config.appsec) bodyParser.subscribe(onRequestBodyParsed) + multerParser.subscribe(onRequestBodyParsed) cookieParser.subscribe(onRequestCookieParser) incomingHttpRequestStart.subscribe(incomingHttpStartTranslator) incomingHttpRequestEnd.subscribe(incomingHttpEndTranslator) @@ -299,6 +301,7 @@ function disable () { // Channel#unsubscribe() is undefined for non active channels if (bodyParser.hasSubscribers) bodyParser.unsubscribe(onRequestBodyParsed) + if (multerParser.hasSubscribers) multerParser.unsubscribe(onRequestBodyParsed) if (cookieParser.hasSubscribers) cookieParser.unsubscribe(onRequestCookieParser) if (incomingHttpRequestStart.hasSubscribers) incomingHttpRequestStart.unsubscribe(incomingHttpStartTranslator) if (incomingHttpRequestEnd.hasSubscribers) incomingHttpRequestEnd.unsubscribe(incomingHttpEndTranslator) diff --git a/packages/dd-trace/src/appsec/rasp/ssrf.js b/packages/dd-trace/src/appsec/rasp/ssrf.js index ae45ed7daf2..38a3c150d74 100644 --- a/packages/dd-trace/src/appsec/rasp/ssrf.js +++ b/packages/dd-trace/src/appsec/rasp/ssrf.js @@ -1,5 +1,6 @@ 'use strict' +const { format } = require('url') const { httpClientRequestStart } = require('../channels') const { storage } = require('../../../../datadog-core') const addresses = require('../addresses') @@ -20,12 +21,12 @@ function disable () { function analyzeSsrf (ctx) { const store = storage.getStore() const req = store?.req - const url = ctx.args.uri + const outgoingUrl = (ctx.args.options?.uri && format(ctx.args.options.uri)) ?? ctx.args.uri - if (!req || !url) return + if (!req || !outgoingUrl) return const persistent = { - [addresses.HTTP_OUTGOING_URL]: url + [addresses.HTTP_OUTGOING_URL]: outgoingUrl } const result = waf.run({ persistent }, req, RULE_TYPES.SSRF) diff --git a/packages/dd-trace/src/appsec/recommended.json b/packages/dd-trace/src/appsec/recommended.json index 158c33a8ccd..01156e6f206 100644 --- a/packages/dd-trace/src/appsec/recommended.json +++ b/packages/dd-trace/src/appsec/recommended.json @@ -1,7 +1,7 @@ { "version": "2.2", "metadata": { - "rules_version": "1.13.1" + "rules_version": "1.13.2" }, "rules": [ { @@ -6335,7 +6335,6 @@ { "id": "rasp-934-100", "name": "Server-side request forgery exploit", - "enabled": false, "tags": { "type": "ssrf", "category": "vulnerability_trigger", @@ -6384,7 +6383,6 @@ { "id": "rasp-942-100", "name": "SQL injection exploit", - "enabled": false, "tags": { "type": "sql_injection", "category": "vulnerability_trigger", @@ -6424,7 +6422,7 @@ } ] }, - "operator": "sqli_detector" + "operator": "sqli_detector@v2" } ], "transformers": [], diff --git a/packages/dd-trace/src/appsec/reporter.js b/packages/dd-trace/src/appsec/reporter.js index dd2bde9fb06..3cd23b1f003 100644 --- a/packages/dd-trace/src/appsec/reporter.js +++ b/packages/dd-trace/src/appsec/reporter.js @@ -13,8 +13,9 @@ const { getRequestMetrics } = require('./telemetry') const zlib = require('zlib') -const { MANUAL_KEEP } = require('../../../../ext/tags') const standalone = require('./standalone') +const { SAMPLING_MECHANISM_APPSEC } = require('../constants') +const { keepTrace } = require('../priority_sampler') // default limiter, configurable with setRateLimit() let limiter = new Limiter(100) @@ -96,8 +97,6 @@ function reportWafInit (wafVersion, rulesVersion, diagnosticsRules = {}) { metricsQueue.set('_dd.appsec.event_rules.errors', JSON.stringify(diagnosticsRules.errors)) } - metricsQueue.set(MANUAL_KEEP, 'true') - incrementWafInitMetric(wafVersion, rulesVersion) } @@ -129,7 +128,7 @@ function reportAttack (attackData) { } if (limiter.isAllowed()) { - newTags[MANUAL_KEEP] = 'true' + keepTrace(rootSpan, SAMPLING_MECHANISM_APPSEC) standalone.sample(rootSpan) } @@ -184,6 +183,8 @@ function finishRequest (req, res) { if (metricsQueue.size) { rootSpan.addTags(Object.fromEntries(metricsQueue)) + keepTrace(rootSpan, SAMPLING_MECHANISM_APPSEC) + standalone.sample(rootSpan) metricsQueue.clear() diff --git a/packages/dd-trace/src/appsec/sdk/track_event.js b/packages/dd-trace/src/appsec/sdk/track_event.js index 36c40093b19..e95081314de 100644 --- a/packages/dd-trace/src/appsec/sdk/track_event.js +++ b/packages/dd-trace/src/appsec/sdk/track_event.js @@ -2,10 +2,11 @@ const log = require('../../log') const { getRootSpan } = require('./utils') -const { MANUAL_KEEP } = require('../../../../../ext/tags') const { setUserTags } = require('./set_user') const standalone = require('../standalone') const waf = require('../waf') +const { SAMPLING_MECHANISM_APPSEC } = require('../../constants') +const { keepTrace } = require('../../priority_sampler') function trackUserLoginSuccessEvent (tracer, user, metadata) { // TODO: better user check here and in _setUser() ? @@ -55,9 +56,10 @@ function trackEvent (eventName, fields, sdkMethodName, rootSpan, mode) { return } + keepTrace(rootSpan, SAMPLING_MECHANISM_APPSEC) + const tags = { - [`appsec.events.${eventName}.track`]: 'true', - [MANUAL_KEEP]: 'true' + [`appsec.events.${eventName}.track`]: 'true' } if (mode === 'sdk') { diff --git a/packages/dd-trace/src/appsec/waf/waf_manager.js b/packages/dd-trace/src/appsec/waf/waf_manager.js index 8d044764705..b3cc91e6104 100644 --- a/packages/dd-trace/src/appsec/waf/waf_manager.js +++ b/packages/dd-trace/src/appsec/waf/waf_manager.js @@ -51,6 +51,10 @@ class WAFManager { update (newRules) { this.ddwaf.update(newRules) + if (this.ddwaf.diagnostics.ruleset_version) { + this.rulesVersion = this.ddwaf.diagnostics.ruleset_version + } + Reporter.reportWafUpdate(this.ddwafVersion, this.rulesVersion) } diff --git a/packages/dd-trace/src/azure_metadata.js b/packages/dd-trace/src/azure_metadata.js new file mode 100644 index 00000000000..94c29c9dd16 --- /dev/null +++ b/packages/dd-trace/src/azure_metadata.js @@ -0,0 +1,120 @@ +'use strict' + +// eslint-disable-next-line max-len +// Modeled after https://github.com/DataDog/libdatadog/blob/f3994857a59bb5679a65967138c5a3aec418a65f/ddcommon/src/azure_app_services.rs + +const os = require('os') +const { getIsAzureFunction } = require('./serverless') + +function extractSubscriptionID (ownerName) { + if (ownerName !== undefined) { + const subId = ownerName.split('+')[0].trim() + if (subId.length > 0) { + return subId + } + } + return undefined +} + +function extractResourceGroup (ownerName) { + return /.+\+(.+)-.+webspace(-Linux)?/.exec(ownerName)?.[1] +} + +function buildResourceID (subscriptionID, siteName, resourceGroup) { + if (subscriptionID === undefined || siteName === undefined || resourceGroup === undefined) { + return undefined + } + return `/subscriptions/${subscriptionID}/resourcegroups/${resourceGroup}/providers/microsoft.web/sites/${siteName}` + .toLowerCase() +} + +function trimObject (obj) { + Object.entries(obj) + .filter(([_, value]) => value === undefined) + .forEach(([key, _]) => { delete obj[key] }) + return obj +} + +function buildMetadata () { + const { + COMPUTERNAME, + DD_AAS_DOTNET_EXTENSION_VERSION, + FUNCTIONS_EXTENSION_VERSION, + FUNCTIONS_WORKER_RUNTIME, + FUNCTIONS_WORKER_RUNTIME_VERSION, + WEBSITE_INSTANCE_ID, + WEBSITE_OWNER_NAME, + WEBSITE_OS, + WEBSITE_RESOURCE_GROUP, + WEBSITE_SITE_NAME + } = process.env + + const subscriptionID = extractSubscriptionID(WEBSITE_OWNER_NAME) + + const siteName = WEBSITE_SITE_NAME + + const [siteKind, siteType] = getIsAzureFunction() + ? ['functionapp', 'function'] + : ['app', 'app'] + + const resourceGroup = WEBSITE_RESOURCE_GROUP ?? extractResourceGroup(WEBSITE_OWNER_NAME) + + return trimObject({ + extensionVersion: DD_AAS_DOTNET_EXTENSION_VERSION, + functionRuntimeVersion: FUNCTIONS_EXTENSION_VERSION, + instanceID: WEBSITE_INSTANCE_ID, + instanceName: COMPUTERNAME, + operatingSystem: WEBSITE_OS ?? os.platform(), + resourceGroup, + resourceID: buildResourceID(subscriptionID, siteName, resourceGroup), + runtime: FUNCTIONS_WORKER_RUNTIME, + runtimeVersion: FUNCTIONS_WORKER_RUNTIME_VERSION, + siteKind, + siteName, + siteType, + subscriptionID + }) +} + +function getAzureAppMetadata () { + // DD_AZURE_APP_SERVICES is an environment variable introduced by the .NET APM team and is set automatically for + // anyone using the Datadog APM Extensions (.NET, Java, or Node) for Windows Azure App Services + // eslint-disable-next-line max-len + // See: https://github.com/DataDog/datadog-aas-extension/blob/01f94b5c28b7fa7a9ab264ca28bd4e03be603900/node/src/applicationHost.xdt#L20-L21 + return process.env.DD_AZURE_APP_SERVICES !== undefined ? buildMetadata() : undefined +} + +function getAzureFunctionMetadata () { + return getIsAzureFunction() ? buildMetadata() : undefined +} + +// eslint-disable-next-line max-len +// Modeled after https://github.com/DataDog/libdatadog/blob/92272e90a7919f07178f3246ef8f82295513cfed/profiling/src/exporter/mod.rs#L187 +// eslint-disable-next-line max-len +// and https://github.com/DataDog/libdatadog/blob/f3994857a59bb5679a65967138c5a3aec418a65f/trace-utils/src/trace_utils.rs#L533 +function getAzureTagsFromMetadata (metadata) { + if (metadata === undefined) { + return {} + } + return trimObject({ + 'aas.environment.extension_version': metadata.extensionVersion, + 'aas.environment.function_runtime': metadata.functionRuntimeVersion, + 'aas.environment.instance_id': metadata.instanceID, + 'aas.environment.instance_name': metadata.instanceName, + 'aas.environment.os': metadata.operatingSystem, + 'aas.environment.runtime': metadata.runtime, + 'aas.environment.runtime_version': metadata.runtimeVersion, + 'aas.resource.group': metadata.resourceGroup, + 'aas.resource.id': metadata.resourceID, + 'aas.site.kind': metadata.siteKind, + 'aas.site.name': metadata.siteName, + 'aas.site.type': metadata.siteType, + 'aas.subscription.id': metadata.subscriptionID + }) +} + +module.exports = { + getAzureAppMetadata, + getAzureFunctionMetadata, + getAzureTagsFromMetadata +} diff --git a/packages/dd-trace/src/ci-visibility/exporters/agent-proxy/index.js b/packages/dd-trace/src/ci-visibility/exporters/agent-proxy/index.js index bb1367057f4..991031dd3e4 100644 --- a/packages/dd-trace/src/ci-visibility/exporters/agent-proxy/index.js +++ b/packages/dd-trace/src/ci-visibility/exporters/agent-proxy/index.js @@ -7,6 +7,7 @@ const CiVisibilityExporter = require('../ci-visibility-exporter') const AGENT_EVP_PROXY_PATH_PREFIX = '/evp_proxy/v' const AGENT_EVP_PROXY_PATH_REGEX = /\/evp_proxy\/v(\d+)\/?/ +const AGENT_DEBUGGER_INPUT = '/debugger/v1/input' function getLatestEvpProxyVersion (err, agentInfo) { if (err) { @@ -24,6 +25,10 @@ function getLatestEvpProxyVersion (err, agentInfo) { }, 0) } +function getCanForwardDebuggerLogs (err, agentInfo) { + return !err && agentInfo.endpoints.some(endpoint => endpoint === AGENT_DEBUGGER_INPUT) +} + class AgentProxyCiVisibilityExporter extends CiVisibilityExporter { constructor (config) { super(config) @@ -33,7 +38,8 @@ class AgentProxyCiVisibilityExporter extends CiVisibilityExporter { prioritySampler, lookup, protocolVersion, - headers + headers, + isTestDynamicInstrumentationEnabled } = config this.getAgentInfo((err, agentInfo) => { @@ -60,6 +66,18 @@ class AgentProxyCiVisibilityExporter extends CiVisibilityExporter { url: this._url, evpProxyPrefix }) + if (isTestDynamicInstrumentationEnabled) { + const canFowardLogs = getCanForwardDebuggerLogs(err, agentInfo) + if (canFowardLogs) { + const DynamicInstrumentationLogsWriter = require('../agentless/di-logs-writer') + this._logsWriter = new DynamicInstrumentationLogsWriter({ + url: this._url, + tags, + isAgentProxy: true + }) + this._canForwardLogs = true + } + } } else { this._writer = new AgentWriter({ url: this._url, diff --git a/packages/dd-trace/src/ci-visibility/exporters/agentless/di-logs-writer.js b/packages/dd-trace/src/ci-visibility/exporters/agentless/di-logs-writer.js new file mode 100644 index 00000000000..eebc3c5e6a9 --- /dev/null +++ b/packages/dd-trace/src/ci-visibility/exporters/agentless/di-logs-writer.js @@ -0,0 +1,53 @@ +'use strict' +const request = require('../../../exporters/common/request') +const log = require('../../../log') +const { safeJSONStringify } = require('../../../exporters/common/util') +const { JSONEncoder } = require('../../encode/json-encoder') + +const BaseWriter = require('../../../exporters/common/writer') + +// Writer used by the integration between Dynamic Instrumentation and Test Visibility +// It is used to encode and send logs to both the logs intake directly and the +// `/debugger/v1/input` endpoint in the agent, which is a proxy to the logs intake. +class DynamicInstrumentationLogsWriter extends BaseWriter { + constructor ({ url, timeout, isAgentProxy = false }) { + super(...arguments) + this._url = url + this._encoder = new JSONEncoder() + this._isAgentProxy = isAgentProxy + this.timeout = timeout + } + + _sendPayload (data, _, done) { + const options = { + path: '/api/v2/logs', + method: 'POST', + headers: { + 'dd-api-key': process.env.DATADOG_API_KEY || process.env.DD_API_KEY, + 'Content-Type': 'application/json' + }, + // TODO: what's a good value for timeout for the logs intake? + timeout: this.timeout || 15000, + url: this._url + } + + if (this._isAgentProxy) { + delete options.headers['dd-api-key'] + options.path = '/debugger/v1/input' + } + + log.debug(() => `Request to the logs intake: ${safeJSONStringify(options)}`) + + request(data, options, (err, res) => { + if (err) { + log.error(err) + done() + return + } + log.debug(`Response from the logs intake: ${res}`) + done() + }) + } +} + +module.exports = DynamicInstrumentationLogsWriter diff --git a/packages/dd-trace/src/ci-visibility/exporters/agentless/index.js b/packages/dd-trace/src/ci-visibility/exporters/agentless/index.js index dcbded6a54e..5895bb573cd 100644 --- a/packages/dd-trace/src/ci-visibility/exporters/agentless/index.js +++ b/packages/dd-trace/src/ci-visibility/exporters/agentless/index.js @@ -9,10 +9,11 @@ const log = require('../../../log') class AgentlessCiVisibilityExporter extends CiVisibilityExporter { constructor (config) { super(config) - const { tags, site, url } = config + const { tags, site, url, isTestDynamicInstrumentationEnabled } = config // we don't need to request /info because we are using agentless by configuration this._isInitialized = true this._resolveCanUseCiVisProtocol(true) + this._canForwardLogs = true this._url = url || new URL(`https://citestcycle-intake.${site}`) this._writer = new Writer({ url: this._url, tags }) @@ -20,6 +21,12 @@ class AgentlessCiVisibilityExporter extends CiVisibilityExporter { this._coverageUrl = url || new URL(`https://citestcov-intake.${site}`) this._coverageWriter = new CoverageWriter({ url: this._coverageUrl }) + if (isTestDynamicInstrumentationEnabled) { + const DynamicInstrumentationLogsWriter = require('./di-logs-writer') + this._logsUrl = url || new URL(`https://http-intake.logs.${site}`) + this._logsWriter = new DynamicInstrumentationLogsWriter({ url: this._logsUrl, tags }) + } + this._apiUrl = url || new URL(`https://api.${site}`) // Agentless is always gzip compatible this._isGzipCompatible = true diff --git a/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js b/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js index 9dabd34f7f3..f555603e0cb 100644 --- a/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js +++ b/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js @@ -8,6 +8,7 @@ const { getSkippableSuites: getSkippableSuitesRequest } = require('../intelligen const { getKnownTests: getKnownTestsRequest } = require('../early-flake-detection/get-known-tests') const log = require('../../log') const AgentInfoExporter = require('../../exporters/common/agent-info-exporter') +const { GIT_REPOSITORY_URL, GIT_COMMIT_SHA } = require('../../plugins/util/tags') function getTestConfigurationTags (tags) { if (!tags) { @@ -36,6 +37,7 @@ class CiVisibilityExporter extends AgentInfoExporter { super(config) this._timer = undefined this._coverageTimer = undefined + this._logsTimer = undefined this._coverageBuffer = [] // The library can use new features like ITR and test suite level visibility // AKA CI Vis Protocol @@ -255,6 +257,47 @@ class CiVisibilityExporter extends AgentInfoExporter { this._export(formattedCoverage, this._coverageWriter, '_coverageTimer') } + formatLogMessage (testConfiguration, logMessage) { + const { + [GIT_REPOSITORY_URL]: gitRepositoryUrl, + [GIT_COMMIT_SHA]: gitCommitSha + } = testConfiguration + + const { service, env, version } = this._config + + return { + ddtags: [ + ...(logMessage.ddtags || []), + `${GIT_REPOSITORY_URL}:${gitRepositoryUrl}`, + `${GIT_COMMIT_SHA}:${gitCommitSha}` + ].join(','), + level: 'error', + service, + dd: { + ...(logMessage.dd || []), + service, + env, + version + }, + ddsource: 'dd_debugger', + ...logMessage + } + } + + // DI logs + exportDiLogs (testConfiguration, logMessage) { + // TODO: could we lose logs if it's not initialized? + if (!this._config.isTestDynamicInstrumentationEnabled || !this._isInitialized || !this._canForwardLogs) { + return + } + + this._export( + this.formatLogMessage(testConfiguration, logMessage), + this._logsWriter, + '_logsTimer' + ) + } + flush (done = () => {}) { if (!this._isInitialized) { return done() diff --git a/packages/dd-trace/src/config.js b/packages/dd-trace/src/config.js index e827d1b6d0f..5a9ec19f4a2 100644 --- a/packages/dd-trace/src/config.js +++ b/packages/dd-trace/src/config.js @@ -17,7 +17,7 @@ const { getGitMetadataFromGitProperties, removeUserSensitiveInfo } = require('./ const { updateConfig } = require('./telemetry') const telemetryMetrics = require('./telemetry/metrics') const { getIsGCPFunction, getIsAzureFunction } = require('./serverless') -const { ORIGIN_KEY } = require('./constants') +const { ORIGIN_KEY, GRPC_CLIENT_ERROR_STATUSES, GRPC_SERVER_ERROR_STATUSES } = require('./constants') const { appendRules } = require('./payload-tagging/config') const tracerMetrics = telemetryMetrics.manager.namespace('tracers') @@ -477,6 +477,8 @@ class Config { this._setValue(defaults, 'flushInterval', 2000) this._setValue(defaults, 'flushMinSpans', 1000) this._setValue(defaults, 'gitMetadataEnabled', true) + this._setValue(defaults, 'grpc.client.error.statuses', GRPC_CLIENT_ERROR_STATUSES) + this._setValue(defaults, 'grpc.server.error.statuses', GRPC_SERVER_ERROR_STATUSES) this._setValue(defaults, 'headerTags', []) this._setValue(defaults, 'hostname', '127.0.0.1') this._setValue(defaults, 'iast.cookieFilterPattern', '.{32,}') @@ -499,8 +501,12 @@ class Config { this._setValue(defaults, 'isGitUploadEnabled', false) this._setValue(defaults, 'isIntelligentTestRunnerEnabled', false) this._setValue(defaults, 'isManualApiEnabled', false) + this._setValue(defaults, 'llmobs.agentlessEnabled', false) + this._setValue(defaults, 'llmobs.enabled', false) + this._setValue(defaults, 'llmobs.mlApp', undefined) this._setValue(defaults, 'ciVisibilityTestSessionName', '') this._setValue(defaults, 'ciVisAgentlessLogSubmissionEnabled', false) + this._setValue(defaults, 'isTestDynamicInstrumentationEnabled', false) this._setValue(defaults, 'logInjection', false) this._setValue(defaults, 'lookup', undefined) this._setValue(defaults, 'memcachedCommandEnabled', false) @@ -520,7 +526,7 @@ class Config { this._setValue(defaults, 'reportHostname', false) this._setValue(defaults, 'runtimeMetrics', false) this._setValue(defaults, 'sampleRate', undefined) - this._setValue(defaults, 'sampler.rateLimit', undefined) + this._setValue(defaults, 'sampler.rateLimit', 100) this._setValue(defaults, 'sampler.rules', []) this._setValue(defaults, 'sampler.spanSamplingRules', []) this._setValue(defaults, 'scope', undefined) @@ -541,6 +547,7 @@ class Config { this._setValue(defaults, 'telemetry.heartbeatInterval', 60000) this._setValue(defaults, 'telemetry.logCollection', false) this._setValue(defaults, 'telemetry.metrics', true) + this._setValue(defaults, 'traceEnabled', true) this._setValue(defaults, 'traceId128BitGenerationEnabled', true) this._setValue(defaults, 'traceId128BitLoggingEnabled', false) this._setValue(defaults, 'tracePropagationExtractFirst', false) @@ -584,6 +591,8 @@ class Config { DD_EXPERIMENTAL_API_SECURITY_ENABLED, DD_EXPERIMENTAL_APPSEC_STANDALONE_ENABLED, DD_EXPERIMENTAL_PROFILING_ENABLED, + DD_GRPC_CLIENT_ERROR_STATUSES, + DD_GRPC_SERVER_ERROR_STATUSES, JEST_WORKER_ID, DD_IAST_COOKIE_FILTER_PATTERN, DD_IAST_DEDUPLICATION_ENABLED, @@ -599,6 +608,9 @@ class Config { DD_INSTRUMENTATION_TELEMETRY_ENABLED, DD_INSTRUMENTATION_CONFIG_ID, DD_LOGS_INJECTION, + DD_LLMOBS_AGENTLESS_ENABLED, + DD_LLMOBS_ENABLED, + DD_LLMOBS_ML_APP, DD_OPENAI_LOGS_ENABLED, DD_OPENAI_SPAN_CHAR_LIMIT, DD_PROFILING_ENABLED, @@ -627,6 +639,7 @@ class Config { DD_TRACE_AGENT_PROTOCOL_VERSION, DD_TRACE_CLIENT_IP_ENABLED, DD_TRACE_CLIENT_IP_HEADER, + DD_TRACE_ENABLED, DD_TRACE_EXPERIMENTAL_EXPORTER, DD_TRACE_EXPERIMENTAL_GET_RUM_DATA_ENABLED, DD_TRACE_EXPERIMENTAL_RUNTIME_ID_ENABLED, @@ -713,6 +726,7 @@ class Config { this._setBoolean(env, 'dsmEnabled', DD_DATA_STREAMS_ENABLED) this._setBoolean(env, 'dynamicInstrumentationEnabled', DD_DYNAMIC_INSTRUMENTATION_ENABLED) this._setString(env, 'env', DD_ENV || tags.env) + this._setBoolean(env, 'traceEnabled', DD_TRACE_ENABLED) this._setBoolean(env, 'experimental.enableGetRumData', DD_TRACE_EXPERIMENTAL_GET_RUM_DATA_ENABLED) this._setString(env, 'experimental.exporter', DD_TRACE_EXPERIMENTAL_EXPORTER) this._setBoolean(env, 'experimental.runtimeId', DD_TRACE_EXPERIMENTAL_RUNTIME_ID_ENABLED) @@ -720,6 +734,8 @@ class Config { this._setValue(env, 'flushMinSpans', maybeInt(DD_TRACE_PARTIAL_FLUSH_MIN_SPANS)) this._envUnprocessed.flushMinSpans = DD_TRACE_PARTIAL_FLUSH_MIN_SPANS this._setBoolean(env, 'gitMetadataEnabled', DD_TRACE_GIT_METADATA_ENABLED) + this._setIntegerRangeSet(env, 'grpc.client.error.statuses', DD_GRPC_CLIENT_ERROR_STATUSES) + this._setIntegerRangeSet(env, 'grpc.server.error.statuses', DD_GRPC_SERVER_ERROR_STATUSES) this._setArray(env, 'headerTags', DD_TRACE_HEADER_TAGS) this._setString(env, 'hostname', coalesce(DD_AGENT_HOST, DD_TRACE_AGENT_HOSTNAME)) this._setString(env, 'iast.cookieFilterPattern', DD_IAST_COOKIE_FILTER_PATTERN) @@ -741,6 +757,9 @@ class Config { this._setArray(env, 'injectionEnabled', DD_INJECTION_ENABLED) this._setBoolean(env, 'isAzureFunction', getIsAzureFunction()) this._setBoolean(env, 'isGCPFunction', getIsGCPFunction()) + this._setBoolean(env, 'llmobs.agentlessEnabled', DD_LLMOBS_AGENTLESS_ENABLED) + this._setBoolean(env, 'llmobs.enabled', DD_LLMOBS_ENABLED) + this._setString(env, 'llmobs.mlApp', DD_LLMOBS_ML_APP) this._setBoolean(env, 'logInjection', DD_LOGS_INJECTION) // Requires an accompanying DD_APM_OBFUSCATION_MEMCACHED_KEEP_COMMAND=true in the agent this._setBoolean(env, 'memcachedCommandEnabled', DD_TRACE_MEMCACHED_COMMAND_ENABLED) @@ -911,6 +930,8 @@ class Config { } this._setString(opts, 'iast.telemetryVerbosity', options.iast && options.iast.telemetryVerbosity) this._setBoolean(opts, 'isCiVisibility', options.isCiVisibility) + this._setBoolean(opts, 'llmobs.agentlessEnabled', options.llmobs?.agentlessEnabled) + this._setString(opts, 'llmobs.mlApp', options.llmobs?.mlApp) this._setBoolean(opts, 'logInjection', options.logInjection) this._setString(opts, 'lookup', options.lookup) this._setBoolean(opts, 'openAiLogsEnabled', options.openAiLogsEnabled) @@ -946,6 +967,15 @@ class Config { this._setBoolean(opts, 'traceId128BitGenerationEnabled', options.traceId128BitGenerationEnabled) this._setBoolean(opts, 'traceId128BitLoggingEnabled', options.traceId128BitLoggingEnabled) this._setString(opts, 'version', options.version || tags.version) + + // For LLMObs, we want the environment variable to take precedence over the options. + // This is reliant on environment config being set before options. + // This is to make sure the origins of each value are tracked appropriately for telemetry. + // We'll only set `llmobs.enabled` on the opts when it's not set on the environment, and options.llmobs is provided. + const llmobsEnabledEnv = this._env['llmobs.enabled'] + if (llmobsEnabledEnv == null && options.llmobs) { + this._setBoolean(opts, 'llmobs.enabled', !!options.llmobs) + } } _isCiVisibility () { @@ -1045,7 +1075,8 @@ class Config { DD_CIVISIBILITY_FLAKY_RETRY_ENABLED, DD_CIVISIBILITY_FLAKY_RETRY_COUNT, DD_TEST_SESSION_NAME, - DD_AGENTLESS_LOG_SUBMISSION_ENABLED + DD_AGENTLESS_LOG_SUBMISSION_ENABLED, + DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED } = process.env if (DD_CIVISIBILITY_AGENTLESS_URL) { @@ -1063,6 +1094,7 @@ class Config { this._setBoolean(calc, 'isManualApiEnabled', !isFalse(this._isCiVisibilityManualApiEnabled())) this._setString(calc, 'ciVisibilityTestSessionName', DD_TEST_SESSION_NAME) this._setBoolean(calc, 'ciVisAgentlessLogSubmissionEnabled', isTrue(DD_AGENTLESS_LOG_SUBMISSION_ENABLED)) + this._setBoolean(calc, 'isTestDynamicInstrumentationEnabled', isTrue(DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED)) } this._setString(calc, 'dogstatsd.hostname', this._getHostname()) this._setBoolean(calc, 'isGitUploadEnabled', @@ -1155,7 +1187,11 @@ class Config { } if (typeof value === 'string') { - value = value.split(',') + value = value.split(',').map(item => { + // Trim each item and remove whitespace around the colon + const [key, val] = item.split(':').map(part => part.trim()) + return val !== undefined ? `${key}:${val}` : key + }) } if (Array.isArray(value)) { @@ -1163,6 +1199,26 @@ class Config { } } + _setIntegerRangeSet (obj, name, value) { + if (value == null) { + return this._setValue(obj, name, null) + } + value = value.split(',') + const result = [] + + value.forEach(val => { + if (val.includes('-')) { + const [start, end] = val.split('-').map(Number) + for (let i = start; i <= end; i++) { + result.push(i) + } + } else { + result.push(Number(val)) + } + }) + this._setValue(obj, name, result) + } + _setSamplingRule (obj, name, value) { if (value == null) { return this._setValue(obj, name, null) diff --git a/packages/dd-trace/src/constants.js b/packages/dd-trace/src/constants.js index 61f5b705ddb..a242f717a37 100644 --- a/packages/dd-trace/src/constants.js +++ b/packages/dd-trace/src/constants.js @@ -44,5 +44,7 @@ module.exports = { SCHEMA_ID: 'schema.id', SCHEMA_TOPIC: 'schema.topic', SCHEMA_OPERATION: 'schema.operation', - SCHEMA_NAME: 'schema.name' + SCHEMA_NAME: 'schema.name', + GRPC_CLIENT_ERROR_STATUSES: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], + GRPC_SERVER_ERROR_STATUSES: [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] } diff --git a/packages/dd-trace/src/datastreams/pathway.js b/packages/dd-trace/src/datastreams/pathway.js index 066af789e64..ed2f6cc85f8 100644 --- a/packages/dd-trace/src/datastreams/pathway.js +++ b/packages/dd-trace/src/datastreams/pathway.js @@ -21,6 +21,7 @@ function shaHash (checkpointString) { } function computeHash (service, env, edgeTags, parentHash) { + edgeTags.sort() const hashableEdgeTags = edgeTags.filter(item => item !== 'manual_checkpoint:true') const key = `${service}${env}` + hashableEdgeTags.join('') + parentHash.toString() diff --git a/packages/dd-trace/src/debugger/devtools_client/index.js b/packages/dd-trace/src/debugger/devtools_client/index.js index aa19c14ef64..1228a9af823 100644 --- a/packages/dd-trace/src/debugger/devtools_client/index.js +++ b/packages/dd-trace/src/debugger/devtools_client/index.js @@ -23,12 +23,14 @@ session.on('Debugger.paused', async ({ params }) => { const timestamp = Date.now() let captureSnapshotForProbe = null - let maxReferenceDepth, maxLength + let maxReferenceDepth, maxCollectionSize, maxFieldCount, maxLength const probes = params.hitBreakpoints.map((id) => { const probe = breakpoints.get(id) if (probe.captureSnapshot) { captureSnapshotForProbe = probe maxReferenceDepth = highestOrUndefined(probe.capture.maxReferenceDepth, maxReferenceDepth) + maxCollectionSize = highestOrUndefined(probe.capture.maxCollectionSize, maxCollectionSize) + maxFieldCount = highestOrUndefined(probe.capture.maxFieldCount, maxFieldCount) maxLength = highestOrUndefined(probe.capture.maxLength, maxLength) } return probe @@ -38,7 +40,10 @@ session.on('Debugger.paused', async ({ params }) => { if (captureSnapshotForProbe !== null) { try { // TODO: Create unique states for each affected probe based on that probes unique `capture` settings (DEBUG-2863) - processLocalState = await getLocalStateForCallFrame(params.callFrames[0], { maxReferenceDepth, maxLength }) + processLocalState = await getLocalStateForCallFrame( + params.callFrames[0], + { maxReferenceDepth, maxCollectionSize, maxFieldCount, maxLength } + ) } catch (err) { // TODO: This error is not tied to a specific probe, but to all probes with `captureSnapshot: true`. // However, in 99,99% of cases, there will be just a single probe, so I guess this simplification is ok? diff --git a/packages/dd-trace/src/debugger/devtools_client/send.js b/packages/dd-trace/src/debugger/devtools_client/send.js index 593c3ea235d..f2ba5befd46 100644 --- a/packages/dd-trace/src/debugger/devtools_client/send.js +++ b/packages/dd-trace/src/debugger/devtools_client/send.js @@ -9,6 +9,8 @@ const { GIT_COMMIT_SHA, GIT_REPOSITORY_URL } = require('../../plugins/util/tags' module.exports = send +const MAX_PAYLOAD_SIZE = 1024 * 1024 // 1MB + const ddsource = 'dd_debugger' const hostname = getHostname() const service = config.service @@ -37,5 +39,17 @@ function send (message, logger, snapshot, cb) { 'debugger.snapshot': snapshot } - request(JSON.stringify(payload), opts, cb) + let json = JSON.stringify(payload) + + if (Buffer.byteLength(json) > MAX_PAYLOAD_SIZE) { + // TODO: This is a very crude way to handle large payloads. Proper pruning will be implemented later (DEBUG-2624) + const line = Object.values(payload['debugger.snapshot'].captures.lines)[0] + line.locals = { + notCapturedReason: 'Snapshot was too large', + size: Object.keys(line.locals).length + } + json = JSON.stringify(payload) + } + + request(json, opts, cb) } diff --git a/packages/dd-trace/src/debugger/devtools_client/snapshot/collector.js b/packages/dd-trace/src/debugger/devtools_client/snapshot/collector.js index 0a8848ce5e5..77f59173743 100644 --- a/packages/dd-trace/src/debugger/devtools_client/snapshot/collector.js +++ b/packages/dd-trace/src/debugger/devtools_client/snapshot/collector.js @@ -1,5 +1,6 @@ 'use strict' +const { collectionSizeSym, fieldCountSym } = require('./symbols') const session = require('../session') const LEAF_SUBTYPES = new Set(['date', 'regexp']) @@ -14,22 +15,38 @@ module.exports = { // each lookup will just finish in its own time and traverse the child nodes when the event loop allows it. // Alternatively, use `Promise.all` or something like that, but the code would probably be more complex. -async function getObject (objectId, maxDepth, depth = 0) { +async function getObject (objectId, opts, depth = 0, collection = false) { const { result, privateProperties } = await session.post('Runtime.getProperties', { objectId, ownProperties: true // exclude inherited properties }) - if (privateProperties) result.push(...privateProperties) + if (collection) { + // Trim the collection if it's too large. + // Collections doesn't contain private properties, so the code in this block doesn't have to deal with it. + removeNonEnumerableProperties(result) // remove the `length` property + const size = result.length + if (size > opts.maxCollectionSize) { + result.splice(opts.maxCollectionSize) + result[collectionSizeSym] = size + } + } else if (result.length > opts.maxFieldCount) { + // Trim the number of properties on the object if there's too many. + const size = result.length + result.splice(opts.maxFieldCount) + result[fieldCountSym] = size + } else if (privateProperties) { + result.push(...privateProperties) + } - return traverseGetPropertiesResult(result, maxDepth, depth) + return traverseGetPropertiesResult(result, opts, depth) } -async function traverseGetPropertiesResult (props, maxDepth, depth) { +async function traverseGetPropertiesResult (props, opts, depth) { // TODO: Decide if we should filter out non-enumerable properties or not: // props = props.filter((e) => e.enumerable) - if (depth >= maxDepth) return props + if (depth >= opts.maxReferenceDepth) return props for (const prop of props) { if (prop.value === undefined) continue @@ -37,33 +54,33 @@ async function traverseGetPropertiesResult (props, maxDepth, depth) { if (type === 'object') { if (objectId === undefined) continue // if `subtype` is "null" if (LEAF_SUBTYPES.has(subtype)) continue // don't waste time with these subtypes - prop.value.properties = await getObjectProperties(subtype, objectId, maxDepth, depth) + prop.value.properties = await getObjectProperties(subtype, objectId, opts, depth) } else if (type === 'function') { - prop.value.properties = await getFunctionProperties(objectId, maxDepth, depth + 1) + prop.value.properties = await getFunctionProperties(objectId, opts, depth + 1) } } return props } -async function getObjectProperties (subtype, objectId, maxDepth, depth) { +async function getObjectProperties (subtype, objectId, opts, depth) { if (ITERABLE_SUBTYPES.has(subtype)) { - return getIterable(objectId, maxDepth, depth) + return getIterable(objectId, opts, depth) } else if (subtype === 'promise') { - return getInternalProperties(objectId, maxDepth, depth) + return getInternalProperties(objectId, opts, depth) } else if (subtype === 'proxy') { - return getProxy(objectId, maxDepth, depth) + return getProxy(objectId, opts, depth) } else if (subtype === 'arraybuffer') { - return getArrayBuffer(objectId, maxDepth, depth) + return getArrayBuffer(objectId, opts, depth) } else { - return getObject(objectId, maxDepth, depth + 1) + return getObject(objectId, opts, depth + 1, subtype === 'array' || subtype === 'typedarray') } } // TODO: The following extra information from `internalProperties` might be relevant to include for functions: // - Bound function: `[[TargetFunction]]`, `[[BoundThis]]` and `[[BoundArgs]]` // - Non-bound function: `[[FunctionLocation]]`, and `[[Scopes]]` -async function getFunctionProperties (objectId, maxDepth, depth) { +async function getFunctionProperties (objectId, opts, depth) { let { result } = await session.post('Runtime.getProperties', { objectId, ownProperties: true // exclude inherited properties @@ -72,10 +89,12 @@ async function getFunctionProperties (objectId, maxDepth, depth) { // For legacy reasons (I assume) functions has a `prototype` property besides the internal `[[Prototype]]` result = result.filter(({ name }) => name !== 'prototype') - return traverseGetPropertiesResult(result, maxDepth, depth) + return traverseGetPropertiesResult(result, opts, depth) } -async function getIterable (objectId, maxDepth, depth) { +async function getIterable (objectId, opts, depth) { + // TODO: If the iterable has any properties defined on the object directly, instead of in its collection, they will + // exist in the return value below in the `result` property. We currently do not collect these. const { internalProperties } = await session.post('Runtime.getProperties', { objectId, ownProperties: true // exclude inherited properties @@ -93,10 +112,17 @@ async function getIterable (objectId, maxDepth, depth) { ownProperties: true // exclude inherited properties }) - return traverseGetPropertiesResult(result, maxDepth, depth) + removeNonEnumerableProperties(result) // remove the `length` property + const size = result.length + if (size > opts.maxCollectionSize) { + result.splice(opts.maxCollectionSize) + result[collectionSizeSym] = size + } + + return traverseGetPropertiesResult(result, opts, depth) } -async function getInternalProperties (objectId, maxDepth, depth) { +async function getInternalProperties (objectId, opts, depth) { const { internalProperties } = await session.post('Runtime.getProperties', { objectId, ownProperties: true // exclude inherited properties @@ -105,10 +131,10 @@ async function getInternalProperties (objectId, maxDepth, depth) { // We want all internal properties except the prototype const props = internalProperties.filter(({ name }) => name !== '[[Prototype]]') - return traverseGetPropertiesResult(props, maxDepth, depth) + return traverseGetPropertiesResult(props, opts, depth) } -async function getProxy (objectId, maxDepth, depth) { +async function getProxy (objectId, opts, depth) { const { internalProperties } = await session.post('Runtime.getProperties', { objectId, ownProperties: true // exclude inherited properties @@ -127,14 +153,14 @@ async function getProxy (objectId, maxDepth, depth) { ownProperties: true // exclude inherited properties }) - return traverseGetPropertiesResult(result, maxDepth, depth) + return traverseGetPropertiesResult(result, opts, depth) } // Support for ArrayBuffer is a bit trickly because the internal structure stored in `internalProperties` is not // documented and is not straight forward. E.g. ArrayBuffer(3) will internally contain both Int8Array(3) and // UInt8Array(3), whereas ArrayBuffer(8) internally contains both Int8Array(8), Uint8Array(8), Int16Array(4), and // Int32Array(2) - all representing the same data in different ways. -async function getArrayBuffer (objectId, maxDepth, depth) { +async function getArrayBuffer (objectId, opts, depth) { const { internalProperties } = await session.post('Runtime.getProperties', { objectId, ownProperties: true // exclude inherited properties @@ -149,5 +175,13 @@ async function getArrayBuffer (objectId, maxDepth, depth) { ownProperties: true // exclude inherited properties }) - return traverseGetPropertiesResult(result, maxDepth, depth) + return traverseGetPropertiesResult(result, opts, depth) +} + +function removeNonEnumerableProperties (props) { + for (let i = 0; i < props.length; i++) { + if (props[i].enumerable === false) { + props.splice(i--, 1) + } + } } diff --git a/packages/dd-trace/src/debugger/devtools_client/snapshot/index.js b/packages/dd-trace/src/debugger/devtools_client/snapshot/index.js index add097ac755..6b66ec76766 100644 --- a/packages/dd-trace/src/debugger/devtools_client/snapshot/index.js +++ b/packages/dd-trace/src/debugger/devtools_client/snapshot/index.js @@ -4,6 +4,8 @@ const { getRuntimeObject } = require('./collector') const { processRawState } = require('./processor') const DEFAULT_MAX_REFERENCE_DEPTH = 3 +const DEFAULT_MAX_COLLECTION_SIZE = 100 +const DEFAULT_MAX_FIELD_COUNT = 20 const DEFAULT_MAX_LENGTH = 255 module.exports = { @@ -12,14 +14,22 @@ module.exports = { async function getLocalStateForCallFrame ( callFrame, - { maxReferenceDepth = DEFAULT_MAX_REFERENCE_DEPTH, maxLength = DEFAULT_MAX_LENGTH } = {} + { + maxReferenceDepth = DEFAULT_MAX_REFERENCE_DEPTH, + maxCollectionSize = DEFAULT_MAX_COLLECTION_SIZE, + maxFieldCount = DEFAULT_MAX_FIELD_COUNT, + maxLength = DEFAULT_MAX_LENGTH + } = {} ) { const rawState = [] let processedState = null for (const scope of callFrame.scopeChain) { if (scope.type === 'global') continue // The global scope is too noisy - rawState.push(...await getRuntimeObject(scope.object.objectId, maxReferenceDepth)) + rawState.push(...await getRuntimeObject( + scope.object.objectId, + { maxReferenceDepth, maxCollectionSize, maxFieldCount } + )) } // Deplay calling `processRawState` so the caller gets a chance to resume the main thread before processing `rawState` diff --git a/packages/dd-trace/src/debugger/devtools_client/snapshot/processor.js b/packages/dd-trace/src/debugger/devtools_client/snapshot/processor.js index 2cac9ef0b1c..ea52939ab0e 100644 --- a/packages/dd-trace/src/debugger/devtools_client/snapshot/processor.js +++ b/packages/dd-trace/src/debugger/devtools_client/snapshot/processor.js @@ -1,5 +1,7 @@ 'use strict' +const { collectionSizeSym, fieldCountSym } = require('./symbols') + module.exports = { processRawState: processProperties } @@ -137,38 +139,46 @@ function toString (str, maxLength) { function toObject (type, props, maxLength) { if (props === undefined) return notCapturedDepth(type) - return { type, fields: processProperties(props, maxLength) } + + const result = { + type, + fields: processProperties(props, maxLength) + } + + if (fieldCountSym in props) { + result.notCapturedReason = 'fieldCount' + result.size = props[fieldCountSym] + } + + return result } function toArray (type, elements, maxLength) { if (elements === undefined) return notCapturedDepth(type) // Perf: Create array of expected size in advance (expect that it contains only one non-enumrable element) - const expectedLength = elements.length - 1 - const result = { type, elements: new Array(expectedLength) } + const result = { type, elements: new Array(elements.length) } + + setNotCaptureReasonOnCollection(result, elements) let i = 0 for (const elm of elements) { - if (elm.enumerable === false) continue // the value of the `length` property should not be part of the array result.elements[i++] = getPropertyValue(elm, maxLength) } - // Safe-guard in case there were more than one non-enumerable element - if (i < expectedLength) result.elements.length = i - return result } function toMap (type, pairs, maxLength) { if (pairs === undefined) return notCapturedDepth(type) - // Perf: Create array of expected size in advance (expect that it contains only one non-enumrable element) - const expectedLength = pairs.length - 1 - const result = { type, entries: new Array(expectedLength) } + // Perf: Create array of expected size in advance + const result = { type, entries: new Array(pairs.length) } + + setNotCaptureReasonOnCollection(result, pairs) let i = 0 for (const pair of pairs) { - if (pair.enumerable === false) continue // the value of the `length` property should not be part of the map // The following code is based on assumptions made when researching the output of the Chrome DevTools Protocol. // There doesn't seem to be any documentation to back it up: // @@ -180,9 +190,6 @@ function toMap (type, pairs, maxLength) { result.entries[i++] = [key, val] } - // Safe-guard in case there were more than one non-enumerable element - if (i < expectedLength) result.entries.length = i - return result } @@ -190,12 +197,12 @@ function toSet (type, values, maxLength) { if (values === undefined) return notCapturedDepth(type) // Perf: Create array of expected size in advance (expect that it contains only one non-enumrable element) - const expectedLength = values.length - 1 - const result = { type, elements: new Array(expectedLength) } + const result = { type, elements: new Array(values.length) } + + setNotCaptureReasonOnCollection(result, values) let i = 0 for (const value of values) { - if (value.enumerable === false) continue // the value of the `length` property should not be part of the set // The following code is based on assumptions made when researching the output of the Chrome DevTools Protocol. // There doesn't seem to be any documentation to back it up: // @@ -205,9 +212,6 @@ function toSet (type, values, maxLength) { result.elements[i++] = getPropertyValue(value.value.properties[0], maxLength) } - // Safe-guard in case there were more than one non-enumerable element - if (i < expectedLength) result.elements.length = i - return result } @@ -236,6 +240,13 @@ function arrayBufferToString (bytes, size) { return buf.toString() } +function setNotCaptureReasonOnCollection (result, collection) { + if (collectionSizeSym in collection) { + result.notCapturedReason = 'collectionSize' + result.size = collection[collectionSizeSym] + } +} + function notCapturedDepth (type) { return { type, notCapturedReason: 'depth' } } diff --git a/packages/dd-trace/src/debugger/devtools_client/snapshot/symbols.js b/packages/dd-trace/src/debugger/devtools_client/snapshot/symbols.js new file mode 100644 index 00000000000..66a82d0a160 --- /dev/null +++ b/packages/dd-trace/src/debugger/devtools_client/snapshot/symbols.js @@ -0,0 +1,6 @@ +'use stict' + +module.exports = { + collectionSizeSym: Symbol('datadog.collectionSize'), + fieldCountSym: Symbol('datadog.fieldCount') +} diff --git a/packages/dd-trace/src/debugger/index.js b/packages/dd-trace/src/debugger/index.js index 5db1a440cf2..3638119c6f1 100644 --- a/packages/dd-trace/src/debugger/index.js +++ b/packages/dd-trace/src/debugger/index.js @@ -6,6 +6,7 @@ const log = require('../log') let worker = null let configChannel = null +let ackId = 0 const { NODE_OPTIONS, ...env } = process.env @@ -24,13 +25,19 @@ function start (config, rc) { configChannel = new MessageChannel() rc.setProductHandler('LIVE_DEBUGGING', (action, conf, id, ack) => { - const ackId = `${id}-${conf.version}` - rcAckCallbacks.set(ackId, ack) + rcAckCallbacks.set(++ackId, ack) rcChannel.port2.postMessage({ action, conf, ackId }) }) rcChannel.port2.on('message', ({ ackId, error }) => { - rcAckCallbacks.get(ackId)(error) + const ack = rcAckCallbacks.get(ackId) + if (ack === undefined) { + // This should never happen, but just in case something changes in the future, we should guard against it + log.error(`Received an unknown ackId: ${ackId}`) + if (error) log.error(error) + return + } + ack(error) rcAckCallbacks.delete(ackId) }) rcChannel.port2.on('messageerror', (err) => log.error(err)) diff --git a/packages/dd-trace/src/llmobs/constants/tags.js b/packages/dd-trace/src/llmobs/constants/tags.js new file mode 100644 index 00000000000..eee9a6b9890 --- /dev/null +++ b/packages/dd-trace/src/llmobs/constants/tags.js @@ -0,0 +1,34 @@ +'use strict' + +module.exports = { + SPAN_KINDS: ['llm', 'agent', 'workflow', 'task', 'tool', 'embedding', 'retrieval'], + SPAN_KIND: '_ml_obs.meta.span.kind', + SESSION_ID: '_ml_obs.session_id', + METADATA: '_ml_obs.meta.metadata', + METRICS: '_ml_obs.metrics', + ML_APP: '_ml_obs.meta.ml_app', + PROPAGATED_PARENT_ID_KEY: '_dd.p.llmobs_parent_id', + PARENT_ID_KEY: '_ml_obs.llmobs_parent_id', + TAGS: '_ml_obs.tags', + NAME: '_ml_obs.name', + TRACE_ID: '_ml_obs.trace_id', + PROPAGATED_TRACE_ID_KEY: '_dd.p.llmobs_trace_id', + ROOT_PARENT_ID: 'undefined', + + MODEL_NAME: '_ml_obs.meta.model_name', + MODEL_PROVIDER: '_ml_obs.meta.model_provider', + + INPUT_DOCUMENTS: '_ml_obs.meta.input.documents', + INPUT_MESSAGES: '_ml_obs.meta.input.messages', + INPUT_VALUE: '_ml_obs.meta.input.value', + + OUTPUT_DOCUMENTS: '_ml_obs.meta.output.documents', + OUTPUT_MESSAGES: '_ml_obs.meta.output.messages', + OUTPUT_VALUE: '_ml_obs.meta.output.value', + + INPUT_TOKENS_METRIC_KEY: 'input_tokens', + OUTPUT_TOKENS_METRIC_KEY: 'output_tokens', + TOTAL_TOKENS_METRIC_KEY: 'total_tokens', + + DROPPED_IO_COLLECTION_ERROR: 'dropped_io' +} diff --git a/packages/dd-trace/src/llmobs/constants/text.js b/packages/dd-trace/src/llmobs/constants/text.js new file mode 100644 index 00000000000..3c19b9febb6 --- /dev/null +++ b/packages/dd-trace/src/llmobs/constants/text.js @@ -0,0 +1,6 @@ +'use strict' + +module.exports = { + DROPPED_VALUE_TEXT: "[This value has been dropped because this span's size exceeds the 1MB size limit.]", + UNSERIALIZABLE_VALUE_TEXT: 'Unserializable value' +} diff --git a/packages/dd-trace/src/llmobs/constants/writers.js b/packages/dd-trace/src/llmobs/constants/writers.js new file mode 100644 index 00000000000..3726c33c7c0 --- /dev/null +++ b/packages/dd-trace/src/llmobs/constants/writers.js @@ -0,0 +1,13 @@ +'use strict' + +module.exports = { + EVP_PROXY_AGENT_BASE_PATH: 'evp_proxy/v2', + EVP_PROXY_AGENT_ENDPOINT: 'evp_proxy/v2/api/v2/llmobs', + EVP_SUBDOMAIN_HEADER_NAME: 'X-Datadog-EVP-Subdomain', + EVP_SUBDOMAIN_HEADER_VALUE: 'llmobs-intake', + AGENTLESS_SPANS_ENDPOINT: '/api/v2/llmobs', + AGENTLESS_EVALULATIONS_ENDPOINT: '/api/intake/llm-obs/v1/eval-metric', + + EVP_PAYLOAD_SIZE_LIMIT: 5 << 20, // 5MB (actual limit is 5.1MB) + EVP_EVENT_SIZE_LIMIT: (1 << 20) - 1024 // 999KB (actual limit is 1MB) +} diff --git a/packages/dd-trace/src/llmobs/index.js b/packages/dd-trace/src/llmobs/index.js new file mode 100644 index 00000000000..5d33ecb4c5d --- /dev/null +++ b/packages/dd-trace/src/llmobs/index.js @@ -0,0 +1,103 @@ +'use strict' + +const log = require('../log') +const { PROPAGATED_PARENT_ID_KEY } = require('./constants/tags') +const { storage } = require('./storage') + +const LLMObsSpanProcessor = require('./span_processor') + +const { channel } = require('dc-polyfill') +const spanProcessCh = channel('dd-trace:span:process') +const evalMetricAppendCh = channel('llmobs:eval-metric:append') +const flushCh = channel('llmobs:writers:flush') +const injectCh = channel('dd-trace:span:inject') + +const LLMObsAgentlessSpanWriter = require('./writers/spans/agentless') +const LLMObsAgentProxySpanWriter = require('./writers/spans/agentProxy') +const LLMObsEvalMetricsWriter = require('./writers/evaluations') + +/** + * Setting writers and processor globally when LLMObs is enabled + * We're setting these in this module instead of on the SDK. + * This is to isolate any subscribers and periodic tasks to this module, + * and not conditionally instantiate in the SDK, since the SDK is always instantiated + * if the tracer is `init`ed. But, in those cases, we don't want to start writers or subscribe + * to channels. + */ +let spanProcessor +let spanWriter +let evalWriter + +function enable (config) { + // create writers and eval writer append and flush channels + // span writer append is handled by the span processor + evalWriter = new LLMObsEvalMetricsWriter(config) + spanWriter = createSpanWriter(config) + + evalMetricAppendCh.subscribe(handleEvalMetricAppend) + flushCh.subscribe(handleFlush) + + // span processing + spanProcessor = new LLMObsSpanProcessor(config) + spanProcessor.setWriter(spanWriter) + spanProcessCh.subscribe(handleSpanProcess) + + // distributed tracing for llmobs + injectCh.subscribe(handleLLMObsParentIdInjection) +} + +function disable () { + if (evalMetricAppendCh.hasSubscribers) evalMetricAppendCh.unsubscribe(handleEvalMetricAppend) + if (flushCh.hasSubscribers) flushCh.unsubscribe(handleFlush) + if (spanProcessCh.hasSubscribers) spanProcessCh.unsubscribe(handleSpanProcess) + if (injectCh.hasSubscribers) injectCh.unsubscribe(handleLLMObsParentIdInjection) + + spanWriter?.destroy() + evalWriter?.destroy() + spanProcessor?.setWriter(null) + + spanWriter = null + evalWriter = null +} + +// since LLMObs traces can extend between services and be the same trace, +// we need to propogate the parent id. +function handleLLMObsParentIdInjection ({ carrier }) { + const parent = storage.getStore()?.span + if (!parent) return + + const parentId = parent?.context().toSpanId() + + carrier['x-datadog-tags'] += `,${PROPAGATED_PARENT_ID_KEY}=${parentId}` +} + +function createSpanWriter (config) { + const SpanWriter = config.llmobs.agentlessEnabled ? LLMObsAgentlessSpanWriter : LLMObsAgentProxySpanWriter + return new SpanWriter(config) +} + +function handleFlush () { + try { + spanWriter.flush() + evalWriter.flush() + } catch (e) { + log.warn(`Failed to flush LLMObs spans and evaluation metrics: ${e.message}`) + } +} + +function handleSpanProcess (data) { + spanProcessor.process(data) +} + +function handleEvalMetricAppend (payload) { + try { + evalWriter.append(payload) + } catch (e) { + log.warn(` + Failed to append evaluation metric to LLM Observability writer, likely due to an unserializable property. + Evaluation metrics won't be sent to LLM Observability: ${e.message} + `) + } +} + +module.exports = { enable, disable } diff --git a/packages/dd-trace/src/llmobs/noop.js b/packages/dd-trace/src/llmobs/noop.js new file mode 100644 index 00000000000..4eba48cd51c --- /dev/null +++ b/packages/dd-trace/src/llmobs/noop.js @@ -0,0 +1,82 @@ +'use strict' + +class NoopLLMObs { + constructor (noopTracer) { + this._tracer = noopTracer + } + + get enabled () { + return false + } + + enable (options) {} + + disable () {} + + trace (options = {}, fn) { + if (typeof options === 'function') { + fn = options + options = {} + } + + const name = options.name || options.kind || fn.name + + return this._tracer.trace(name, options, fn) + } + + wrap (options = {}, fn) { + if (typeof options === 'function') { + fn = options + options = {} + } + + const name = options.name || options.kind || fn.name + + return this._tracer.wrap(name, options, fn) + } + + decorate (options = {}) { + const llmobs = this + return function (target, ctxOrPropertyKey, descriptor) { + if (!ctxOrPropertyKey) return target + if (typeof ctxOrPropertyKey === 'object') { + const ctx = ctxOrPropertyKey + if (ctx.kind !== 'method') return target + + return llmobs.wrap({ name: ctx.name, ...options }, target) + } else { + const propertyKey = ctxOrPropertyKey + if (descriptor) { + if (typeof descriptor.value !== 'function') return descriptor + + const original = descriptor.value + descriptor.value = llmobs.wrap({ name: propertyKey, ...options }, original) + + return descriptor + } else { + if (typeof target[propertyKey] !== 'function') return target[propertyKey] + + const original = target[propertyKey] + Object.defineProperty(target, propertyKey, { + ...Object.getOwnPropertyDescriptor(target, propertyKey), + value: llmobs.wrap({ name: propertyKey, ...options }, original) + }) + + return target + } + } + } + } + + annotate (span, options) {} + + exportSpan (span) { + return {} + } + + submitEvaluation (llmobsSpanContext, options) {} + + flush () {} +} + +module.exports = NoopLLMObs diff --git a/packages/dd-trace/src/llmobs/plugins/base.js b/packages/dd-trace/src/llmobs/plugins/base.js new file mode 100644 index 00000000000..f7f4d2b5e94 --- /dev/null +++ b/packages/dd-trace/src/llmobs/plugins/base.js @@ -0,0 +1,65 @@ +'use strict' + +const log = require('../../log') +const { storage } = require('../storage') + +const TracingPlugin = require('../../plugins/tracing') +const LLMObsTagger = require('../tagger') + +// we make this a `Plugin` so we don't have to worry about `finish` being called +class LLMObsPlugin extends TracingPlugin { + constructor (...args) { + super(...args) + + this._tagger = new LLMObsTagger(this._tracerConfig, true) + } + + getName () {} + + setLLMObsTags (ctx) { + throw new Error('setLLMObsTags must be implemented by the subclass') + } + + getLLMObsSPanRegisterOptions (ctx) { + throw new Error('getLLMObsSPanRegisterOptions must be implemented by the subclass') + } + + start (ctx) { + const oldStore = storage.getStore() + const parent = oldStore?.span + const span = ctx.currentStore?.span + + const registerOptions = this.getLLMObsSPanRegisterOptions(ctx) + + this._tagger.registerLLMObsSpan(span, { parent, ...registerOptions }) + } + + asyncEnd (ctx) { + // even though llmobs span events won't be enqueued if llmobs is disabled + // we should avoid doing any computations here (these listeners aren't disabled) + const enabled = this._tracerConfig.llmobs.enabled + if (!enabled) return + + const span = ctx.currentStore?.span + if (!span) { + log.debug( + `Tried to start an LLMObs span for ${this.constructor.name} without an active APM span. + Not starting LLMObs span.` + ) + return + } + + this.setLLMObsTags(ctx) + } + + configure (config) { + // we do not want to enable any LLMObs plugins if it is disabled on the tracer + const llmobsEnabled = this._tracerConfig.llmobs.enabled + if (llmobsEnabled === false) { + config = typeof config === 'boolean' ? false : { ...config, enabled: false } // override to false + } + super.configure(config) + } +} + +module.exports = LLMObsPlugin diff --git a/packages/dd-trace/src/llmobs/plugins/openai.js b/packages/dd-trace/src/llmobs/plugins/openai.js new file mode 100644 index 00000000000..431760a04f8 --- /dev/null +++ b/packages/dd-trace/src/llmobs/plugins/openai.js @@ -0,0 +1,205 @@ +'use strict' + +const LLMObsPlugin = require('./base') + +class OpenAiLLMObsPlugin extends LLMObsPlugin { + static get prefix () { + return 'tracing:apm:openai:request' + } + + getLLMObsSPanRegisterOptions (ctx) { + const resource = ctx.methodName + const methodName = gateResource(normalizeOpenAIResourceName(resource)) + if (!methodName) return // we will not trace all openai methods for llmobs + + const inputs = ctx.args[0] // completion, chat completion, and embeddings take one argument + const operation = getOperation(methodName) + const kind = operation === 'embedding' ? 'embedding' : 'llm' + const name = `openai.${methodName}` + + return { + modelProvider: 'openai', + modelName: inputs.model, + kind, + name + } + } + + setLLMObsTags (ctx) { + const span = ctx.currentStore?.span + const resource = ctx.methodName + const methodName = gateResource(normalizeOpenAIResourceName(resource)) + if (!methodName) return // we will not trace all openai methods for llmobs + + const inputs = ctx.args[0] // completion, chat completion, and embeddings take one argument + const response = ctx.result?.data // no result if error + const error = !!span.context()._tags.error + + const operation = getOperation(methodName) + + if (operation === 'completion') { + this._tagCompletion(span, inputs, response, error) + } else if (operation === 'chat') { + this._tagChatCompletion(span, inputs, response, error) + } else if (operation === 'embedding') { + this._tagEmbedding(span, inputs, response, error) + } + + if (!error) { + const metrics = this._extractMetrics(response) + this._tagger.tagMetrics(span, metrics) + } + } + + _extractMetrics (response) { + const metrics = {} + const tokenUsage = response.usage + + if (tokenUsage) { + const inputTokens = tokenUsage.prompt_tokens + if (inputTokens) metrics.inputTokens = inputTokens + + const outputTokens = tokenUsage.completion_tokens + if (outputTokens) metrics.outputTokens = outputTokens + + const totalTokens = tokenUsage.total_toksn || (inputTokens + outputTokens) + if (totalTokens) metrics.totalTokens = totalTokens + } + + return metrics + } + + _tagEmbedding (span, inputs, response, error) { + const { model, ...parameters } = inputs + + const metadata = { + encoding_format: parameters.encoding_format || 'float' + } + if (inputs.dimensions) metadata.dimensions = inputs.dimensions + this._tagger.tagMetadata(span, metadata) + + let embeddingInputs = inputs.input + if (!Array.isArray(embeddingInputs)) embeddingInputs = [embeddingInputs] + const embeddingInput = embeddingInputs.map(input => ({ text: input })) + + if (error) { + this._tagger.tagEmbeddingIO(span, embeddingInput, undefined) + return + } + + const float = Array.isArray(response.data[0].embedding) + let embeddingOutput + if (float) { + const embeddingDim = response.data[0].embedding.length + embeddingOutput = `[${response.data.length} embedding(s) returned with size ${embeddingDim}]` + } else { + embeddingOutput = `[${response.data.length} embedding(s) returned]` + } + + this._tagger.tagEmbeddingIO(span, embeddingInput, embeddingOutput) + } + + _tagCompletion (span, inputs, response, error) { + let { prompt, model, ...parameters } = inputs + if (!Array.isArray(prompt)) prompt = [prompt] + + const completionInput = prompt.map(p => ({ content: p })) + + const completionOutput = error ? [{ content: '' }] : response.choices.map(choice => ({ content: choice.text })) + + this._tagger.tagLLMIO(span, completionInput, completionOutput) + this._tagger.tagMetadata(span, parameters) + } + + _tagChatCompletion (span, inputs, response, error) { + const { messages, model, ...parameters } = inputs + + if (error) { + this._tagger.tagLLMIO(span, messages, [{ content: '' }]) + return + } + + const outputMessages = [] + const { choices } = response + for (const choice of choices) { + const message = choice.message || choice.delta + const content = message.content || '' + const role = message.role + + if (message.function_call) { + const functionCallInfo = { + name: message.function_call.name, + arguments: JSON.parse(message.function_call.arguments) + } + outputMessages.push({ content, role, toolCalls: [functionCallInfo] }) + } else if (message.tool_calls) { + const toolCallsInfo = [] + for (const toolCall of message.tool_calls) { + const toolCallInfo = { + arguments: JSON.parse(toolCall.function.arguments), + name: toolCall.function.name, + toolId: toolCall.id, + type: toolCall.type + } + toolCallsInfo.push(toolCallInfo) + } + outputMessages.push({ content, role, toolCalls: toolCallsInfo }) + } else { + outputMessages.push({ content, role }) + } + } + + this._tagger.tagLLMIO(span, messages, outputMessages) + + const metadata = Object.entries(parameters).reduce((obj, [key, value]) => { + if (!['tools', 'functions'].includes(key)) { + obj[key] = value + } + + return obj + }, {}) + + this._tagger.tagMetadata(span, metadata) + } +} + +// TODO: this will be moved to the APM integration +function normalizeOpenAIResourceName (resource) { + switch (resource) { + // completions + case 'completions.create': + return 'createCompletion' + + // chat completions + case 'chat.completions.create': + return 'createChatCompletion' + + // embeddings + case 'embeddings.create': + return 'createEmbedding' + default: + return resource + } +} + +function gateResource (resource) { + return ['createCompletion', 'createChatCompletion', 'createEmbedding'].includes(resource) + ? resource + : undefined +} + +function getOperation (resource) { + switch (resource) { + case 'createCompletion': + return 'completion' + case 'createChatCompletion': + return 'chat' + case 'createEmbedding': + return 'embedding' + default: + // should never happen + return 'unknown' + } +} + +module.exports = OpenAiLLMObsPlugin diff --git a/packages/dd-trace/src/llmobs/sdk.js b/packages/dd-trace/src/llmobs/sdk.js new file mode 100644 index 00000000000..5717a8a0f19 --- /dev/null +++ b/packages/dd-trace/src/llmobs/sdk.js @@ -0,0 +1,377 @@ +'use strict' + +const { SPAN_KIND, OUTPUT_VALUE } = require('./constants/tags') + +const { + getFunctionArguments, + validateKind +} = require('./util') +const { isTrue } = require('../util') + +const { storage } = require('./storage') + +const Span = require('../opentracing/span') + +const tracerVersion = require('../../../../package.json').version +const logger = require('../log') + +const LLMObsTagger = require('./tagger') + +// communicating with writer +const { channel } = require('dc-polyfill') +const evalMetricAppendCh = channel('llmobs:eval-metric:append') +const flushCh = channel('llmobs:writers:flush') +const NoopLLMObs = require('./noop') + +class LLMObs extends NoopLLMObs { + constructor (tracer, llmobsModule, config) { + super(tracer) + + this._config = config + this._llmobsModule = llmobsModule + this._tagger = new LLMObsTagger(config) + } + + get enabled () { + return this._config.llmobs.enabled + } + + enable (options = {}) { + if (this.enabled) { + logger.debug('LLMObs is already enabled.') + return + } + + logger.debug('Enabling LLMObs') + + const { mlApp, agentlessEnabled } = options + + const { DD_LLMOBS_ENABLED } = process.env + + const llmobsConfig = { + mlApp, + agentlessEnabled + } + + const enabled = DD_LLMOBS_ENABLED == null || isTrue(DD_LLMOBS_ENABLED) + if (!enabled) { + logger.debug('LLMObs.enable() called when DD_LLMOBS_ENABLED is false. No action taken.') + return + } + + this._config.llmobs.enabled = true + this._config.configure({ ...this._config, llmobs: llmobsConfig }) + + // configure writers and channel subscribers + this._llmobsModule.enable(this._config) + } + + disable () { + if (!this.enabled) { + logger.debug('LLMObs is already disabled.') + return + } + + logger.debug('Disabling LLMObs') + + this._config.llmobs.enabled = false + + // disable writers and channel subscribers + this._llmobsModule.disable() + } + + trace (options = {}, fn) { + if (typeof options === 'function') { + fn = options + options = {} + } + + const kind = validateKind(options.kind) // will throw if kind is undefined or not an expected kind + + // name is required for spans generated with `trace` + // while `kind` is required, this should never throw (as otherwise it would have thrown above) + const name = options.name || kind + if (!name) { + throw new Error('No span name provided for `trace`.') + } + + const { + spanOptions, + ...llmobsOptions + } = this._extractOptions(options) + + if (fn.length > 1) { + return this._tracer.trace(name, spanOptions, (span, cb) => + this._activate(span, { kind, options: llmobsOptions }, () => fn(span, cb)) + ) + } + + return this._tracer.trace(name, spanOptions, span => + this._activate(span, { kind, options: llmobsOptions }, () => fn(span)) + ) + } + + wrap (options = {}, fn) { + if (typeof options === 'function') { + fn = options + options = {} + } + + const kind = validateKind(options.kind) // will throw if kind is undefined or not an expected kind + let name = options.name || (fn?.name ? fn.name : undefined) || kind + + if (!name) { + logger.warn('No span name provided for `wrap`. Defaulting to "unnamed-anonymous-function".') + name = 'unnamed-anonymous-function' + } + + const { + spanOptions, + ...llmobsOptions + } = this._extractOptions(options) + + const llmobs = this + + function wrapped () { + const span = llmobs._tracer.scope().active() + + const result = llmobs._activate(span, { kind, options: llmobsOptions }, () => { + if (!['llm', 'embedding'].includes(kind)) { + llmobs.annotate(span, { inputData: getFunctionArguments(fn, arguments) }) + } + + return fn.apply(this, arguments) + }) + + if (result && typeof result.then === 'function') { + return result.then(value => { + if (value && !['llm', 'retrieval'].includes(kind) && !LLMObsTagger.tagMap.get(span)?.[OUTPUT_VALUE]) { + llmobs.annotate(span, { outputData: value }) + } + return value + }) + } + + if (result && !['llm', 'retrieval'].includes(kind) && !LLMObsTagger.tagMap.get(span)?.[OUTPUT_VALUE]) { + llmobs.annotate(span, { outputData: result }) + } + + return result + } + + return this._tracer.wrap(name, spanOptions, wrapped) + } + + annotate (span, options) { + if (!this.enabled) return + + if (!span) { + span = this._active() + } + + if ((span && !options) && !(span instanceof Span)) { + options = span + span = this._active() + } + + if (!span) { + throw new Error('No span provided and no active LLMObs-generated span found') + } + if (!options) { + throw new Error('No options provided for annotation.') + } + + if (!LLMObsTagger.tagMap.has(span)) { + throw new Error('Span must be an LLMObs-generated span') + } + if (span._duration !== undefined) { + throw new Error('Cannot annotate a finished span') + } + + const spanKind = LLMObsTagger.tagMap.get(span)[SPAN_KIND] + if (!spanKind) { + throw new Error('LLMObs span must have a span kind specified') + } + + const { inputData, outputData, metadata, metrics, tags } = options + + if (inputData || outputData) { + if (spanKind === 'llm') { + this._tagger.tagLLMIO(span, inputData, outputData) + } else if (spanKind === 'embedding') { + this._tagger.tagEmbeddingIO(span, inputData, outputData) + } else if (spanKind === 'retrieval') { + this._tagger.tagRetrievalIO(span, inputData, outputData) + } else { + this._tagger.tagTextIO(span, inputData, outputData) + } + } + + if (metadata) { + this._tagger.tagMetadata(span, metadata) + } + + if (metrics) { + this._tagger.tagMetrics(span, metrics) + } + + if (tags) { + this._tagger.tagSpanTags(span, tags) + } + } + + exportSpan (span) { + span = span || this._active() + + if (!span) { + throw new Error('No span provided and no active LLMObs-generated span found') + } + + if (!(span instanceof Span)) { + throw new Error('Span must be a valid Span object.') + } + + if (!LLMObsTagger.tagMap.has(span)) { + throw new Error('Span must be an LLMObs-generated span') + } + + try { + return { + traceId: span.context().toTraceId(true), + spanId: span.context().toSpanId() + } + } catch { + logger.warn('Faild to export span. Span must be a valid Span object.') + } + } + + submitEvaluation (llmobsSpanContext, options = {}) { + if (!this.enabled) return + + if (!this._config.apiKey) { + throw new Error( + 'DD_API_KEY is required for sending evaluation metrics. Evaluation metric data will not be sent.\n' + + 'Ensure this configuration is set before running your application.' + ) + } + + const { traceId, spanId } = llmobsSpanContext + if (!traceId || !spanId) { + throw new Error( + 'spanId and traceId must both be specified for the given evaluation metric to be submitted.' + ) + } + + const mlApp = options.mlApp || this._config.llmobs.mlApp + if (!mlApp) { + throw new Error( + 'ML App name is required for sending evaluation metrics. Evaluation metric data will not be sent.' + ) + } + + const timestampMs = options.timestampMs || Date.now() + if (typeof timestampMs !== 'number' || timestampMs < 0) { + throw new Error('timestampMs must be a non-negative integer. Evaluation metric data will not be sent') + } + + const { label, value, tags } = options + const metricType = options.metricType?.toLowerCase() + if (!label) { + throw new Error('label must be the specified name of the evaluation metric') + } + if (!metricType || !['categorical', 'score'].includes(metricType)) { + throw new Error('metricType must be one of "categorical" or "score"') + } + + if (metricType === 'categorical' && typeof value !== 'string') { + throw new Error('value must be a string for a categorical metric.') + } + if (metricType === 'score' && typeof value !== 'number') { + throw new Error('value must be a number for a score metric.') + } + + const evaluationTags = { + 'dd-trace.version': tracerVersion, + ml_app: mlApp + } + + if (tags) { + for (const key in tags) { + const tag = tags[key] + if (typeof tag === 'string') { + evaluationTags[key] = tag + } else if (typeof tag.toString === 'function') { + evaluationTags[key] = tag.toString() + } else if (tag == null) { + evaluationTags[key] = Object.prototype.toString.call(tag) + } else { + // should be a rare case + // every object in JS has a toString, otherwise every primitive has its own toString + // null and undefined are handled above + throw new Error('Failed to parse tags. Tags for evaluation metrics must be strings') + } + } + } + + const payload = { + span_id: spanId, + trace_id: traceId, + label, + metric_type: metricType, + ml_app: mlApp, + [`${metricType}_value`]: value, + timestamp_ms: timestampMs, + tags: Object.entries(evaluationTags).map(([key, value]) => `${key}:${value}`) + } + + evalMetricAppendCh.publish(payload) + } + + flush () { + if (!this.enabled) return + + flushCh.publish() + } + + _active () { + const store = storage.getStore() + return store?.span + } + + _activate (span, { kind, options } = {}, fn) { + const parent = this._active() + if (this.enabled) storage.enterWith({ span }) + + this._tagger.registerLLMObsSpan(span, { + ...options, + parent, + kind + }) + + try { + return fn() + } finally { + if (this.enabled) storage.enterWith({ span: parent }) + } + } + + _extractOptions (options) { + const { + modelName, + modelProvider, + sessionId, + mlApp, + ...spanOptions + } = options + + return { + mlApp, + modelName, + modelProvider, + sessionId, + spanOptions + } + } +} + +module.exports = LLMObs diff --git a/packages/dd-trace/src/llmobs/span_processor.js b/packages/dd-trace/src/llmobs/span_processor.js new file mode 100644 index 00000000000..bc8eeda06b7 --- /dev/null +++ b/packages/dd-trace/src/llmobs/span_processor.js @@ -0,0 +1,195 @@ +'use strict' + +const { + SPAN_KIND, + MODEL_NAME, + MODEL_PROVIDER, + METADATA, + INPUT_MESSAGES, + INPUT_VALUE, + OUTPUT_MESSAGES, + INPUT_DOCUMENTS, + OUTPUT_DOCUMENTS, + OUTPUT_VALUE, + METRICS, + ML_APP, + TAGS, + PARENT_ID_KEY, + SESSION_ID, + NAME +} = require('./constants/tags') +const { UNSERIALIZABLE_VALUE_TEXT } = require('./constants/text') + +const { + ERROR_MESSAGE, + ERROR_TYPE, + ERROR_STACK +} = require('../constants') + +const LLMObsTagger = require('./tagger') + +const tracerVersion = require('../../../../package.json').version +const logger = require('../log') + +class LLMObsSpanProcessor { + constructor (config) { + this._config = config + } + + setWriter (writer) { + this._writer = writer + } + + // TODO: instead of relying on the tagger's weakmap registry, can we use some namespaced storage correlation? + process ({ span }) { + if (!this._config.llmobs.enabled) return + // if the span is not in our private tagger map, it is not an llmobs span + if (!LLMObsTagger.tagMap.has(span)) return + + try { + const formattedEvent = this.format(span) + this._writer.append(formattedEvent) + } catch (e) { + // this should be a rare case + // we protect against unserializable properties in the format function, and in + // safeguards in the tagger + logger.warn(` + Failed to append span to LLM Observability writer, likely due to an unserializable property. + Span won't be sent to LLM Observability: ${e.message} + `) + } + } + + format (span) { + const spanTags = span.context()._tags + const mlObsTags = LLMObsTagger.tagMap.get(span) + + const spanKind = mlObsTags[SPAN_KIND] + + const meta = { 'span.kind': spanKind, input: {}, output: {} } + const input = {} + const output = {} + + if (['llm', 'embedding'].includes(spanKind)) { + meta.model_name = mlObsTags[MODEL_NAME] || 'custom' + meta.model_provider = (mlObsTags[MODEL_PROVIDER] || 'custom').toLowerCase() + } + if (mlObsTags[METADATA]) { + this._addObject(mlObsTags[METADATA], meta.metadata = {}) + } + if (spanKind === 'llm' && mlObsTags[INPUT_MESSAGES]) { + input.messages = mlObsTags[INPUT_MESSAGES] + } + if (mlObsTags[INPUT_VALUE]) { + input.value = mlObsTags[INPUT_VALUE] + } + if (spanKind === 'llm' && mlObsTags[OUTPUT_MESSAGES]) { + output.messages = mlObsTags[OUTPUT_MESSAGES] + } + if (spanKind === 'embedding' && mlObsTags[INPUT_DOCUMENTS]) { + input.documents = mlObsTags[INPUT_DOCUMENTS] + } + if (mlObsTags[OUTPUT_VALUE]) { + output.value = mlObsTags[OUTPUT_VALUE] + } + if (spanKind === 'retrieval' && mlObsTags[OUTPUT_DOCUMENTS]) { + output.documents = mlObsTags[OUTPUT_DOCUMENTS] + } + + const error = spanTags.error || spanTags[ERROR_TYPE] + if (error) { + meta[ERROR_MESSAGE] = spanTags[ERROR_MESSAGE] || error.message || error.code + meta[ERROR_TYPE] = spanTags[ERROR_TYPE] || error.name + meta[ERROR_STACK] = spanTags[ERROR_STACK] || error.stack + } + + if (input) meta.input = input + if (output) meta.output = output + + const metrics = mlObsTags[METRICS] || {} + + const mlApp = mlObsTags[ML_APP] + const sessionId = mlObsTags[SESSION_ID] + const parentId = mlObsTags[PARENT_ID_KEY] + + const name = mlObsTags[NAME] || span._name + + const llmObsSpanEvent = { + trace_id: span.context().toTraceId(true), + span_id: span.context().toSpanId(), + parent_id: parentId, + name, + tags: this._processTags(span, mlApp, sessionId, error), + start_ns: Math.round(span._startTime * 1e6), + duration: Math.round(span._duration * 1e6), + status: error ? 'error' : 'ok', + meta, + metrics, + _dd: { + span_id: span.context().toSpanId(), + trace_id: span.context().toTraceId(true) + } + } + + if (sessionId) llmObsSpanEvent.session_id = sessionId + + return llmObsSpanEvent + } + + // For now, this only applies to metadata, as we let users annotate this field with any object + // However, we want to protect against circular references or BigInts (unserializable) + // This function can be reused for other fields if needed + // Messages, Documents, and Metrics are safeguarded in `llmobs/tagger.js` + _addObject (obj, carrier) { + const seenObjects = new WeakSet() + seenObjects.add(obj) // capture root object + + const isCircular = value => { + if (typeof value !== 'object') return false + if (seenObjects.has(value)) return true + seenObjects.add(value) + return false + } + + const add = (obj, carrier) => { + for (const key in obj) { + const value = obj[key] + if (!Object.prototype.hasOwnProperty.call(obj, key)) continue + if (typeof value === 'bigint' || isCircular(value)) { + // mark as unserializable instead of dropping + logger.warn(`Unserializable property found in metadata: ${key}`) + carrier[key] = UNSERIALIZABLE_VALUE_TEXT + continue + } + if (typeof value === 'object') { + add(value, carrier[key] = {}) + } else { + carrier[key] = value + } + } + } + + add(obj, carrier) + } + + _processTags (span, mlApp, sessionId, error) { + let tags = { + version: this._config.version, + env: this._config.env, + service: this._config.service, + source: 'integration', + ml_app: mlApp, + 'dd-trace.version': tracerVersion, + error: Number(!!error) || 0, + language: 'javascript' + } + const errType = span.context()._tags[ERROR_TYPE] || error?.name + if (errType) tags.error_type = errType + if (sessionId) tags.session_id = sessionId + const existingTags = LLMObsTagger.tagMap.get(span)?.[TAGS] || {} + if (existingTags) tags = { ...tags, ...existingTags } + return Object.entries(tags).map(([key, value]) => `${key}:${value ?? ''}`) + } +} + +module.exports = LLMObsSpanProcessor diff --git a/packages/dd-trace/src/llmobs/storage.js b/packages/dd-trace/src/llmobs/storage.js new file mode 100644 index 00000000000..1362aaf966e --- /dev/null +++ b/packages/dd-trace/src/llmobs/storage.js @@ -0,0 +1,7 @@ +'use strict' + +// TODO: remove this and use namespaced storage once available +const { AsyncLocalStorage } = require('async_hooks') +const storage = new AsyncLocalStorage() + +module.exports = { storage } diff --git a/packages/dd-trace/src/llmobs/tagger.js b/packages/dd-trace/src/llmobs/tagger.js new file mode 100644 index 00000000000..9f1728e5d7b --- /dev/null +++ b/packages/dd-trace/src/llmobs/tagger.js @@ -0,0 +1,322 @@ +'use strict' + +const log = require('../log') +const { + MODEL_NAME, + MODEL_PROVIDER, + SESSION_ID, + ML_APP, + SPAN_KIND, + INPUT_VALUE, + OUTPUT_DOCUMENTS, + INPUT_DOCUMENTS, + OUTPUT_VALUE, + METADATA, + METRICS, + PARENT_ID_KEY, + INPUT_MESSAGES, + OUTPUT_MESSAGES, + TAGS, + NAME, + PROPAGATED_PARENT_ID_KEY, + ROOT_PARENT_ID, + INPUT_TOKENS_METRIC_KEY, + OUTPUT_TOKENS_METRIC_KEY, + TOTAL_TOKENS_METRIC_KEY +} = require('./constants/tags') + +// global registry of LLMObs spans +// maps LLMObs spans to their annotations +const registry = new WeakMap() + +class LLMObsTagger { + constructor (config, softFail = false) { + this._config = config + + this.softFail = softFail + } + + static get tagMap () { + return registry + } + + registerLLMObsSpan (span, { + modelName, + modelProvider, + sessionId, + mlApp, + parent, + kind, + name + } = {}) { + if (!this._config.llmobs.enabled) return + if (!kind) return // do not register it in the map if it doesn't have an llmobs span kind + + this._register(span) + + if (name) this._setTag(span, NAME, name) + + this._setTag(span, SPAN_KIND, kind) + if (modelName) this._setTag(span, MODEL_NAME, modelName) + if (modelProvider) this._setTag(span, MODEL_PROVIDER, modelProvider) + + sessionId = sessionId || parent?.context()._tags[SESSION_ID] + if (sessionId) this._setTag(span, SESSION_ID, sessionId) + + if (!mlApp) mlApp = parent?.context()._tags[ML_APP] || this._config.llmobs.mlApp + this._setTag(span, ML_APP, mlApp) + + const parentId = + parent?.context().toSpanId() || + span.context()._trace.tags[PROPAGATED_PARENT_ID_KEY] || + ROOT_PARENT_ID + this._setTag(span, PARENT_ID_KEY, parentId) + } + + // TODO: similarly for the following `tag` methods, + // how can we transition from a span weakmap to core API functionality + tagLLMIO (span, inputData, outputData) { + this._tagMessages(span, inputData, INPUT_MESSAGES) + this._tagMessages(span, outputData, OUTPUT_MESSAGES) + } + + tagEmbeddingIO (span, inputData, outputData) { + this._tagDocuments(span, inputData, INPUT_DOCUMENTS) + this._tagText(span, outputData, OUTPUT_VALUE) + } + + tagRetrievalIO (span, inputData, outputData) { + this._tagText(span, inputData, INPUT_VALUE) + this._tagDocuments(span, outputData, OUTPUT_DOCUMENTS) + } + + tagTextIO (span, inputData, outputData) { + this._tagText(span, inputData, INPUT_VALUE) + this._tagText(span, outputData, OUTPUT_VALUE) + } + + tagMetadata (span, metadata) { + this._setTag(span, METADATA, metadata) + } + + tagMetrics (span, metrics) { + const filterdMetrics = {} + for (const [key, value] of Object.entries(metrics)) { + let processedKey = key + + // processing these specifically for our metrics ingestion + switch (key) { + case 'inputTokens': + processedKey = INPUT_TOKENS_METRIC_KEY + break + case 'outputTokens': + processedKey = OUTPUT_TOKENS_METRIC_KEY + break + case 'totalTokens': + processedKey = TOTAL_TOKENS_METRIC_KEY + break + } + + if (typeof value === 'number') { + filterdMetrics[processedKey] = value + } else { + this._handleFailure(`Value for metric '${key}' must be a number, instead got ${value}`) + } + } + + this._setTag(span, METRICS, filterdMetrics) + } + + tagSpanTags (span, tags) { + // new tags will be merged with existing tags + const currentTags = registry.get(span)?.[TAGS] + if (currentTags) { + Object.assign(tags, currentTags) + } + this._setTag(span, TAGS, tags) + } + + _tagText (span, data, key) { + if (data) { + if (typeof data === 'string') { + this._setTag(span, key, data) + } else { + try { + this._setTag(span, key, JSON.stringify(data)) + } catch { + const type = key === INPUT_VALUE ? 'input' : 'output' + this._handleFailure(`Failed to parse ${type} value, must be JSON serializable.`) + } + } + } + } + + _tagDocuments (span, data, key) { + if (data) { + if (!Array.isArray(data)) { + data = [data] + } + + const documents = data.map(document => { + if (typeof document === 'string') { + return { text: document } + } + + if (document == null || typeof document !== 'object') { + this._handleFailure('Documents must be a string, object, or list of objects.') + return undefined + } + + const { text, name, id, score } = document + let validDocument = true + + if (typeof text !== 'string') { + this._handleFailure('Document text must be a string.') + validDocument = false + } + + const documentObj = { text } + + validDocument = this._tagConditionalString(name, 'Document name', documentObj, 'name') && validDocument + validDocument = this._tagConditionalString(id, 'Document ID', documentObj, 'id') && validDocument + validDocument = this._tagConditionalNumber(score, 'Document score', documentObj, 'score') && validDocument + + return validDocument ? documentObj : undefined + }).filter(doc => !!doc) + + if (documents.length) { + this._setTag(span, key, documents) + } + } + } + + _tagMessages (span, data, key) { + if (data) { + if (!Array.isArray(data)) { + data = [data] + } + + const messages = data.map(message => { + if (typeof message === 'string') { + return { content: message } + } + + if (message == null || typeof message !== 'object') { + this._handleFailure('Messages must be a string, object, or list of objects') + return undefined + } + + let validMessage = true + + const { content = '', role } = message + let toolCalls = message.toolCalls + const messageObj = { content } + + if (typeof content !== 'string') { + this._handleFailure('Message content must be a string.') + validMessage = false + } + + validMessage = this._tagConditionalString(role, 'Message role', messageObj, 'role') && validMessage + + if (toolCalls) { + if (!Array.isArray(toolCalls)) { + toolCalls = [toolCalls] + } + + const filteredToolCalls = toolCalls.map(toolCall => { + if (typeof toolCall !== 'object') { + this._handleFailure('Tool call must be an object.') + return undefined + } + + let validTool = true + + const { name, arguments: args, toolId, type } = toolCall + const toolCallObj = {} + + validTool = this._tagConditionalString(name, 'Tool name', toolCallObj, 'name') && validTool + validTool = this._tagConditionalObject(args, 'Tool arguments', toolCallObj, 'arguments') && validTool + validTool = this._tagConditionalString(toolId, 'Tool ID', toolCallObj, 'tool_id') && validTool + validTool = this._tagConditionalString(type, 'Tool type', toolCallObj, 'type') && validTool + + return validTool ? toolCallObj : undefined + }).filter(toolCall => !!toolCall) + + if (filteredToolCalls.length) { + messageObj.tool_calls = filteredToolCalls + } + } + + return validMessage ? messageObj : undefined + }).filter(msg => !!msg) + + if (messages.length) { + this._setTag(span, key, messages) + } + } + } + + _tagConditionalString (data, type, carrier, key) { + if (!data) return true + if (typeof data !== 'string') { + this._handleFailure(`"${type}" must be a string.`) + return false + } + carrier[key] = data + return true + } + + _tagConditionalNumber (data, type, carrier, key) { + if (!data) return true + if (typeof data !== 'number') { + this._handleFailure(`"${type}" must be a number.`) + return false + } + carrier[key] = data + return true + } + + _tagConditionalObject (data, type, carrier, key) { + if (!data) return true + if (typeof data !== 'object') { + this._handleFailure(`"${type}" must be an object.`) + return false + } + carrier[key] = data + return true + } + + // any public-facing LLMObs APIs using this tagger should not soft fail + // auto-instrumentation should soft fail + _handleFailure (msg) { + if (this.softFail) { + log.warn(msg) + } else { + throw new Error(msg) + } + } + + _register (span) { + if (!this._config.llmobs.enabled) return + if (registry.has(span)) { + this._handleFailure(`LLMObs Span "${span._name}" already registered.`) + return + } + + registry.set(span, {}) + } + + _setTag (span, key, value) { + if (!this._config.llmobs.enabled) return + if (!registry.has(span)) { + this._handleFailure('Span must be an LLMObs generated span.') + return + } + + const tagsCarrier = registry.get(span) + Object.assign(tagsCarrier, { [key]: value }) + } +} + +module.exports = LLMObsTagger diff --git a/packages/dd-trace/src/llmobs/util.js b/packages/dd-trace/src/llmobs/util.js new file mode 100644 index 00000000000..feba656f952 --- /dev/null +++ b/packages/dd-trace/src/llmobs/util.js @@ -0,0 +1,176 @@ +'use strict' + +const { SPAN_KINDS } = require('./constants/tags') + +function encodeUnicode (str) { + if (!str) return str + return str.split('').map(char => { + const code = char.charCodeAt(0) + if (code > 127) { + return `\\u${code.toString(16).padStart(4, '0')}` + } + return char + }).join('') +} + +function validateKind (kind) { + if (!SPAN_KINDS.includes(kind)) { + throw new Error(` + Invalid span kind specified: "${kind}" + Must be one of: ${SPAN_KINDS.join(', ')} + `) + } + + return kind +} + +// extracts the argument names from a function string +function parseArgumentNames (str) { + const result = [] + let current = '' + let closerCount = 0 + let recording = true + let inSingleLineComment = false + let inMultiLineComment = false + + for (let i = 0; i < str.length; i++) { + const char = str[i] + const nextChar = str[i + 1] + + // Handle single-line comments + if (!inMultiLineComment && char === '/' && nextChar === '/') { + inSingleLineComment = true + i++ // Skip the next character + continue + } + + // Handle multi-line comments + if (!inSingleLineComment && char === '/' && nextChar === '*') { + inMultiLineComment = true + i++ // Skip the next character + continue + } + + // End of single-line comment + if (inSingleLineComment && char === '\n') { + inSingleLineComment = false + continue + } + + // End of multi-line comment + if (inMultiLineComment && char === '*' && nextChar === '/') { + inMultiLineComment = false + i++ // Skip the next character + continue + } + + // Skip characters inside comments + if (inSingleLineComment || inMultiLineComment) { + continue + } + + if (['{', '[', '('].includes(char)) { + closerCount++ + } else if (['}', ']', ')'].includes(char)) { + closerCount-- + } else if (char === '=' && nextChar !== '>' && closerCount === 0) { + recording = false + // record the variable name early, and stop counting characters until we reach the next comma + result.push(current.trim()) + current = '' + continue + } else if (char === ',' && closerCount === 0) { + if (recording) { + result.push(current.trim()) + current = '' + } + + recording = true + continue + } + + if (recording) { + current += char + } + } + + if (current && recording) { + result.push(current.trim()) + } + + return result +} + +// finds the bounds of the arguments in a function string +function findArgumentsBounds (str) { + let start = -1 + let end = -1 + let closerCount = 0 + + for (let i = 0; i < str.length; i++) { + const char = str[i] + + if (char === '(') { + if (closerCount === 0) { + start = i + } + + closerCount++ + } else if (char === ')') { + closerCount-- + + if (closerCount === 0) { + end = i + break + } + } + } + + return { start, end } +} + +const memo = new WeakMap() +function getFunctionArguments (fn, args = []) { + if (!fn) return + if (!args.length) return + if (args.length === 1) return args[0] + + try { + let names + if (memo.has(fn)) { + names = memo.get(fn) + } else { + const fnString = fn.toString() + const { start, end } = findArgumentsBounds(fnString) + names = parseArgumentNames(fnString.slice(start + 1, end)) + memo.set(fn, names) + } + + const argsObject = {} + + for (const argIdx in args) { + const name = names[argIdx] + const arg = args[argIdx] + + const spread = name?.startsWith('...') + + // this can only be the last argument + if (spread) { + argsObject[name.slice(3)] = args.slice(argIdx) + break + } + + argsObject[name] = arg + } + + return argsObject + } catch { + return args + } +} + +module.exports = { + encodeUnicode, + validateKind, + getFunctionArguments +} diff --git a/packages/dd-trace/src/llmobs/writers/base.js b/packages/dd-trace/src/llmobs/writers/base.js new file mode 100644 index 00000000000..8a6cdae9c2f --- /dev/null +++ b/packages/dd-trace/src/llmobs/writers/base.js @@ -0,0 +1,111 @@ +'use strict' + +const request = require('../../exporters/common/request') +const { URL, format } = require('url') + +const logger = require('../../log') + +const { encodeUnicode } = require('../util') +const log = require('../../log') + +class BaseLLMObsWriter { + constructor ({ interval, timeout, endpoint, intake, eventType, protocol, port }) { + this._interval = interval || 1000 // 1s + this._timeout = timeout || 5000 // 5s + this._eventType = eventType + + this._buffer = [] + this._bufferLimit = 1000 + this._bufferSize = 0 + + this._url = new URL(format({ + protocol: protocol || 'https:', + hostname: intake, + port: port || 443, + pathname: endpoint + })) + + this._headers = { + 'Content-Type': 'application/json' + } + + this._periodic = setInterval(() => { + this.flush() + }, this._interval).unref() + + process.once('beforeExit', () => { + this.destroy() + }) + + this._destroyed = false + + logger.debug(`Started ${this.constructor.name} to ${this._url}`) + } + + append (event, byteLength) { + if (this._buffer.length >= this._bufferLimit) { + logger.warn(`${this.constructor.name} event buffer full (limit is ${this._bufferLimit}), dropping event`) + return + } + + this._bufferSize += byteLength || Buffer.from(JSON.stringify(event)).byteLength + this._buffer.push(event) + } + + flush () { + if (this._buffer.length === 0) { + return + } + + const events = this._buffer + this._buffer = [] + this._bufferSize = 0 + const payload = this._encode(this.makePayload(events)) + + const options = { + headers: this._headers, + method: 'POST', + url: this._url, + timeout: this._timeout + } + + log.debug(`Encoded LLMObs payload: ${payload}`) + + request(payload, options, (err, resp, code) => { + if (err) { + logger.error( + `Error sending ${events.length} LLMObs ${this._eventType} events to ${this._url}: ${err.message}` + ) + } else if (code >= 300) { + logger.error( + `Error sending ${events.length} LLMObs ${this._eventType} events to ${this._url}: ${code}` + ) + } else { + logger.debug(`Sent ${events.length} LLMObs ${this._eventType} events to ${this._url}`) + } + }) + } + + makePayload (events) {} + + destroy () { + if (!this._destroyed) { + logger.debug(`Stopping ${this.constructor.name}`) + clearInterval(this._periodic) + process.removeListener('beforeExit', this.destroy) + this.flush() + this._destroyed = true + } + } + + _encode (payload) { + return JSON.stringify(payload, (key, value) => { + if (typeof value === 'string') { + return encodeUnicode(value) // serialize unicode characters + } + return value + }).replace(/\\\\u/g, '\\u') // remove double escaping + } +} + +module.exports = BaseLLMObsWriter diff --git a/packages/dd-trace/src/llmobs/writers/evaluations.js b/packages/dd-trace/src/llmobs/writers/evaluations.js new file mode 100644 index 00000000000..d737f68c82c --- /dev/null +++ b/packages/dd-trace/src/llmobs/writers/evaluations.js @@ -0,0 +1,29 @@ +'use strict' + +const { AGENTLESS_EVALULATIONS_ENDPOINT } = require('../constants/writers') +const BaseWriter = require('./base') + +class LLMObsEvalMetricsWriter extends BaseWriter { + constructor (config) { + super({ + endpoint: AGENTLESS_EVALULATIONS_ENDPOINT, + intake: `api.${config.site}`, + eventType: 'evaluation_metric' + }) + + this._headers['DD-API-KEY'] = config.apiKey + } + + makePayload (events) { + return { + data: { + type: this._eventType, + attributes: { + metrics: events + } + } + } + } +} + +module.exports = LLMObsEvalMetricsWriter diff --git a/packages/dd-trace/src/llmobs/writers/spans/agentProxy.js b/packages/dd-trace/src/llmobs/writers/spans/agentProxy.js new file mode 100644 index 00000000000..6274f6117e0 --- /dev/null +++ b/packages/dd-trace/src/llmobs/writers/spans/agentProxy.js @@ -0,0 +1,23 @@ +'use strict' + +const { + EVP_SUBDOMAIN_HEADER_NAME, + EVP_SUBDOMAIN_HEADER_VALUE, + EVP_PROXY_AGENT_ENDPOINT +} = require('../../constants/writers') +const LLMObsBaseSpanWriter = require('./base') + +class LLMObsAgentProxySpanWriter extends LLMObsBaseSpanWriter { + constructor (config) { + super({ + intake: config.hostname || 'localhost', + protocol: 'http:', + endpoint: EVP_PROXY_AGENT_ENDPOINT, + port: config.port + }) + + this._headers[EVP_SUBDOMAIN_HEADER_NAME] = EVP_SUBDOMAIN_HEADER_VALUE + } +} + +module.exports = LLMObsAgentProxySpanWriter diff --git a/packages/dd-trace/src/llmobs/writers/spans/agentless.js b/packages/dd-trace/src/llmobs/writers/spans/agentless.js new file mode 100644 index 00000000000..452f41d541a --- /dev/null +++ b/packages/dd-trace/src/llmobs/writers/spans/agentless.js @@ -0,0 +1,17 @@ +'use strict' + +const { AGENTLESS_SPANS_ENDPOINT } = require('../../constants/writers') +const LLMObsBaseSpanWriter = require('./base') + +class LLMObsAgentlessSpanWriter extends LLMObsBaseSpanWriter { + constructor (config) { + super({ + intake: `llmobs-intake.${config.site}`, + endpoint: AGENTLESS_SPANS_ENDPOINT + }) + + this._headers['DD-API-KEY'] = config.apiKey + } +} + +module.exports = LLMObsAgentlessSpanWriter diff --git a/packages/dd-trace/src/llmobs/writers/spans/base.js b/packages/dd-trace/src/llmobs/writers/spans/base.js new file mode 100644 index 00000000000..f5fe3443f2d --- /dev/null +++ b/packages/dd-trace/src/llmobs/writers/spans/base.js @@ -0,0 +1,49 @@ +'use strict' + +const { EVP_EVENT_SIZE_LIMIT, EVP_PAYLOAD_SIZE_LIMIT } = require('../../constants/writers') +const { DROPPED_VALUE_TEXT } = require('../../constants/text') +const { DROPPED_IO_COLLECTION_ERROR } = require('../../constants/tags') +const BaseWriter = require('../base') +const logger = require('../../../log') + +class LLMObsSpanWriter extends BaseWriter { + constructor (options) { + super({ + ...options, + eventType: 'span' + }) + } + + append (event) { + const eventSizeBytes = Buffer.from(JSON.stringify(event)).byteLength + if (eventSizeBytes > EVP_EVENT_SIZE_LIMIT) { + logger.warn(`Dropping event input/output because its size (${eventSizeBytes}) exceeds the 1MB event size limit`) + event = this._truncateSpanEvent(event) + } + + if (this._bufferSize + eventSizeBytes > EVP_PAYLOAD_SIZE_LIMIT) { + logger.debug('Flusing queue because queing next event will exceed EvP payload limit') + this.flush() + } + + super.append(event, eventSizeBytes) + } + + makePayload (events) { + return { + '_dd.stage': 'raw', + event_type: this._eventType, + spans: events + } + } + + _truncateSpanEvent (event) { + event.meta.input = { value: DROPPED_VALUE_TEXT } + event.meta.output = { value: DROPPED_VALUE_TEXT } + + event.collection_errors = [DROPPED_IO_COLLECTION_ERROR] + return event + } +} + +module.exports = LLMObsSpanWriter diff --git a/packages/dd-trace/src/noop/proxy.js b/packages/dd-trace/src/noop/proxy.js index 417cb846f8d..ec8671a371e 100644 --- a/packages/dd-trace/src/noop/proxy.js +++ b/packages/dd-trace/src/noop/proxy.js @@ -3,16 +3,19 @@ const NoopTracer = require('./tracer') const NoopAppsecSdk = require('../appsec/sdk/noop') const NoopDogStatsDClient = require('./dogstatsd') +const NoopLLMObsSDK = require('../llmobs/noop') const noop = new NoopTracer() const noopAppsec = new NoopAppsecSdk() const noopDogStatsDClient = new NoopDogStatsDClient() +const noopLLMObs = new NoopLLMObsSDK(noop) class Tracer { constructor () { this._tracer = noop this.appsec = noopAppsec this.dogstatsd = noopDogStatsDClient + this.llmobs = noopLLMObs } init () { diff --git a/packages/dd-trace/src/opentelemetry/span.js b/packages/dd-trace/src/opentelemetry/span.js index a62902d8457..d2c216c138e 100644 --- a/packages/dd-trace/src/opentelemetry/span.js +++ b/packages/dd-trace/src/opentelemetry/span.js @@ -142,7 +142,7 @@ class Span { context: spanContext._ddContext, startTime, hostname: _tracer._hostname, - integrationName: 'otel', + integrationName: parentTracer?._isOtelLibrary ? 'otel.library' : 'otel', tags: { [SERVICE_NAME]: _tracer._service, [RESOURCE_NAME]: spanName diff --git a/packages/dd-trace/src/opentelemetry/tracer.js b/packages/dd-trace/src/opentelemetry/tracer.js index bb9b81e6ccd..bf2a0c3f86b 100644 --- a/packages/dd-trace/src/opentelemetry/tracer.js +++ b/packages/dd-trace/src/opentelemetry/tracer.js @@ -16,6 +16,7 @@ class Tracer { this._tracerProvider = tracerProvider // Is there a reason this is public? this.instrumentationLibrary = library + this._isOtelLibrary = library?.name?.startsWith('@opentelemetry/instrumentation-') this._spanLimits = {} } diff --git a/packages/dd-trace/src/opentracing/propagation/text_map.js b/packages/dd-trace/src/opentracing/propagation/text_map.js index 1346f85de72..42a482853ee 100644 --- a/packages/dd-trace/src/opentracing/propagation/text_map.js +++ b/packages/dd-trace/src/opentracing/propagation/text_map.js @@ -53,6 +53,8 @@ class TextMapPropagator { } inject (spanContext, carrier) { + if (!spanContext || !carrier) return + this._injectBaggageItems(spanContext, carrier) this._injectDatadog(spanContext, carrier) this._injectB3MultipleHeaders(spanContext, carrier) @@ -383,7 +385,7 @@ class TextMapPropagator { return null } const matches = headerValue.trim().match(traceparentExpr) - if (matches.length) { + if (matches?.length) { const [version, traceId, spanId, flags, tail] = matches.slice(1) const traceparent = { version } const tracestate = TraceState.fromString(carrier.tracestate) diff --git a/packages/dd-trace/src/opentracing/tracer.js b/packages/dd-trace/src/opentracing/tracer.js index 37b1c68a635..2d854442cc3 100644 --- a/packages/dd-trace/src/opentracing/tracer.js +++ b/packages/dd-trace/src/opentracing/tracer.js @@ -52,8 +52,15 @@ class DatadogTracer { ? getContext(options.childOf) : getParent(options.references) + // as per spec, allow the setting of service name through options const tags = { - 'service.name': this._service + 'service.name': options?.tags?.service ? String(options.tags.service) : this._service + } + + // As per unified service tagging spec if a span is created with a service name different from the global + // service name it will not inherit the global version value + if (options?.tags?.service && options.tags.service !== this._service) { + options.tags.version = undefined } const span = new Span(this, this._processor, this._prioritySampler, { diff --git a/packages/dd-trace/src/plugins/outbound.js b/packages/dd-trace/src/plugins/outbound.js index f0a9509269e..44dbfa35aaa 100644 --- a/packages/dd-trace/src/plugins/outbound.js +++ b/packages/dd-trace/src/plugins/outbound.js @@ -7,6 +7,7 @@ const { PEER_SERVICE_REMAP_KEY } = require('../constants') const TracingPlugin = require('./tracing') +const { exitTags } = require('../../../datadog-code-origin') const COMMON_PEER_SVC_SOURCE_TAGS = [ 'net.peer.name', @@ -25,6 +26,14 @@ class OutboundPlugin extends TracingPlugin { }) } + startSpan (...args) { + const span = super.startSpan(...args) + if (this._tracerConfig.codeOriginForSpans.enabled) { + span.addTags(exitTags(this.startSpan)) + } + return span + } + getPeerService (tags) { /** * Compute `peer.service` and associated metadata from available tags, based diff --git a/packages/dd-trace/src/priority_sampler.js b/packages/dd-trace/src/priority_sampler.js index aae366c2622..f9968a41194 100644 --- a/packages/dd-trace/src/priority_sampler.js +++ b/packages/dd-trace/src/priority_sampler.js @@ -108,6 +108,18 @@ class PrioritySampler { } } + setPriority (span, samplingPriority, mechanism = SAMPLING_MECHANISM_MANUAL) { + if (!span || !this.validate(samplingPriority)) return + + const context = this._getContext(span) + + context._sampling.priority = samplingPriority + context._sampling.mechanism = mechanism + + const root = context._trace.started[0] + this._addDecisionMaker(root) + } + _getContext (span) { return typeof span.context === 'function' ? span.context() : span } @@ -201,6 +213,10 @@ class PrioritySampler { if (rule.match(span)) return rule } } + + static keepTrace (span, mechanism) { + span?._prioritySampler?.setPriority(span, USER_KEEP, mechanism) + } } module.exports = PrioritySampler diff --git a/packages/dd-trace/src/profiling/config.js b/packages/dd-trace/src/profiling/config.js index 538400aaa7a..3c360d65f7a 100644 --- a/packages/dd-trace/src/profiling/config.js +++ b/packages/dd-trace/src/profiling/config.js @@ -14,6 +14,7 @@ const { oomExportStrategies, snapshotKinds } = require('./constants') const { GIT_REPOSITORY_URL, GIT_COMMIT_SHA } = require('../plugins/util/tags') const { tagger } = require('./tagger') const { isFalse, isTrue } = require('../util') +const { getAzureTagsFromMetadata, getAzureAppMetadata } = require('../azure_metadata') class Config { constructor (options = {}) { @@ -71,7 +72,8 @@ class Config { this.tags = Object.assign( tagger.parse(DD_TAGS), tagger.parse(options.tags), - tagger.parse({ env, host, service, version, functionname }) + tagger.parse({ env, host, service, version, functionname }), + getAzureTagsFromMetadata(getAzureAppMetadata()) ) // Add source code integration tags if available diff --git a/packages/dd-trace/src/profiling/exporters/agent.js b/packages/dd-trace/src/profiling/exporters/agent.js index b34ab3c9d94..01363d6d2c5 100644 --- a/packages/dd-trace/src/profiling/exporters/agent.js +++ b/packages/dd-trace/src/profiling/exporters/agent.js @@ -195,11 +195,13 @@ class AgentExporter { }) sendRequest(options, form, (err, response) => { - if (operation.retry(err)) { - this._logger.error(`Error from the agent: ${err.message}`) - return - } else if (err) { - reject(err) + if (err) { + const { status } = err + if ((typeof status !== 'number' || status >= 500 || status === 429) && operation.retry(err)) { + this._logger.error(`Error from the agent: ${err.message}`) + } else { + reject(err) + } return } diff --git a/packages/dd-trace/src/profiling/profilers/wall.js b/packages/dd-trace/src/profiling/profilers/wall.js index 3d7041cfecf..dc3c0ba61ba 100644 --- a/packages/dd-trace/src/profiling/profilers/wall.js +++ b/packages/dd-trace/src/profiling/profilers/wall.js @@ -301,7 +301,8 @@ class NativeWallProfiler { const labels = { ...getThreadLabels() } - const { context: { ref: { spanId, rootSpanId, webTags, endpoint } }, timestamp } = context + const { context: { ref }, timestamp } = context + const { spanId, rootSpanId, webTags, endpoint } = ref ?? {} if (this._timelineEnabled) { // Incoming timestamps are in microseconds, we emit nanos. diff --git a/packages/dd-trace/src/proxy.js b/packages/dd-trace/src/proxy.js index b8916b205d4..32a7dcee10a 100644 --- a/packages/dd-trace/src/proxy.js +++ b/packages/dd-trace/src/proxy.js @@ -16,6 +16,7 @@ const NoopDogStatsDClient = require('./noop/dogstatsd') const spanleak = require('./spanleak') const { SSIHeuristics } = require('./profiling/ssi-heuristics') const appsecStandalone = require('./appsec/standalone') +const LLMObsSDK = require('./llmobs/sdk') class LazyModule { constructor (provider) { @@ -46,7 +47,8 @@ class Tracer extends NoopProxy { // these requires must work with esm bundler this._modules = { appsec: new LazyModule(() => require('./appsec')), - iast: new LazyModule(() => require('./appsec/iast')) + iast: new LazyModule(() => require('./appsec/iast')), + llmobs: new LazyModule(() => require('./llmobs')) } } @@ -195,11 +197,15 @@ class Tracer extends NoopProxy { if (config.appsec.enabled) { this._modules.appsec.enable(config) } + if (config.llmobs.enabled) { + this._modules.llmobs.enable(config) + } if (!this._tracingInitialized) { const prioritySampler = appsecStandalone.configure(config) this._tracer = new DatadogTracer(config, prioritySampler) this.dataStreamsCheckpointer = this._tracer.dataStreamsCheckpointer this.appsec = new AppsecSdk(this._tracer, config) + this.llmobs = new LLMObsSDK(this._tracer, this._modules.llmobs, config) this._tracingInitialized = true } if (config.iast.enabled) { @@ -208,6 +214,7 @@ class Tracer extends NoopProxy { } else if (this._tracingInitialized) { this._modules.appsec.disable() this._modules.iast.disable() + this._modules.llmobs.disable() } if (this._tracingInitialized) { diff --git a/packages/dd-trace/src/span_processor.js b/packages/dd-trace/src/span_processor.js index 6dc19407d56..deb92c02f34 100644 --- a/packages/dd-trace/src/span_processor.js +++ b/packages/dd-trace/src/span_processor.js @@ -10,6 +10,9 @@ const { SpanStatsProcessor } = require('./span_stats') const startedSpans = new WeakSet() const finishedSpans = new WeakSet() +const { channel } = require('dc-polyfill') +const spanProcessCh = channel('dd-trace:span:process') + class SpanProcessor { constructor (exporter, prioritySampler, config) { this._exporter = exporter @@ -45,6 +48,8 @@ class SpanProcessor { const formattedSpan = format(span) this._stats.onSpanFinished(formattedSpan) formatted.push(formattedSpan) + + spanProcessCh.publish({ span }) } else { active.push(span) } diff --git a/packages/dd-trace/src/telemetry/index.js b/packages/dd-trace/src/telemetry/index.js index 612c23b7ca1..5df7d6fcae3 100644 --- a/packages/dd-trace/src/telemetry/index.js +++ b/packages/dd-trace/src/telemetry/index.js @@ -314,7 +314,17 @@ function updateConfig (changes, config) { logInjection: 'DD_LOG_INJECTION', headerTags: 'DD_TRACE_HEADER_TAGS', tags: 'DD_TAGS', - 'sampler.rules': 'DD_TRACE_SAMPLING_RULES' + 'sampler.rules': 'DD_TRACE_SAMPLING_RULES', + traceEnabled: 'DD_TRACE_ENABLED', + url: 'DD_TRACE_AGENT_URL', + 'sampler.rateLimit': 'DD_TRACE_RATE_LIMIT', + queryStringObfuscation: 'DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP', + version: 'DD_VERSION', + env: 'DD_ENV', + service: 'DD_SERVICE', + clientIpHeader: 'DD_TRACE_CLIENT_IP_HEADER', + 'grpc.client.error.statuses': 'DD_GRPC_CLIENT_ERROR_STATUSES', + 'grpc.server.error.statuses': 'DD_GRPC_SERVER_ERROR_STATUSES' } const namesNeedFormatting = new Set(['DD_TAGS', 'peerServiceMapping', 'serviceMapping']) diff --git a/packages/dd-trace/test/appsec/iast/analyzers/header-injection.express.plugin.spec.js b/packages/dd-trace/test/appsec/iast/analyzers/header-injection.express.plugin.spec.js index bdb9734377a..dbb54802da2 100644 --- a/packages/dd-trace/test/appsec/iast/analyzers/header-injection.express.plugin.spec.js +++ b/packages/dd-trace/test/appsec/iast/analyzers/header-injection.express.plugin.spec.js @@ -226,7 +226,7 @@ describe('Header injection vulnerability', () => { testDescription: 'should have HEADER_INJECTION vulnerability when ' + 'the header is "access-control-allow-origin" and the origin is not a header', fn: (req, res) => { - setHeaderFunction('set-cookie', req.body.test, res) + setHeaderFunction('access-control-allow-origin', req.body.test, res) }, vulnerability: 'HEADER_INJECTION', makeRequest: (done, config) => { diff --git a/packages/dd-trace/test/appsec/iast/taint-tracking/plugin.spec.js b/packages/dd-trace/test/appsec/iast/taint-tracking/plugin.spec.js index 59b7c524aae..f4bab360663 100644 --- a/packages/dd-trace/test/appsec/iast/taint-tracking/plugin.spec.js +++ b/packages/dd-trace/test/appsec/iast/taint-tracking/plugin.spec.js @@ -42,13 +42,14 @@ describe('IAST Taint tracking plugin', () => { }) it('Should subscribe to body parser, qs, cookie and process_params channel', () => { - expect(taintTrackingPlugin._subscriptions).to.have.lengthOf(6) + expect(taintTrackingPlugin._subscriptions).to.have.lengthOf(7) expect(taintTrackingPlugin._subscriptions[0]._channel.name).to.equals('datadog:body-parser:read:finish') - expect(taintTrackingPlugin._subscriptions[1]._channel.name).to.equals('datadog:qs:parse:finish') - expect(taintTrackingPlugin._subscriptions[2]._channel.name).to.equals('apm:express:middleware:next') - expect(taintTrackingPlugin._subscriptions[3]._channel.name).to.equals('datadog:cookie:parse:finish') - expect(taintTrackingPlugin._subscriptions[4]._channel.name).to.equals('datadog:express:process_params:start') - expect(taintTrackingPlugin._subscriptions[5]._channel.name).to.equals('apm:graphql:resolve:start') + expect(taintTrackingPlugin._subscriptions[1]._channel.name).to.equals('datadog:multer:read:finish') + expect(taintTrackingPlugin._subscriptions[2]._channel.name).to.equals('datadog:qs:parse:finish') + expect(taintTrackingPlugin._subscriptions[3]._channel.name).to.equals('apm:express:middleware:next') + expect(taintTrackingPlugin._subscriptions[4]._channel.name).to.equals('datadog:cookie:parse:finish') + expect(taintTrackingPlugin._subscriptions[5]._channel.name).to.equals('datadog:express:process_params:start') + expect(taintTrackingPlugin._subscriptions[6]._channel.name).to.equals('apm:graphql:resolve:start') }) describe('taint sources', () => { diff --git a/packages/dd-trace/test/appsec/iast/utils.js b/packages/dd-trace/test/appsec/iast/utils.js index 2ef5a77ee30..7ceb7d5d5bd 100644 --- a/packages/dd-trace/test/appsec/iast/utils.js +++ b/packages/dd-trace/test/appsec/iast/utils.js @@ -288,9 +288,10 @@ function prepareTestServerForIastInExpress (description, expressVersion, loadMid before((done) => { const express = require(`../../../../../versions/express@${expressVersion}`).get() const bodyParser = require('../../../../../versions/body-parser').get() + const expressApp = express() - if (loadMiddlewares) loadMiddlewares(expressApp) + if (loadMiddlewares) loadMiddlewares(expressApp, listener) expressApp.use(bodyParser.json()) try { diff --git a/packages/dd-trace/test/appsec/iast/vulnerability-reporter.spec.js b/packages/dd-trace/test/appsec/iast/vulnerability-reporter.spec.js index f498ef6e122..1f4516218af 100644 --- a/packages/dd-trace/test/appsec/iast/vulnerability-reporter.spec.js +++ b/packages/dd-trace/test/appsec/iast/vulnerability-reporter.spec.js @@ -2,7 +2,8 @@ const { addVulnerability, sendVulnerabilities, clearCache, start, stop } = require('../../../src/appsec/iast/vulnerability-reporter') const VulnerabilityAnalyzer = require('../../../../dd-trace/src/appsec/iast/analyzers/vulnerability-analyzer') const appsecStandalone = require('../../../src/appsec/standalone') -const { APPSEC_PROPAGATION_KEY } = require('../../../src/constants') +const { APPSEC_PROPAGATION_KEY, SAMPLING_MECHANISM_APPSEC } = require('../../../src/constants') +const { USER_KEEP } = require('../../../../../ext/priority') describe('vulnerability-reporter', () => { let vulnerabilityAnalyzer @@ -82,9 +83,14 @@ describe('vulnerability-reporter', () => { describe('without rootSpan', () => { let fakeTracer let onTheFlySpan + let prioritySampler beforeEach(() => { + prioritySampler = { + setPriority: sinon.stub() + } onTheFlySpan = { + _prioritySampler: prioritySampler, finish: sinon.spy(), addTags: sinon.spy(), context () { @@ -120,10 +126,11 @@ describe('vulnerability-reporter', () => { '_dd.iast.enabled': 1 }) expect(onTheFlySpan.addTags.secondCall).to.have.been.calledWithExactly({ - 'manual.keep': 'true', '_dd.iast.json': '{"sources":[],"vulnerabilities":[{"type":"INSECURE_HASHING","hash":3410512655,' + '"evidence":{"value":"sha1"},"location":{"spanId":42,"path":"filename.js","line":73}}]}' }) + expect(prioritySampler.setPriority) + .to.have.been.calledOnceWithExactly(onTheFlySpan, USER_KEEP, SAMPLING_MECHANISM_APPSEC) expect(onTheFlySpan.finish).to.have.been.calledOnce }) @@ -140,10 +147,15 @@ describe('vulnerability-reporter', () => { describe('sendVulnerabilities', () => { let span let context + let prioritySampler beforeEach(() => { context = { _trace: { tags: {} } } + prioritySampler = { + setPriority: sinon.stub() + } span = { + _prioritySampler: prioritySampler, addTags: sinon.stub(), context: sinon.stub().returns(context) } @@ -178,10 +190,10 @@ describe('vulnerability-reporter', () => { vulnerabilityAnalyzer._createVulnerability('INSECURE_HASHING', { value: 'sha1' }, 888)) sendVulnerabilities(iastContext.vulnerabilities, span) expect(span.addTags).to.have.been.calledOnceWithExactly({ - 'manual.keep': 'true', '_dd.iast.json': '{"sources":[],"vulnerabilities":[{"type":"INSECURE_HASHING","hash":3254801297,' + '"evidence":{"value":"sha1"},"location":{"spanId":888}}]}' }) + expect(prioritySampler.setPriority).to.have.been.calledOnceWithExactly(span, USER_KEEP, SAMPLING_MECHANISM_APPSEC) }) it('should send only valid vulnerabilities', () => { @@ -191,10 +203,10 @@ describe('vulnerability-reporter', () => { iastContext.vulnerabilities.push({ invalid: 'vulnerability' }) sendVulnerabilities(iastContext.vulnerabilities, span) expect(span.addTags).to.have.been.calledOnceWithExactly({ - 'manual.keep': 'true', '_dd.iast.json': '{"sources":[],"vulnerabilities":[{"type":"INSECURE_HASHING","hash":3254801297,' + '"evidence":{"value":"sha1"},"location":{"spanId":888}}]}' }) + expect(prioritySampler.setPriority).to.have.been.calledOnceWithExactly(span, USER_KEEP, SAMPLING_MECHANISM_APPSEC) }) it('should send vulnerabilities with evidence, ranges and sources', () => { @@ -239,7 +251,6 @@ describe('vulnerability-reporter', () => { sendVulnerabilities(iastContext.vulnerabilities, span) expect(span.addTags).to.have.been.calledOnceWithExactly({ - 'manual.keep': 'true', '_dd.iast.json': '{"sources":[{"origin":"ORIGIN_TYPE_1","name":"PARAMETER_NAME_1","value":"joe"},' + '{"origin":"ORIGIN_TYPE_2","name":"PARAMETER_NAME_2","value":"joe@mail.com"}],' + '"vulnerabilities":[{"type":"SQL_INJECTION","hash":4676753086,' + @@ -249,6 +260,7 @@ describe('vulnerability-reporter', () => { '[{"value":"SELECT id FROM u WHERE email = \'"},{"value":"joe@mail.com","source":1},{"value":"\';"}]},' + '"location":{"spanId":888,"path":"filename.js","line":99}}]}' }) + expect(prioritySampler.setPriority).to.have.been.calledOnceWithExactly(span, USER_KEEP, SAMPLING_MECHANISM_APPSEC) }) it('should send multiple vulnerabilities with same tainted source', () => { @@ -293,7 +305,6 @@ describe('vulnerability-reporter', () => { sendVulnerabilities(iastContext.vulnerabilities, span) expect(span.addTags).to.have.been.calledOnceWithExactly({ - 'manual.keep': 'true', '_dd.iast.json': '{"sources":[{"origin":"ORIGIN_TYPE_1","name":"PARAMETER_NAME_1","value":"joe"}],' + '"vulnerabilities":[{"type":"SQL_INJECTION","hash":4676753086,' + '"evidence":{"valueParts":[{"value":"SELECT * FROM u WHERE name = \'"},{"value":"joe","source":0},' + @@ -302,6 +313,7 @@ describe('vulnerability-reporter', () => { '[{"value":"UPDATE u SET name=\'"},{"value":"joe","source":0},{"value":"\' WHERE id=1;"}]},' + '"location":{"spanId":888,"path":"filename.js","line":99}}]}' }) + expect(prioritySampler.setPriority).to.have.been.calledOnceWithExactly(span, USER_KEEP, SAMPLING_MECHANISM_APPSEC) }) it('should send once with multiple vulnerabilities', () => { @@ -314,7 +326,6 @@ describe('vulnerability-reporter', () => { { path: '/path/to/file3.js', line: 3 })) sendVulnerabilities(iastContext.vulnerabilities, span) expect(span.addTags).to.have.been.calledOnceWithExactly({ - 'manual.keep': 'true', '_dd.iast.json': '{"sources":[],"vulnerabilities":[' + '{"type":"INSECURE_HASHING","hash":1697980169,"evidence":{"value":"sha1"},' + '"location":{"spanId":888,"path":"/path/to/file1.js","line":1}},' + @@ -323,6 +334,7 @@ describe('vulnerability-reporter', () => { '{"type":"INSECURE_HASHING","hash":1755238473,"evidence":{"value":"md5"},' + '"location":{"spanId":-5,"path":"/path/to/file3.js","line":3}}]}' }) + expect(prioritySampler.setPriority).to.have.been.calledOnceWithExactly(span, USER_KEEP, SAMPLING_MECHANISM_APPSEC) }) it('should send once vulnerability with one vulnerability', () => { @@ -332,10 +344,10 @@ describe('vulnerability-reporter', () => { { path: 'filename.js', line: 88 })) sendVulnerabilities(iastContext.vulnerabilities, span) expect(span.addTags).to.have.been.calledOnceWithExactly({ - 'manual.keep': 'true', '_dd.iast.json': '{"sources":[],"vulnerabilities":[{"type":"INSECURE_HASHING","hash":3410512691,' + '"evidence":{"value":"sha1"},"location":{"spanId":888,"path":"filename.js","line":88}}]}' }) + expect(prioritySampler.setPriority).to.have.been.calledOnceWithExactly(span, USER_KEEP, SAMPLING_MECHANISM_APPSEC) }) it('should not send duplicated vulnerabilities', () => { @@ -348,10 +360,10 @@ describe('vulnerability-reporter', () => { { path: 'filename.js', line: 88 })) sendVulnerabilities(iastContext.vulnerabilities, span) expect(span.addTags).to.have.been.calledOnceWithExactly({ - 'manual.keep': 'true', '_dd.iast.json': '{"sources":[],"vulnerabilities":[{"type":"INSECURE_HASHING","hash":3410512691,' + '"evidence":{"value":"sha1"},"location":{"spanId":888,"path":"filename.js","line":88}}]}' }) + expect(prioritySampler.setPriority).to.have.been.calledOnceWithExactly(span, USER_KEEP, SAMPLING_MECHANISM_APPSEC) }) it('should not send duplicated vulnerabilities in multiple sends', () => { @@ -365,10 +377,10 @@ describe('vulnerability-reporter', () => { sendVulnerabilities(iastContext.vulnerabilities, span) sendVulnerabilities(iastContext.vulnerabilities, span) expect(span.addTags).to.have.been.calledOnceWithExactly({ - 'manual.keep': 'true', '_dd.iast.json': '{"sources":[],"vulnerabilities":[{"type":"INSECURE_HASHING","hash":3410512691,' + '"evidence":{"value":"sha1"},"location":{"spanId":888,"path":"filename.js","line":88}}]}' }) + expect(prioritySampler.setPriority).to.have.been.calledOnceWithExactly(span, USER_KEEP, SAMPLING_MECHANISM_APPSEC) }) it('should not deduplicate vulnerabilities if not enabled', () => { @@ -384,12 +396,12 @@ describe('vulnerability-reporter', () => { { value: 'sha1' }, 888, { path: 'filename.js', line: 88 })) sendVulnerabilities(iastContext.vulnerabilities, span) expect(span.addTags).to.have.been.calledOnceWithExactly({ - 'manual.keep': 'true', '_dd.iast.json': '{"sources":[],"vulnerabilities":[{"type":"INSECURE_HASHING","hash":3410512691,' + '"evidence":{"value":"sha1"},"location":{"spanId":888,"path":"filename.js","line":88}},' + '{"type":"INSECURE_HASHING","hash":3410512691,"evidence":{"value":"sha1"},"location":' + '{"spanId":888,"path":"filename.js","line":88}}]}' }) + expect(prioritySampler.setPriority).to.have.been.calledOnceWithExactly(span, USER_KEEP, SAMPLING_MECHANISM_APPSEC) }) it('should add _dd.p.appsec trace tag with standalone enabled', () => { @@ -401,11 +413,12 @@ describe('vulnerability-reporter', () => { sendVulnerabilities(iastContext.vulnerabilities, span) expect(span.addTags).to.have.been.calledOnceWithExactly({ - 'manual.keep': 'true', '_dd.iast.json': '{"sources":[],"vulnerabilities":[{"type":"INSECURE_HASHING","hash":3254801297,' + '"evidence":{"value":"sha1"},"location":{"spanId":999}}]}' }) + expect(prioritySampler.setPriority).to.have.been.calledOnceWithExactly(span, USER_KEEP, SAMPLING_MECHANISM_APPSEC) + expect(span.context()._trace.tags).to.have.property(APPSEC_PROPAGATION_KEY) }) @@ -418,11 +431,12 @@ describe('vulnerability-reporter', () => { sendVulnerabilities(iastContext.vulnerabilities, span) expect(span.addTags).to.have.been.calledOnceWithExactly({ - 'manual.keep': 'true', '_dd.iast.json': '{"sources":[],"vulnerabilities":[{"type":"INSECURE_HASHING","hash":3254801297,' + '"evidence":{"value":"sha1"},"location":{"spanId":999}}]}' }) + expect(prioritySampler.setPriority).to.have.been.calledOnceWithExactly(span, USER_KEEP, SAMPLING_MECHANISM_APPSEC) + expect(span.context()._trace.tags).to.not.have.property(APPSEC_PROPAGATION_KEY) }) }) @@ -441,7 +455,8 @@ describe('vulnerability-reporter', () => { global.setInterval = sinon.spy(global.setInterval) global.clearInterval = sinon.spy(global.clearInterval) span = { - addTags: sinon.stub() + addTags: sinon.stub(), + keep: sinon.stub() } }) diff --git a/packages/dd-trace/test/appsec/rasp/ssrf.express.plugin.spec.js b/packages/dd-trace/test/appsec/rasp/ssrf.express.plugin.spec.js index 26dc25219f4..6b5ba45ad0a 100644 --- a/packages/dd-trace/test/appsec/rasp/ssrf.express.plugin.spec.js +++ b/packages/dd-trace/test/appsec/rasp/ssrf.express.plugin.spec.js @@ -162,6 +162,49 @@ describe('RASP - ssrf', () => { }) }) }) + + describe('Test using request', () => { + withVersions('express', 'request', requestVersion => { + let requestToTest + + beforeEach(() => { + requestToTest = require(`../../../../../versions/request@${requestVersion}`).get() + }) + + it('Should not detect threat', async () => { + app = (req, res) => { + requestToTest.get(`https://${req.query.host}`).on('response', () => { + res.end('end') + }) + } + + axios.get('/?host=www.datadoghq.com') + + return checkRaspExecutedAndNotThreat(agent) + }) + + it('Should detect threat doing a GET request', async () => { + app = async (req, res) => { + try { + requestToTest.get(`https://${req.query.host}`) + .on('error', (e) => { + if (e.message === 'DatadogRaspAbortError') { + res.writeHead(500) + } + res.end('end') + }) + } catch (e) { + if (e.cause.message === 'DatadogRaspAbortError') { + res.writeHead(500) + } + res.end('end') + } + } + + await testBlockingRequest() + }) + }) + }) }) }) diff --git a/packages/dd-trace/test/appsec/reporter.spec.js b/packages/dd-trace/test/appsec/reporter.spec.js index 0860b2c75ac..757884c3566 100644 --- a/packages/dd-trace/test/appsec/reporter.spec.js +++ b/packages/dd-trace/test/appsec/reporter.spec.js @@ -3,6 +3,8 @@ const proxyquire = require('proxyquire') const { storage } = require('../../../datadog-core') const zlib = require('zlib') +const { SAMPLING_MECHANISM_APPSEC } = require('../../src/constants') +const { USER_KEEP } = require('../../../../ext/priority') describe('reporter', () => { let Reporter @@ -10,14 +12,21 @@ describe('reporter', () => { let web let telemetry let sample + let prioritySampler beforeEach(() => { + prioritySampler = { + setPriority: sinon.stub() + } + span = { + _prioritySampler: prioritySampler, context: sinon.stub().returns({ _tags: {} }), addTags: sinon.stub(), - setTag: sinon.stub() + setTag: sinon.stub(), + keep: sinon.stub() } web = { @@ -105,7 +114,6 @@ describe('reporter', () => { expect(Reporter.metricsQueue.get('_dd.appsec.event_rules.error_count')).to.be.eq(1) expect(Reporter.metricsQueue.get('_dd.appsec.event_rules.errors')) .to.be.eq(JSON.stringify(diagnosticsRules.errors)) - expect(Reporter.metricsQueue.get('manual.keep')).to.be.eq('true') }) it('should call incrementWafInitMetric', () => { @@ -222,11 +230,11 @@ describe('reporter', () => { expect(span.addTags).to.have.been.calledOnceWithExactly({ 'appsec.event': 'true', - 'manual.keep': 'true', '_dd.origin': 'appsec', '_dd.appsec.json': '{"triggers":[{"rule":{},"rule_matches":[{}]}]}', 'network.client.ip': '8.8.8.8' }) + expect(prioritySampler.setPriority).to.have.been.calledOnceWithExactly(span, USER_KEEP, SAMPLING_MECHANISM_APPSEC) }) it('should not add manual.keep when rate limit is reached', (done) => { @@ -234,24 +242,23 @@ describe('reporter', () => { const params = {} expect(Reporter.reportAttack('', params)).to.not.be.false - expect(addTags.getCall(0).firstArg).to.have.property('manual.keep').that.equals('true') expect(Reporter.reportAttack('', params)).to.not.be.false - expect(addTags.getCall(1).firstArg).to.have.property('manual.keep').that.equals('true') expect(Reporter.reportAttack('', params)).to.not.be.false - expect(addTags.getCall(2).firstArg).to.have.property('manual.keep').that.equals('true') + + expect(prioritySampler.setPriority).to.have.callCount(3) Reporter.setRateLimit(1) expect(Reporter.reportAttack('', params)).to.not.be.false expect(addTags.getCall(3).firstArg).to.have.property('appsec.event').that.equals('true') - expect(addTags.getCall(3).firstArg).to.have.property('manual.keep').that.equals('true') + expect(prioritySampler.setPriority).to.have.callCount(4) expect(Reporter.reportAttack('', params)).to.not.be.false expect(addTags.getCall(4).firstArg).to.have.property('appsec.event').that.equals('true') - expect(addTags.getCall(4).firstArg).to.not.have.property('manual.keep') + expect(prioritySampler.setPriority).to.have.callCount(4) setTimeout(() => { expect(Reporter.reportAttack('', params)).to.not.be.false - expect(addTags.getCall(5).firstArg).to.have.property('manual.keep').that.equals('true') + expect(prioritySampler.setPriority).to.have.callCount(5) done() }, 1020) }) @@ -265,10 +272,10 @@ describe('reporter', () => { expect(span.addTags).to.have.been.calledOnceWithExactly({ 'appsec.event': 'true', - 'manual.keep': 'true', '_dd.appsec.json': '{"triggers":[]}', 'network.client.ip': '8.8.8.8' }) + expect(prioritySampler.setPriority).to.have.been.calledOnceWithExactly(span, USER_KEEP, SAMPLING_MECHANISM_APPSEC) }) it('should merge attacks json', () => { @@ -280,11 +287,11 @@ describe('reporter', () => { expect(span.addTags).to.have.been.calledOnceWithExactly({ 'appsec.event': 'true', - 'manual.keep': 'true', '_dd.origin': 'appsec', '_dd.appsec.json': '{"triggers":[{"rule":{},"rule_matches":[{}]},{"rule":{}},{"rule":{},"rule_matches":[{}]}]}', 'network.client.ip': '8.8.8.8' }) + expect(prioritySampler.setPriority).to.have.been.calledOnceWithExactly(span, USER_KEEP, SAMPLING_MECHANISM_APPSEC) }) it('should call standalone sample', () => { @@ -296,12 +303,13 @@ describe('reporter', () => { expect(span.addTags).to.have.been.calledOnceWithExactly({ 'appsec.event': 'true', - 'manual.keep': 'true', '_dd.origin': 'appsec', '_dd.appsec.json': '{"triggers":[{"rule":{},"rule_matches":[{}]},{"rule":{}},{"rule":{},"rule_matches":[{}]}]}', 'network.client.ip': '8.8.8.8' }) + expect(prioritySampler.setPriority).to.have.been.calledOnceWithExactly(span, USER_KEEP, SAMPLING_MECHANISM_APPSEC) + expect(sample).to.have.been.calledOnceWithExactly(span) }) }) @@ -642,5 +650,16 @@ describe('reporter', () => { expect(span.setTag).to.have.been.calledWithExactly('_dd.appsec.rasp.duration_ext', 321) expect(span.setTag).to.have.been.calledWithExactly('_dd.appsec.rasp.rule.eval', 3) }) + + it('should keep span if there are metrics', () => { + const req = {} + + Reporter.metricsQueue.set('a', 1) + Reporter.metricsQueue.set('b', 2) + + Reporter.finishRequest(req, wafContext, {}) + + expect(prioritySampler.setPriority).to.have.been.calledOnceWithExactly(span, USER_KEEP, SAMPLING_MECHANISM_APPSEC) + }) }) }) diff --git a/packages/dd-trace/test/appsec/sdk/track_event.spec.js b/packages/dd-trace/test/appsec/sdk/track_event.spec.js index e3739488b81..fca01030c03 100644 --- a/packages/dd-trace/test/appsec/sdk/track_event.spec.js +++ b/packages/dd-trace/test/appsec/sdk/track_event.spec.js @@ -5,6 +5,8 @@ const agent = require('../../plugins/agent') const axios = require('axios') const tracer = require('../../../../../index') const { LOGIN_SUCCESS, LOGIN_FAILURE } = require('../../../src/appsec/addresses') +const { SAMPLING_MECHANISM_APPSEC } = require('../../../src/constants') +const { USER_KEEP } = require('../../../../../ext/priority') describe('track_event', () => { describe('Internal API', () => { @@ -16,14 +18,21 @@ describe('track_event', () => { let trackUserLoginSuccessEvent, trackUserLoginFailureEvent, trackCustomEvent, trackEvent let sample let waf + let prioritySampler beforeEach(() => { log = { warn: sinon.stub() } + prioritySampler = { + setPriority: sinon.stub() + } + rootSpan = { - addTags: sinon.stub() + _prioritySampler: prioritySampler, + addTags: sinon.stub(), + keep: sinon.stub() } getRootSpan = sinon.stub().callsFake(() => rootSpan) @@ -96,12 +105,13 @@ describe('track_event', () => { expect(rootSpan.addTags).to.have.been.calledOnceWithExactly( { 'appsec.events.users.login.success.track': 'true', - 'manual.keep': 'true', '_dd.appsec.events.users.login.success.sdk': 'true', 'appsec.events.users.login.success.metakey1': 'metaValue1', 'appsec.events.users.login.success.metakey2': 'metaValue2', 'appsec.events.users.login.success.metakey3': 'metaValue3' }) + expect(prioritySampler.setPriority) + .to.have.been.calledOnceWithExactly(rootSpan, USER_KEEP, SAMPLING_MECHANISM_APPSEC) }) it('should call setUser and addTags without metadata', () => { @@ -113,9 +123,10 @@ describe('track_event', () => { expect(setUserTags).to.have.been.calledOnceWithExactly(user, rootSpan) expect(rootSpan.addTags).to.have.been.calledOnceWithExactly({ 'appsec.events.users.login.success.track': 'true', - 'manual.keep': 'true', '_dd.appsec.events.users.login.success.sdk': 'true' }) + expect(prioritySampler.setPriority) + .to.have.been.calledOnceWithExactly(rootSpan, USER_KEEP, SAMPLING_MECHANISM_APPSEC) }) it('should call waf run with login success address', () => { @@ -161,7 +172,6 @@ describe('track_event', () => { expect(setUserTags).to.not.have.been.called expect(rootSpan.addTags).to.have.been.calledOnceWithExactly({ 'appsec.events.users.login.failure.track': 'true', - 'manual.keep': 'true', '_dd.appsec.events.users.login.failure.sdk': 'true', 'appsec.events.users.login.failure.usr.id': 'user_id', 'appsec.events.users.login.failure.usr.exists': 'true', @@ -169,6 +179,8 @@ describe('track_event', () => { 'appsec.events.users.login.failure.metakey2': 'metaValue2', 'appsec.events.users.login.failure.metakey3': 'metaValue3' }) + expect(prioritySampler.setPriority) + .to.have.been.calledOnceWithExactly(rootSpan, USER_KEEP, SAMPLING_MECHANISM_APPSEC) }) it('should send false `usr.exists` property when the user does not exist', () => { @@ -180,7 +192,6 @@ describe('track_event', () => { expect(setUserTags).to.not.have.been.called expect(rootSpan.addTags).to.have.been.calledOnceWithExactly({ 'appsec.events.users.login.failure.track': 'true', - 'manual.keep': 'true', '_dd.appsec.events.users.login.failure.sdk': 'true', 'appsec.events.users.login.failure.usr.id': 'user_id', 'appsec.events.users.login.failure.usr.exists': 'false', @@ -188,6 +199,8 @@ describe('track_event', () => { 'appsec.events.users.login.failure.metakey2': 'metaValue2', 'appsec.events.users.login.failure.metakey3': 'metaValue3' }) + expect(prioritySampler.setPriority) + .to.have.been.calledOnceWithExactly(rootSpan, USER_KEEP, SAMPLING_MECHANISM_APPSEC) }) it('should call addTags without metadata', () => { @@ -197,11 +210,12 @@ describe('track_event', () => { expect(setUserTags).to.not.have.been.called expect(rootSpan.addTags).to.have.been.calledOnceWithExactly({ 'appsec.events.users.login.failure.track': 'true', - 'manual.keep': 'true', '_dd.appsec.events.users.login.failure.sdk': 'true', 'appsec.events.users.login.failure.usr.id': 'user_id', 'appsec.events.users.login.failure.usr.exists': 'true' }) + expect(prioritySampler.setPriority) + .to.have.been.calledOnceWithExactly(rootSpan, USER_KEEP, SAMPLING_MECHANISM_APPSEC) }) it('should call waf run with login failure address', () => { @@ -241,11 +255,12 @@ describe('track_event', () => { expect(setUserTags).to.not.have.been.called expect(rootSpan.addTags).to.have.been.calledOnceWithExactly({ 'appsec.events.custom_event.track': 'true', - 'manual.keep': 'true', '_dd.appsec.events.custom_event.sdk': 'true', 'appsec.events.custom_event.metaKey1': 'metaValue1', 'appsec.events.custom_event.metakey2': 'metaValue2' }) + expect(prioritySampler.setPriority) + .to.have.been.calledOnceWithExactly(rootSpan, USER_KEEP, SAMPLING_MECHANISM_APPSEC) }) it('should call addTags without metadata', () => { @@ -255,9 +270,10 @@ describe('track_event', () => { expect(setUserTags).to.not.have.been.called expect(rootSpan.addTags).to.have.been.calledOnceWithExactly({ 'appsec.events.custom_event.track': 'true', - 'manual.keep': 'true', '_dd.appsec.events.custom_event.sdk': 'true' }) + expect(prioritySampler.setPriority) + .to.have.been.calledOnceWithExactly(rootSpan, USER_KEEP, SAMPLING_MECHANISM_APPSEC) }) }) @@ -266,31 +282,34 @@ describe('track_event', () => { trackEvent('event', { metaKey1: 'metaValue1', metakey2: 'metaValue2' }, 'trackEvent', rootSpan, 'safe') expect(rootSpan.addTags).to.have.been.calledOnceWithExactly({ 'appsec.events.event.track': 'true', - 'manual.keep': 'true', '_dd.appsec.events.event.auto.mode': 'safe', 'appsec.events.event.metaKey1': 'metaValue1', 'appsec.events.event.metakey2': 'metaValue2' }) + expect(prioritySampler.setPriority) + .to.have.been.calledOnceWithExactly(rootSpan, USER_KEEP, SAMPLING_MECHANISM_APPSEC) }) it('should call addTags with extended mode', () => { trackEvent('event', { metaKey1: 'metaValue1', metakey2: 'metaValue2' }, 'trackEvent', rootSpan, 'extended') expect(rootSpan.addTags).to.have.been.calledOnceWithExactly({ 'appsec.events.event.track': 'true', - 'manual.keep': 'true', '_dd.appsec.events.event.auto.mode': 'extended', 'appsec.events.event.metaKey1': 'metaValue1', 'appsec.events.event.metakey2': 'metaValue2' }) + expect(prioritySampler.setPriority) + .to.have.been.calledOnceWithExactly(rootSpan, USER_KEEP, SAMPLING_MECHANISM_APPSEC) }) it('should call standalone sample', () => { trackEvent('event', undefined, 'trackEvent', rootSpan, undefined) expect(rootSpan.addTags).to.have.been.calledOnceWithExactly({ - 'appsec.events.event.track': 'true', - 'manual.keep': 'true' + 'appsec.events.event.track': 'true' }) + expect(prioritySampler.setPriority) + .to.have.been.calledOnceWithExactly(rootSpan, USER_KEEP, SAMPLING_MECHANISM_APPSEC) expect(sample).to.have.been.calledOnceWithExactly(rootSpan) }) }) @@ -339,7 +358,7 @@ describe('track_event', () => { expect(traces[0][0].meta).to.have.property('appsec.events.users.login.success.track', 'true') expect(traces[0][0].meta).to.have.property('usr.id', 'test_user_id') expect(traces[0][0].meta).to.have.property('appsec.events.users.login.success.metakey', 'metaValue') - expect(traces[0][0].meta).to.have.property('manual.keep', 'true') + expect(traces[0][0].metrics).to.have.property('_sampling_priority_v1', USER_KEEP) }).then(done).catch(done) axios.get(`http://localhost:${port}/`) }) @@ -377,7 +396,7 @@ describe('track_event', () => { expect(traces[0][0].meta).to.have.property('appsec.events.users.login.failure.usr.id', 'test_user_id') expect(traces[0][0].meta).to.have.property('appsec.events.users.login.failure.usr.exists', 'true') expect(traces[0][0].meta).to.have.property('appsec.events.users.login.failure.metakey', 'metaValue') - expect(traces[0][0].meta).to.have.property('manual.keep', 'true') + expect(traces[0][0].metrics).to.have.property('_sampling_priority_v1', USER_KEEP) }).then(done).catch(done) axios.get(`http://localhost:${port}/`) }) @@ -392,7 +411,7 @@ describe('track_event', () => { expect(traces[0][0].meta).to.have.property('appsec.events.users.login.failure.usr.id', 'test_user_id') expect(traces[0][0].meta).to.have.property('appsec.events.users.login.failure.usr.exists', 'false') expect(traces[0][0].meta).to.have.property('appsec.events.users.login.failure.metakey', 'metaValue') - expect(traces[0][0].meta).to.have.property('manual.keep', 'true') + expect(traces[0][0].metrics).to.have.property('_sampling_priority_v1', USER_KEEP) }).then(done).catch(done) axios.get(`http://localhost:${port}/`) }) @@ -428,7 +447,7 @@ describe('track_event', () => { agent.use(traces => { expect(traces[0][0].meta).to.have.property('appsec.events.my-custom-event.track', 'true') expect(traces[0][0].meta).to.have.property('appsec.events.my-custom-event.metakey', 'metaValue') - expect(traces[0][0].meta).to.have.property('manual.keep', 'true') + expect(traces[0][0].metrics).to.have.property('_sampling_priority_v1', USER_KEEP) }).then(done).catch(done) axios.get(`http://localhost:${port}/`) }) @@ -440,7 +459,7 @@ describe('track_event', () => { res.end() } agent.use(traces => { - expect(traces[0][0].meta).to.not.have.property('manual.keep', 'true') + expect(traces[0][0].metrics).to.not.have.property('_sampling_priority_v1', USER_KEEP) }).then(done).catch(done) axios.get(`http://localhost:${port}/`) }) diff --git a/packages/dd-trace/test/appsec/waf/index.spec.js b/packages/dd-trace/test/appsec/waf/index.spec.js index b0c16647872..33c0bfbb3a3 100644 --- a/packages/dd-trace/test/appsec/waf/index.spec.js +++ b/packages/dd-trace/test/appsec/waf/index.spec.js @@ -27,7 +27,11 @@ describe('WAF Manager', () => { DDWAF.prototype.constructor.version = sinon.stub() DDWAF.prototype.dispose = sinon.stub() DDWAF.prototype.createContext = sinon.stub() - DDWAF.prototype.update = sinon.stub() + DDWAF.prototype.update = sinon.stub().callsFake(function (newRules) { + if (newRules?.metadata?.rules_version) { + this.diagnostics.ruleset_version = newRules?.metadata?.rules_version + } + }) DDWAF.prototype.diagnostics = { ruleset_version: '1.0.0', rules: { @@ -77,7 +81,6 @@ describe('WAF Manager', () => { expect(Reporter.metricsQueue.set).to.been.calledWithExactly('_dd.appsec.event_rules.loaded', 1) expect(Reporter.metricsQueue.set).to.been.calledWithExactly('_dd.appsec.event_rules.error_count', 0) expect(Reporter.metricsQueue.set).not.to.been.calledWith('_dd.appsec.event_rules.errors') - expect(Reporter.metricsQueue.set).to.been.calledWithExactly('manual.keep', 'true') }) it('should set init metrics with errors', () => { @@ -100,7 +103,6 @@ describe('WAF Manager', () => { expect(Reporter.metricsQueue.set).to.been.calledWithExactly('_dd.appsec.event_rules.error_count', 2) expect(Reporter.metricsQueue.set).to.been.calledWithExactly('_dd.appsec.event_rules.errors', '{"error_1":["invalid_1"],"error_2":["invalid_2","invalid_3"]}') - expect(Reporter.metricsQueue.set).to.been.calledWithExactly('manual.keep', 'true') }) }) @@ -177,10 +179,14 @@ describe('WAF Manager', () => { waf.update(rules) expect(DDWAF.prototype.update).to.be.calledOnceWithExactly(rules) + expect(Reporter.reportWafUpdate).to.be.calledOnceWithExactly(wafVersion, '1.0.0') }) it('should call Reporter.reportWafUpdate', () => { const rules = { + metadata: { + rules_version: '4.2.0' + }, rules_data: [ { id: 'blocked_users', @@ -197,8 +203,7 @@ describe('WAF Manager', () => { waf.update(rules) - expect(Reporter.reportWafUpdate).to.be.calledOnceWithExactly(wafVersion, - DDWAF.prototype.diagnostics.ruleset_version) + expect(Reporter.reportWafUpdate).to.be.calledOnceWithExactly(wafVersion, '4.2.0') }) }) diff --git a/packages/dd-trace/test/azure_metadata.spec.js b/packages/dd-trace/test/azure_metadata.spec.js new file mode 100644 index 00000000000..7a8cb787d75 --- /dev/null +++ b/packages/dd-trace/test/azure_metadata.spec.js @@ -0,0 +1,109 @@ +'use strict' + +require('./setup/tap') + +const os = require('os') +const { getAzureAppMetadata, getAzureTagsFromMetadata } = require('../src/azure_metadata') + +describe('Azure metadata', () => { + describe('for apps is', () => { + it('not provided without DD_AZURE_APP_SERVICES', () => { + delete process.env.DD_AZURE_APP_SERVICES + expect(getAzureAppMetadata()).to.be.undefined + }) + + it('provided with DD_AZURE_APP_SERVICES', () => { + delete process.env.COMPUTERNAME // actually defined on Windows + process.env.DD_AZURE_APP_SERVICES = '1' + delete process.env.WEBSITE_SITE_NAME + expect(getAzureAppMetadata()).to.deep.equal({ operatingSystem: os.platform(), siteKind: 'app', siteType: 'app' }) + }) + }) + + it('provided completely with minimum vars', () => { + delete process.env.WEBSITE_RESOURCE_GROUP + delete process.env.WEBSITE_OS + delete process.env.FUNCTIONS_EXTENSION_VERSION + delete process.env.FUNCTIONS_WORKER_RUNTIME + delete process.env.FUNCTIONS_WORKER_RUNTIME_VERSION + process.env.COMPUTERNAME = 'boaty_mcboatface' + process.env.DD_AZURE_APP_SERVICES = '1' + process.env.WEBSITE_SITE_NAME = 'website_name' + process.env.WEBSITE_OWNER_NAME = 'subscription_id+resource_group-regionwebspace' + process.env.WEBSITE_INSTANCE_ID = 'instance_id' + process.env.DD_AAS_DOTNET_EXTENSION_VERSION = '1.0' + const expected = { + extensionVersion: '1.0', + instanceID: 'instance_id', + instanceName: 'boaty_mcboatface', + operatingSystem: os.platform(), + resourceGroup: 'resource_group', + resourceID: + '/subscriptions/subscription_id/resourcegroups/resource_group/providers/microsoft.web/sites/website_name', + siteKind: 'app', + siteName: 'website_name', + siteType: 'app', + subscriptionID: 'subscription_id' + } + expect(getAzureAppMetadata()).to.deep.equal(expected) + }) + + it('provided completely with complete vars', () => { + process.env.COMPUTERNAME = 'boaty_mcboatface' + process.env.DD_AZURE_APP_SERVICES = '1' + process.env.WEBSITE_SITE_NAME = 'website_name' + process.env.WEBSITE_RESOURCE_GROUP = 'resource_group' + process.env.WEBSITE_OWNER_NAME = 'subscription_id+foo-regionwebspace' + process.env.WEBSITE_OS = 'windows' + process.env.WEBSITE_INSTANCE_ID = 'instance_id' + process.env.FUNCTIONS_EXTENSION_VERSION = '20' + process.env.FUNCTIONS_WORKER_RUNTIME = 'node' + process.env.FUNCTIONS_WORKER_RUNTIME_VERSION = '14' + process.env.DD_AAS_DOTNET_EXTENSION_VERSION = '1.0' + const expected = { + extensionVersion: '1.0', + functionRuntimeVersion: '20', + instanceID: 'instance_id', + instanceName: 'boaty_mcboatface', + operatingSystem: 'windows', + resourceGroup: 'resource_group', + resourceID: + '/subscriptions/subscription_id/resourcegroups/resource_group/providers/microsoft.web/sites/website_name', + runtime: 'node', + runtimeVersion: '14', + siteKind: 'functionapp', + siteName: 'website_name', + siteType: 'function', + subscriptionID: 'subscription_id' + } + expect(getAzureAppMetadata()).to.deep.equal(expected) + }) + + it('tags are correctly generated from vars', () => { + delete process.env.WEBSITE_RESOURCE_GROUP + delete process.env.WEBSITE_OS + delete process.env.FUNCTIONS_EXTENSION_VERSION + delete process.env.FUNCTIONS_WORKER_RUNTIME + delete process.env.FUNCTIONS_WORKER_RUNTIME_VERSION + process.env.COMPUTERNAME = 'boaty_mcboatface' + process.env.DD_AZURE_APP_SERVICES = '1' + process.env.WEBSITE_SITE_NAME = 'website_name' + process.env.WEBSITE_OWNER_NAME = 'subscription_id+resource_group-regionwebspace' + process.env.WEBSITE_INSTANCE_ID = 'instance_id' + process.env.DD_AAS_DOTNET_EXTENSION_VERSION = '1.0' + const expected = { + 'aas.environment.extension_version': '1.0', + 'aas.environment.instance_id': 'instance_id', + 'aas.environment.instance_name': 'boaty_mcboatface', + 'aas.environment.os': os.platform(), + 'aas.resource.group': 'resource_group', + 'aas.resource.id': + '/subscriptions/subscription_id/resourcegroups/resource_group/providers/microsoft.web/sites/website_name', + 'aas.site.kind': 'app', + 'aas.site.name': 'website_name', + 'aas.site.type': 'app', + 'aas.subscription.id': 'subscription_id' + } + expect(getAzureTagsFromMetadata(getAzureAppMetadata())).to.deep.equal(expected) + }) +}) diff --git a/packages/dd-trace/test/ci-visibility/exporters/agent-proxy/agent-proxy.spec.js b/packages/dd-trace/test/ci-visibility/exporters/agent-proxy/agent-proxy.spec.js index 4ff8f12ace6..1abae9e82f1 100644 --- a/packages/dd-trace/test/ci-visibility/exporters/agent-proxy/agent-proxy.spec.js +++ b/packages/dd-trace/test/ci-visibility/exporters/agent-proxy/agent-proxy.spec.js @@ -6,6 +6,7 @@ const nock = require('nock') const AgentProxyCiVisibilityExporter = require('../../../../src/ci-visibility/exporters/agent-proxy') const AgentlessWriter = require('../../../../src/ci-visibility/exporters/agentless/writer') +const DynamicInstrumentationLogsWriter = require('../../../../src/ci-visibility/exporters/agentless/di-logs-writer') const CoverageWriter = require('../../../../src/ci-visibility/exporters/agentless/coverage-writer') const AgentWriter = require('../../../../src/exporters/agent/writer') @@ -68,7 +69,10 @@ describe('AgentProxyCiVisibilityExporter', () => { .get('/info') .delay(queryDelay) .reply(200, JSON.stringify({ - endpoints: ['/evp_proxy/v2/'] + endpoints: [ + '/evp_proxy/v2/', + '/debugger/v1/input' + ] })) }) @@ -112,6 +116,35 @@ describe('AgentProxyCiVisibilityExporter', () => { agentProxyCiVisibilityExporter.exportCoverage(coverage) expect(mockWriter.append).to.have.been.calledWith({ spanId: '1', traceId: '1', files: [] }) }) + + context('if isTestDynamicInstrumentationEnabled is set', () => { + it('should initialise DynamicInstrumentationLogsWriter', async () => { + const agentProxyCiVisibilityExporter = new AgentProxyCiVisibilityExporter({ + port, + tags, + isTestDynamicInstrumentationEnabled: true + }) + await agentProxyCiVisibilityExporter._canUseCiVisProtocolPromise + expect(agentProxyCiVisibilityExporter._logsWriter).to.be.instanceOf(DynamicInstrumentationLogsWriter) + }) + + it('should process logs', async () => { + const mockWriter = { + append: sinon.spy(), + flush: sinon.spy() + } + const agentProxyCiVisibilityExporter = new AgentProxyCiVisibilityExporter({ + port, + tags, + isTestDynamicInstrumentationEnabled: true + }) + await agentProxyCiVisibilityExporter._canUseCiVisProtocolPromise + agentProxyCiVisibilityExporter._logsWriter = mockWriter + const log = { message: 'hello' } + agentProxyCiVisibilityExporter.exportDiLogs({}, log) + expect(mockWriter.append).to.have.been.calledWith(sinon.match(log)) + }) + }) }) describe('agent is not evp compatible', () => { @@ -166,6 +199,35 @@ describe('AgentProxyCiVisibilityExporter', () => { }) expect(mockWriter.append).not.to.have.been.called }) + + context('if isTestDynamicInstrumentationEnabled is set', () => { + it('should not initialise DynamicInstrumentationLogsWriter', async () => { + const agentProxyCiVisibilityExporter = new AgentProxyCiVisibilityExporter({ + port, + tags, + isTestDynamicInstrumentationEnabled: true + }) + await agentProxyCiVisibilityExporter._canUseCiVisProtocolPromise + expect(agentProxyCiVisibilityExporter._logsWriter).to.be.undefined + }) + + it('should not process logs', async () => { + const mockWriter = { + append: sinon.spy(), + flush: sinon.spy() + } + const agentProxyCiVisibilityExporter = new AgentProxyCiVisibilityExporter({ + port, + tags, + isTestDynamicInstrumentationEnabled: true + }) + await agentProxyCiVisibilityExporter._canUseCiVisProtocolPromise + agentProxyCiVisibilityExporter._logsWriter = mockWriter + const log = { message: 'hello' } + agentProxyCiVisibilityExporter.exportDiLogs({}, log) + expect(mockWriter.append).not.to.have.been.called + }) + }) }) describe('export', () => { diff --git a/packages/dd-trace/test/ci-visibility/exporters/agentless/di-logs-writer.spec.js b/packages/dd-trace/test/ci-visibility/exporters/agentless/di-logs-writer.spec.js new file mode 100644 index 00000000000..85a674a0d85 --- /dev/null +++ b/packages/dd-trace/test/ci-visibility/exporters/agentless/di-logs-writer.spec.js @@ -0,0 +1,105 @@ +'use strict' + +require('../../../../../dd-trace/test/setup/tap') + +const { expect } = require('chai') +const sinon = require('sinon') +const nock = require('nock') +const DynamicInstrumentationLogsWriter = require('../../../../src/ci-visibility/exporters/agentless/di-logs-writer') +const log = require('../../../../src/log') + +describe('Test Visibility DI Writer', () => { + beforeEach(() => { + nock.cleanAll() + process.env.DD_API_KEY = '1' + }) + + afterEach(() => { + delete process.env.DD_API_KEY + sinon.restore() + }) + + context('agentless', () => { + it('can send logs to the logs intake', (done) => { + const scope = nock('http://www.example.com') + .post('/api/v2/logs', body => { + expect(body).to.deep.equal([{ message: 'test' }, { message: 'test2' }]) + return true + }) + .reply(202) + + const logsWriter = new DynamicInstrumentationLogsWriter({ url: 'http://www.example.com' }) + + logsWriter.append({ message: 'test' }) + logsWriter.append({ message: 'test2' }) + + logsWriter.flush(() => { + scope.done() + done() + }) + }) + + it('logs an error if the request fails', (done) => { + const logErrorSpy = sinon.spy(log, 'error') + + const scope = nock('http://www.example.com') + .post('/api/v2/logs') + .reply(500) + + const logsWriter = new DynamicInstrumentationLogsWriter({ url: 'http://www.example.com' }) + + logsWriter.append({ message: 'test5' }) + logsWriter.append({ message: 'test6' }) + + logsWriter.flush(() => { + expect(logErrorSpy.called).to.be.true + scope.done() + done() + }) + }) + }) + + context('agent based', () => { + it('can send logs to the debugger endpoint in the agent', (done) => { + delete process.env.DD_API_KEY + + const scope = nock('http://www.example.com') + .post('/debugger/v1/input', body => { + expect(body).to.deep.equal([{ message: 'test3' }, { message: 'test4' }]) + return true + }) + .reply(202) + + const logsWriter = new DynamicInstrumentationLogsWriter({ url: 'http://www.example.com', isAgentProxy: true }) + + logsWriter.append({ message: 'test3' }) + logsWriter.append({ message: 'test4' }) + + logsWriter.flush(() => { + scope.done() + done() + }) + }) + + it('logs an error if the request fails', (done) => { + delete process.env.DD_API_KEY + + const logErrorSpy = sinon.spy(log, 'error') + + const scope = nock('http://www.example.com') + .post('/debugger/v1/input') + .reply(500) + + const logsWriter = new DynamicInstrumentationLogsWriter({ url: 'http://www.example.com', isAgentProxy: true }) + + logsWriter.append({ message: 'test5' }) + logsWriter.append({ message: 'test6' }) + + logsWriter.flush(() => { + expect(logErrorSpy.called).to.be.true + scope.done() + done() + }) + }) + }) +}) diff --git a/packages/dd-trace/test/ci-visibility/exporters/agentless/exporter.spec.js b/packages/dd-trace/test/ci-visibility/exporters/agentless/exporter.spec.js index 11b3bf1ec4c..dd229984bd2 100644 --- a/packages/dd-trace/test/ci-visibility/exporters/agentless/exporter.spec.js +++ b/packages/dd-trace/test/ci-visibility/exporters/agentless/exporter.spec.js @@ -8,6 +8,7 @@ const { expect } = require('chai') const nock = require('nock') const AgentlessCiVisibilityExporter = require('../../../../src/ci-visibility/exporters/agentless') +const DynamicInstrumentationLogsWriter = require('../../../../src/ci-visibility/exporters/agentless/di-logs-writer') describe('CI Visibility Agentless Exporter', () => { const url = new URL('http://www.example.com') @@ -177,6 +178,33 @@ describe('CI Visibility Agentless Exporter', () => { }) }) + context('if isTestDynamicInstrumentationEnabled is set', () => { + it('should initialise DynamicInstrumentationLogsWriter', async () => { + const agentProxyCiVisibilityExporter = new AgentlessCiVisibilityExporter({ + tags: {}, + isTestDynamicInstrumentationEnabled: true + }) + await agentProxyCiVisibilityExporter._canUseCiVisProtocolPromise + expect(agentProxyCiVisibilityExporter._logsWriter).to.be.instanceOf(DynamicInstrumentationLogsWriter) + }) + + it('should process logs', async () => { + const mockWriter = { + append: sinon.spy(), + flush: sinon.spy() + } + const agentProxyCiVisibilityExporter = new AgentlessCiVisibilityExporter({ + tags: {}, + isTestDynamicInstrumentationEnabled: true + }) + await agentProxyCiVisibilityExporter._canUseCiVisProtocolPromise + agentProxyCiVisibilityExporter._logsWriter = mockWriter + const log = { message: 'hello' } + agentProxyCiVisibilityExporter.exportDiLogs({}, log) + expect(mockWriter.append).to.have.been.calledWith(sinon.match(log)) + }) + }) + describe('url', () => { it('sets the default if URL param is not specified', () => { const site = 'd4tad0g.com' diff --git a/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-exporter.spec.js b/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-exporter.spec.js index b92d5b3ae98..7b09f8fba2d 100644 --- a/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-exporter.spec.js +++ b/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-exporter.spec.js @@ -815,4 +815,97 @@ describe('CI Visibility Exporter', () => { }) }) }) + + describe('exportDiLogs', () => { + context('is not initialized', () => { + it('should do nothing', () => { + const log = { message: 'log' } + const ciVisibilityExporter = new CiVisibilityExporter({ port, isTestDynamicInstrumentationEnabled: true }) + ciVisibilityExporter.exportDiLogs(log) + ciVisibilityExporter._export = sinon.spy() + expect(ciVisibilityExporter._export).not.to.be.called + }) + }) + + context('is initialized but can not forward logs', () => { + it('should do nothing', () => { + const writer = { + append: sinon.spy(), + flush: sinon.spy(), + setUrl: sinon.spy() + } + const log = { message: 'log' } + const ciVisibilityExporter = new CiVisibilityExporter({ port, isTestDynamicInstrumentationEnabled: true }) + ciVisibilityExporter._isInitialized = true + ciVisibilityExporter._logsWriter = writer + ciVisibilityExporter._canForwardLogs = false + ciVisibilityExporter.exportDiLogs(log) + expect(ciVisibilityExporter._logsWriter.append).not.to.be.called + }) + }) + + context('is initialized and can forward logs', () => { + it('should export formatted logs', () => { + const writer = { + append: sinon.spy(), + flush: sinon.spy(), + setUrl: sinon.spy() + } + const diLog = { + message: 'log', + debugger: { + snapshot: { + id: '1234', + timestamp: 1234567890, + probe: { + id: '54321', + version: '1', + location: { + file: 'example.js', + lines: ['1'] + } + }, + stack: [ + { + fileName: 'example.js', + function: 'sum', + lineNumber: 1 + } + ], + language: 'javascript' + } + } + } + const ciVisibilityExporter = new CiVisibilityExporter({ + env: 'ci', + version: '1.0.0', + port, + isTestDynamicInstrumentationEnabled: true, + service: 'my-service' + }) + ciVisibilityExporter._isInitialized = true + ciVisibilityExporter._logsWriter = writer + ciVisibilityExporter._canForwardLogs = true + ciVisibilityExporter.exportDiLogs( + { + 'git.repository_url': 'https://github.com/datadog/dd-trace-js.git', + 'git.commit.sha': '1234' + }, + diLog + ) + expect(ciVisibilityExporter._logsWriter.append).to.be.calledWith(sinon.match({ + ddtags: 'git.repository_url:https://github.com/datadog/dd-trace-js.git,git.commit.sha:1234', + level: 'error', + ddsource: 'dd_debugger', + service: 'my-service', + dd: { + service: 'my-service', + env: 'ci', + version: '1.0.0' + }, + ...diLog + })) + }) + }) + }) }) diff --git a/packages/dd-trace/test/config.spec.js b/packages/dd-trace/test/config.spec.js index 6558485b529..804476a87c9 100644 --- a/packages/dd-trace/test/config.spec.js +++ b/packages/dd-trace/test/config.spec.js @@ -5,6 +5,7 @@ require('./setup/tap') const { expect } = require('chai') const { readFileSync } = require('fs') const sinon = require('sinon') +const { GRPC_CLIENT_ERROR_STATUSES, GRPC_SERVER_ERROR_STATUSES } = require('../src/constants') describe('Config', () => { let Config @@ -215,6 +216,7 @@ describe('Config', () => { expect(config).to.have.property('runtimeMetrics', false) expect(config.tags).to.have.property('service', 'node') expect(config).to.have.property('plugins', true) + expect(config).to.have.property('traceEnabled', true) expect(config).to.have.property('env', undefined) expect(config).to.have.property('reportHostname', false) expect(config).to.have.property('scope', undefined) @@ -224,6 +226,8 @@ describe('Config', () => { expect(config).to.have.property('traceId128BitGenerationEnabled', true) expect(config).to.have.property('traceId128BitLoggingEnabled', false) expect(config).to.have.property('spanAttributeSchema', 'v0') + expect(config.grpc.client.error.statuses).to.deep.equal(GRPC_CLIENT_ERROR_STATUSES) + expect(config.grpc.server.error.statuses).to.deep.equal(GRPC_SERVER_ERROR_STATUSES) expect(config).to.have.property('spanComputePeerService', false) expect(config).to.have.property('spanRemoveIntegrationFromService', false) expect(config).to.have.property('instrumentation_config_id', undefined) @@ -262,6 +266,9 @@ describe('Config', () => { expect(config).to.have.nested.property('installSignature.id', null) expect(config).to.have.nested.property('installSignature.time', null) expect(config).to.have.nested.property('installSignature.type', null) + expect(config).to.have.nested.property('llmobs.mlApp', undefined) + expect(config).to.have.nested.property('llmobs.agentlessEnabled', false) + expect(config).to.have.nested.property('llmobs.enabled', false) expect(updateConfig).to.be.calledOnce @@ -326,7 +333,11 @@ describe('Config', () => { { name: 'isGitUploadEnabled', value: false, origin: 'default' }, { name: 'isIntelligentTestRunnerEnabled', value: false, origin: 'default' }, { name: 'isManualApiEnabled', value: false, origin: 'default' }, + { name: 'llmobs.agentlessEnabled', value: false, origin: 'default' }, + { name: 'llmobs.mlApp', value: undefined, origin: 'default' }, { name: 'ciVisibilityTestSessionName', value: '', origin: 'default' }, + { name: 'ciVisAgentlessLogSubmissionEnabled', value: false, origin: 'default' }, + { name: 'isTestDynamicInstrumentationEnabled', value: false, origin: 'default' }, { name: 'logInjection', value: false, origin: 'default' }, { name: 'lookup', value: undefined, origin: 'default' }, { name: 'openAiLogsEnabled', value: false, origin: 'default' }, @@ -350,7 +361,8 @@ describe('Config', () => { { name: 'reportHostname', value: false, origin: 'default' }, { name: 'runtimeMetrics', value: false, origin: 'default' }, { name: 'sampleRate', value: undefined, origin: 'default' }, - { name: 'sampler.rateLimit', value: undefined, origin: 'default' }, + { name: 'sampler.rateLimit', value: 100, origin: 'default' }, + { name: 'traceEnabled', value: true, origin: 'default' }, { name: 'sampler.rules', value: [], origin: 'default' }, { name: 'scope', value: undefined, origin: 'default' }, { name: 'service', value: 'node', origin: 'default' }, @@ -495,6 +507,11 @@ describe('Config', () => { process.env.DD_INSTRUMENTATION_INSTALL_TYPE = 'k8s_single_step' process.env.DD_INSTRUMENTATION_INSTALL_TIME = '1703188212' process.env.DD_INSTRUMENTATION_CONFIG_ID = 'abcdef123' + process.env.DD_LLMOBS_AGENTLESS_ENABLED = 'true' + process.env.DD_LLMOBS_ML_APP = 'myMlApp' + process.env.DD_TRACE_ENABLED = 'true' + process.env.DD_GRPC_CLIENT_ERROR_STATUSES = '3,13,400-403' + process.env.DD_GRPC_SERVER_ERROR_STATUSES = '3,13,400-403' // required if we want to check updates to config.debug and config.logLevel which is fetched from logger reloadLoggerAndConfig() @@ -512,12 +529,15 @@ describe('Config', () => { expect(config).to.have.property('queryStringObfuscation', '.*') expect(config).to.have.property('clientIpEnabled', true) expect(config).to.have.property('clientIpHeader', 'x-true-client-ip') + expect(config.grpc.client.error.statuses).to.deep.equal([3, 13, 400, 401, 402, 403]) + expect(config.grpc.server.error.statuses).to.deep.equal([3, 13, 400, 401, 402, 403]) expect(config).to.have.property('runtimeMetrics', true) expect(config).to.have.property('reportHostname', true) expect(config).to.have.nested.property('codeOriginForSpans.enabled', true) expect(config).to.have.property('dynamicInstrumentationEnabled', true) expect(config).to.have.property('env', 'test') expect(config).to.have.property('sampleRate', 0.5) + expect(config).to.have.property('traceEnabled', true) expect(config).to.have.property('traceId128BitGenerationEnabled', true) expect(config).to.have.property('traceId128BitLoggingEnabled', true) expect(config).to.have.property('spanAttributeSchema', 'v1') @@ -591,6 +611,8 @@ describe('Config', () => { type: 'k8s_single_step', time: '1703188212' }) + expect(config).to.have.nested.property('llmobs.mlApp', 'myMlApp') + expect(config).to.have.nested.property('llmobs.agentlessEnabled', true) expect(updateConfig).to.be.calledOnce @@ -656,7 +678,9 @@ describe('Config', () => { { name: 'traceId128BitGenerationEnabled', value: true, origin: 'env_var' }, { name: 'traceId128BitLoggingEnabled', value: true, origin: 'env_var' }, { name: 'tracing', value: false, origin: 'env_var' }, - { name: 'version', value: '1.0.0', origin: 'env_var' } + { name: 'version', value: '1.0.0', origin: 'env_var' }, + { name: 'llmobs.mlApp', value: 'myMlApp', origin: 'env_var' }, + { name: 'llmobs.agentlessEnabled', value: true, origin: 'env_var' } ]) }) @@ -806,7 +830,12 @@ describe('Config', () => { pollInterval: 42 }, traceId128BitGenerationEnabled: true, - traceId128BitLoggingEnabled: true + traceId128BitLoggingEnabled: true, + llmobs: { + mlApp: 'myMlApp', + agentlessEnabled: true, + apiKey: 'myApiKey' + } }) expect(config).to.have.property('protocolVersion', '0.5') @@ -881,6 +910,8 @@ describe('Config', () => { a: 'aa', b: 'bb' }) + expect(config).to.have.nested.property('llmobs.mlApp', 'myMlApp') + expect(config).to.have.nested.property('llmobs.agentlessEnabled', true) expect(updateConfig).to.be.calledOnce @@ -928,7 +959,9 @@ describe('Config', () => { { name: 'stats.enabled', value: false, origin: 'calculated' }, { name: 'traceId128BitGenerationEnabled', value: true, origin: 'code' }, { name: 'traceId128BitLoggingEnabled', value: true, origin: 'code' }, - { name: 'version', value: '0.1.0', origin: 'code' } + { name: 'version', value: '0.1.0', origin: 'code' }, + { name: 'llmobs.mlApp', value: 'myMlApp', origin: 'code' }, + { name: 'llmobs.agentlessEnabled', value: true, origin: 'code' } ]) }) @@ -997,6 +1030,32 @@ describe('Config', () => { expect(config).to.have.property('spanAttributeSchema', 'v0') }) + it('should parse integer range sets', () => { + process.env.DD_GRPC_CLIENT_ERROR_STATUSES = '3,13,400-403' + process.env.DD_GRPC_SERVER_ERROR_STATUSES = '3,13,400-403' + + let config = new Config() + + expect(config.grpc.client.error.statuses).to.deep.equal([3, 13, 400, 401, 402, 403]) + expect(config.grpc.server.error.statuses).to.deep.equal([3, 13, 400, 401, 402, 403]) + + process.env.DD_GRPC_CLIENT_ERROR_STATUSES = '1' + process.env.DD_GRPC_SERVER_ERROR_STATUSES = '1' + + config = new Config() + + expect(config.grpc.client.error.statuses).to.deep.equal([1]) + expect(config.grpc.server.error.statuses).to.deep.equal([1]) + + process.env.DD_GRPC_CLIENT_ERROR_STATUSES = '2,10,13-15' + process.env.DD_GRPC_SERVER_ERROR_STATUSES = '2,10,13-15' + + config = new Config() + + expect(config.grpc.client.error.statuses).to.deep.equal([2, 10, 13, 14, 15]) + expect(config.grpc.server.error.statuses).to.deep.equal([2, 10, 13, 14, 15]) + }) + context('peer service tagging', () => { it('should activate peer service only if explicitly true in v0', () => { process.env.DD_TRACE_SPAN_ATTRIBUTE_SCHEMA = 'v0' @@ -1103,6 +1162,8 @@ describe('Config', () => { process.env.DD_IAST_REDACTION_VALUE_PATTERN = 'value_pattern_to_be_overriden_by_options' process.env.DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED = 'true' process.env.DD_TRACE_128_BIT_TRACEID_LOGGING_ENABLED = 'true' + process.env.DD_LLMOBS_ML_APP = 'myMlApp' + process.env.DD_LLMOBS_AGENTLESS_ENABLED = 'true' const config = new Config({ protocolVersion: '0.5', @@ -1184,7 +1245,11 @@ describe('Config', () => { enabled: false }, traceId128BitGenerationEnabled: false, - traceId128BitLoggingEnabled: false + traceId128BitLoggingEnabled: false, + llmobs: { + mlApp: 'myOtherMlApp', + agentlessEnabled: false + } }) expect(config).to.have.property('protocolVersion', '0.5') @@ -1245,6 +1310,8 @@ describe('Config', () => { expect(config).to.have.nested.property('iast.redactionEnabled', true) expect(config).to.have.nested.property('iast.redactionNamePattern', 'REDACTION_NAME_PATTERN') expect(config).to.have.nested.property('iast.redactionValuePattern', 'REDACTION_VALUE_PATTERN') + expect(config).to.have.nested.property('llmobs.mlApp', 'myOtherMlApp') + expect(config).to.have.nested.property('llmobs.agentlessEnabled', false) }) it('should give priority to non-experimental options', () => { @@ -1633,7 +1700,7 @@ describe('Config', () => { }, true) expect(config).to.have.deep.nested.property('sampler', { spanSamplingRules: [], - rateLimit: undefined, + rateLimit: 100, rules: [ { resource: '*', @@ -1838,6 +1905,8 @@ describe('Config', () => { delete process.env.DD_CIVISIBILITY_FLAKY_RETRY_COUNT delete process.env.DD_TEST_SESSION_NAME delete process.env.JEST_WORKER_ID + delete process.env.DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED + delete process.env.DD_AGENTLESS_LOG_SUBMISSION_ENABLED options = {} }) context('ci visibility mode is enabled', () => { @@ -1926,6 +1995,24 @@ describe('Config', () => { const config = new Config(options) expect(config).to.have.property('ciVisibilityTestSessionName', 'my-test-session') }) + it('should not enable agentless log submission by default', () => { + const config = new Config(options) + expect(config).to.have.property('ciVisAgentlessLogSubmissionEnabled', false) + }) + it('should enable agentless log submission if DD_AGENTLESS_LOG_SUBMISSION_ENABLED is true', () => { + process.env.DD_AGENTLESS_LOG_SUBMISSION_ENABLED = 'true' + const config = new Config(options) + expect(config).to.have.property('ciVisAgentlessLogSubmissionEnabled', true) + }) + it('should not set isTestDynamicInstrumentationEnabled by default', () => { + const config = new Config(options) + expect(config).to.have.property('isTestDynamicInstrumentationEnabled', false) + }) + it('should set isTestDynamicInstrumentationEnabled if DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED is passed', () => { + process.env.DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED = 'true' + const config = new Config(options) + expect(config).to.have.property('isTestDynamicInstrumentationEnabled', true) + }) }) context('ci visibility mode is not enabled', () => { it('should not activate intelligent test runner or git metadata upload', () => { @@ -2017,6 +2104,61 @@ describe('Config', () => { }) }) + context('llmobs config', () => { + it('should disable llmobs by default', () => { + const config = new Config() + expect(config.llmobs.enabled).to.be.false + + // check origin computation + expect(updateConfig.getCall(0).args[0]).to.deep.include({ + name: 'llmobs.enabled', value: false, origin: 'default' + }) + }) + + it('should enable llmobs if DD_LLMOBS_ENABLED is set to true', () => { + process.env.DD_LLMOBS_ENABLED = 'true' + const config = new Config() + expect(config.llmobs.enabled).to.be.true + + // check origin computation + expect(updateConfig.getCall(0).args[0]).to.deep.include({ + name: 'llmobs.enabled', value: true, origin: 'env_var' + }) + }) + + it('should disable llmobs if DD_LLMOBS_ENABLED is set to false', () => { + process.env.DD_LLMOBS_ENABLED = 'false' + const config = new Config() + expect(config.llmobs.enabled).to.be.false + + // check origin computation + expect(updateConfig.getCall(0).args[0]).to.deep.include({ + name: 'llmobs.enabled', value: false, origin: 'env_var' + }) + }) + + it('should enable llmobs with options and DD_LLMOBS_ENABLED is not set', () => { + const config = new Config({ llmobs: {} }) + expect(config.llmobs.enabled).to.be.true + + // check origin computation + expect(updateConfig.getCall(0).args[0]).to.deep.include({ + name: 'llmobs.enabled', value: true, origin: 'code' + }) + }) + + it('should have DD_LLMOBS_ENABLED take priority over options', () => { + process.env.DD_LLMOBS_ENABLED = 'false' + const config = new Config({ llmobs: {} }) + expect(config.llmobs.enabled).to.be.false + + // check origin computation + expect(updateConfig.getCall(0).args[0]).to.deep.include({ + name: 'llmobs.enabled', value: false, origin: 'env_var' + }) + }) + }) + it('should sanitize values for API Security sampling between 0 and 1', () => { expect(new Config({ appsec: { diff --git a/packages/dd-trace/test/config/disabled_instrumentations.spec.js b/packages/dd-trace/test/config/disabled_instrumentations.spec.js index d54ee38f677..c7f9b935fb5 100644 --- a/packages/dd-trace/test/config/disabled_instrumentations.spec.js +++ b/packages/dd-trace/test/config/disabled_instrumentations.spec.js @@ -1,11 +1,23 @@ 'use strict' -process.env.DD_TRACE_DISABLED_INSTRUMENTATIONS = 'express' - require('../setup/tap') describe('config/disabled_instrumentations', () => { it('should disable loading instrumentations completely', () => { + process.env.DD_TRACE_DISABLED_INSTRUMENTATIONS = 'express' + const handleBefore = require('express').application.handle + const tracer = require('../../../..') + const handleAfterImport = require('express').application.handle + tracer.init() + const handleAfterInit = require('express').application.handle + + expect(handleBefore).to.equal(handleAfterImport) + expect(handleBefore).to.equal(handleAfterInit) + delete process.env.DD_TRACE_DISABLED_INSTRUMENTATIONS + }) + + it('should disable loading instrumentations using DD_TRACE__ENABLED', () => { + process.env.DD_TRACE_EXPRESS_ENABLED = 'false' const handleBefore = require('express').application.handle const tracer = require('../../../..') const handleAfterImport = require('express').application.handle @@ -14,5 +26,6 @@ describe('config/disabled_instrumentations', () => { expect(handleBefore).to.equal(handleAfterImport) expect(handleBefore).to.equal(handleAfterInit) + delete process.env.DD_TRACE_EXPRESS_ENABLED }) }) diff --git a/packages/dd-trace/test/datastreams/data_streams_checkpointer.spec.js b/packages/dd-trace/test/datastreams/data_streams_checkpointer.spec.js index ba33d4c8bdf..db29f96b575 100644 --- a/packages/dd-trace/test/datastreams/data_streams_checkpointer.spec.js +++ b/packages/dd-trace/test/datastreams/data_streams_checkpointer.spec.js @@ -2,8 +2,8 @@ require('../setup/tap') const agent = require('../plugins/agent') -const expectedProducerHash = '13182885521735152072' -const expectedConsumerHash = '5980058680018671020' +const expectedProducerHash = '11369286567396183453' +const expectedConsumerHash = '11204511019589278729' const DSM_CONTEXT_HEADER = 'dd-pathway-ctx-base64' describe('data streams checkpointer manual api', () => { diff --git a/packages/dd-trace/test/debugger/devtools_client/snapshot/complex-types.spec.js b/packages/dd-trace/test/debugger/devtools_client/snapshot/complex-types.spec.js index 22036e4c60a..0e46a2faba0 100644 --- a/packages/dd-trace/test/debugger/devtools_client/snapshot/complex-types.spec.js +++ b/packages/dd-trace/test/debugger/devtools_client/snapshot/complex-types.spec.js @@ -23,7 +23,7 @@ describe('debugger -> devtools client -> snapshot.getLocalStateForCallFrame', fu session.once('Debugger.paused', async ({ params }) => { expect(params.hitBreakpoints.length).to.eq(1) - resolve((await getLocalStateForCallFrame(params.callFrames[0]))()) + resolve((await getLocalStateForCallFrame(params.callFrames[0], { maxFieldCount: Number.MAX_SAFE_INTEGER }))()) }) await setAndTriggerBreakpoint(target, 10) diff --git a/packages/dd-trace/test/debugger/devtools_client/snapshot/max-collection-size.spec.js b/packages/dd-trace/test/debugger/devtools_client/snapshot/max-collection-size.spec.js new file mode 100644 index 00000000000..6b63eec715e --- /dev/null +++ b/packages/dd-trace/test/debugger/devtools_client/snapshot/max-collection-size.spec.js @@ -0,0 +1,129 @@ +'use strict' + +require('../../../setup/mocha') + +const { getTargetCodePath, enable, teardown, assertOnBreakpoint, setAndTriggerBreakpoint } = require('./utils') + +const DEFAULT_MAX_COLLECTION_SIZE = 100 +const target = getTargetCodePath(__filename) + +describe('debugger -> devtools client -> snapshot.getLocalStateForCallFrame', function () { + describe('maxCollectionSize', function () { + const configs = [ + undefined, + { maxCollectionSize: 3 } + ] + + beforeEach(enable(__filename)) + + afterEach(teardown) + + for (const config of configs) { + const maxCollectionSize = config?.maxCollectionSize ?? DEFAULT_MAX_COLLECTION_SIZE + const postfix = config === undefined ? 'not set' : `set to ${config.maxCollectionSize}` + + describe(`shold respect the default maxCollectionSize if ${postfix}`, function () { + let state + + const expectedElements = [] + const expectedEntries = [] + for (let i = 1; i <= maxCollectionSize; i++) { + expectedElements.push({ type: 'number', value: i.toString() }) + expectedEntries.push([ + { type: 'number', value: i.toString() }, + { + type: 'Object', + fields: { i: { type: 'number', value: i.toString() } } + } + ]) + } + + beforeEach(function (done) { + assertOnBreakpoint(done, config, (_state) => { + state = _state + }) + setAndTriggerBreakpoint(target, 24) + }) + + it('should have expected number of elements in state', function () { + expect(state).to.have.keys(['arr', 'map', 'set', 'wmap', 'wset', 'typedArray']) + }) + + it('Array', function () { + expect(state).to.have.deep.property('arr', { + type: 'Array', + elements: expectedElements, + notCapturedReason: 'collectionSize', + size: 1000 + }) + }) + + it('Map', function () { + expect(state).to.have.deep.property('map', { + type: 'Map', + entries: expectedEntries, + notCapturedReason: 'collectionSize', + size: 1000 + }) + }) + + it('Set', function () { + expect(state).to.have.deep.property('set', { + type: 'Set', + elements: expectedElements, + notCapturedReason: 'collectionSize', + size: 1000 + }) + }) + + it('WeakMap', function () { + expect(state.wmap).to.include({ + type: 'WeakMap', + notCapturedReason: 'collectionSize', + size: 1000 + }) + + expect(state.wmap.entries).to.have.lengthOf(maxCollectionSize) + + // The order of the entries is not guaranteed, so we don't know which were removed + for (const entry of state.wmap.entries) { + expect(entry).to.have.lengthOf(2) + expect(entry[0]).to.have.property('type', 'Object') + expect(entry[0].fields).to.have.property('i') + expect(entry[0].fields.i).to.have.property('type', 'number') + expect(entry[0].fields.i).to.have.property('value').to.match(/^\d+$/) + expect(entry[1]).to.have.property('type', 'number') + expect(entry[1]).to.have.property('value', entry[0].fields.i.value) + } + }) + + it('WeakSet', function () { + expect(state.wset).to.include({ + type: 'WeakSet', + notCapturedReason: 'collectionSize', + size: 1000 + }) + + expect(state.wset.elements).to.have.lengthOf(maxCollectionSize) + + // The order of the elements is not guaranteed, so we don't know which were removed + for (const element of state.wset.elements) { + expect(element).to.have.property('type', 'Object') + expect(element.fields).to.have.property('i') + expect(element.fields.i).to.have.property('type', 'number') + expect(element.fields.i).to.have.property('value').to.match(/^\d+$/) + } + }) + + it('TypedArray', function () { + expect(state).to.have.deep.property('typedArray', { + type: 'Uint16Array', + elements: expectedElements, + notCapturedReason: 'collectionSize', + size: 1000 + }) + }) + }) + } + }) +}) diff --git a/packages/dd-trace/test/debugger/devtools_client/snapshot/max-field-count-scopes.spec.js b/packages/dd-trace/test/debugger/devtools_client/snapshot/max-field-count-scopes.spec.js new file mode 100644 index 00000000000..1f3fb8c14c6 --- /dev/null +++ b/packages/dd-trace/test/debugger/devtools_client/snapshot/max-field-count-scopes.spec.js @@ -0,0 +1,32 @@ +'use strict' + +require('../../../setup/mocha') + +const { getTargetCodePath, enable, teardown, assertOnBreakpoint, setAndTriggerBreakpoint } = require('./utils') + +const target = getTargetCodePath(__filename) + +describe('debugger -> devtools client -> snapshot.getLocalStateForCallFrame', function () { + describe('maxFieldCount', function () { + beforeEach(enable(__filename)) + + afterEach(teardown) + + describe('shold respect maxFieldCount on each collected scope', function () { + const maxFieldCount = 3 + let state + + beforeEach(function (done) { + assertOnBreakpoint(done, { maxFieldCount }, (_state) => { + state = _state + }) + setAndTriggerBreakpoint(target, 11) + }) + + it('should capture expected snapshot', function () { + // Expect the snapshot to have captured the first 3 fields from each scope + expect(state).to.have.keys(['a1', 'b1', 'c1', 'a2', 'b2', 'c2']) + }) + }) + }) +}) diff --git a/packages/dd-trace/test/debugger/devtools_client/snapshot/max-field-count.spec.js b/packages/dd-trace/test/debugger/devtools_client/snapshot/max-field-count.spec.js new file mode 100644 index 00000000000..a9507151209 --- /dev/null +++ b/packages/dd-trace/test/debugger/devtools_client/snapshot/max-field-count.spec.js @@ -0,0 +1,49 @@ +'use strict' + +require('../../../setup/mocha') + +const { getTargetCodePath, enable, teardown, assertOnBreakpoint, setAndTriggerBreakpoint } = require('./utils') + +const DEFAULT_MAX_FIELD_COUNT = 20 +const target = getTargetCodePath(__filename) + +describe('debugger -> devtools client -> snapshot.getLocalStateForCallFrame', function () { + describe('maxFieldCount', function () { + beforeEach(enable(__filename)) + + afterEach(teardown) + + describe('shold respect the default maxFieldCount if not set', generateTestCases()) + + describe('shold respect maxFieldCount if set to 10', generateTestCases({ maxFieldCount: 10 })) + }) +}) + +function generateTestCases (config) { + const maxFieldCount = config?.maxFieldCount ?? DEFAULT_MAX_FIELD_COUNT + let state + + const expectedFields = {} + for (let i = 1; i <= maxFieldCount; i++) { + expectedFields[`field${i}`] = { type: 'number', value: i.toString() } + } + + return function () { + beforeEach(function (done) { + assertOnBreakpoint(done, config, (_state) => { + state = _state + }) + setAndTriggerBreakpoint(target, 11) + }) + + it('should capture expected snapshot', function () { + expect(state).to.have.keys(['obj']) + expect(state).to.have.deep.property('obj', { + type: 'Object', + fields: expectedFields, + notCapturedReason: 'fieldCount', + size: 40 + }) + }) + } +} diff --git a/packages/dd-trace/test/debugger/devtools_client/snapshot/target-code/max-collection-size.js b/packages/dd-trace/test/debugger/devtools_client/snapshot/target-code/max-collection-size.js new file mode 100644 index 00000000000..09c8ca81100 --- /dev/null +++ b/packages/dd-trace/test/debugger/devtools_client/snapshot/target-code/max-collection-size.js @@ -0,0 +1,27 @@ +'use stict' + +function run () { + const arr = [] + const map = new Map() + const set = new Set() + const wmap = new WeakMap() + const wset = new WeakSet() + const typedArray = new Uint16Array(new ArrayBuffer(2000)) + + // 1000 is larger the default maxCollectionSize of 100 + for (let i = 1; i <= 1000; i++) { + // A reference that can be used in WeakMap/WeakSet to avoid GC + const obj = { i } + + arr.push(i) + map.set(i, obj) + set.add(i) + wmap.set(obj, i) + wset.add(obj) + typedArray[i - 1] = i + } + + return 'my return value' // breakpoint at this line +} + +module.exports = { run } diff --git a/packages/dd-trace/test/debugger/devtools_client/snapshot/target-code/max-field-count-scopes.js b/packages/dd-trace/test/debugger/devtools_client/snapshot/target-code/max-field-count-scopes.js new file mode 100644 index 00000000000..90b317b8104 --- /dev/null +++ b/packages/dd-trace/test/debugger/devtools_client/snapshot/target-code/max-field-count-scopes.js @@ -0,0 +1,15 @@ +'use stict' + +function run () { + // local scope + const { a1, b1, c1, d1 } = {} + + { + // block scope + const { a2, b2, c2, d2 } = {} + + return { a1, b1, c1, d1, a2, b2, c2, d2 } // breakpoint at this line + } +} + +module.exports = { run } diff --git a/packages/dd-trace/test/debugger/devtools_client/snapshot/target-code/max-field-count.js b/packages/dd-trace/test/debugger/devtools_client/snapshot/target-code/max-field-count.js new file mode 100644 index 00000000000..ea8eb955079 --- /dev/null +++ b/packages/dd-trace/test/debugger/devtools_client/snapshot/target-code/max-field-count.js @@ -0,0 +1,14 @@ +'use stict' + +function run () { + const obj = {} + + // 40 is larger the default maxFieldCount of 20 + for (let i = 1; i <= 40; i++) { + obj[`field${i}`] = i + } + + return 'my return value' // breakpoint at this line +} + +module.exports = { run } diff --git a/packages/dd-trace/test/llmobs/index.spec.js b/packages/dd-trace/test/llmobs/index.spec.js new file mode 100644 index 00000000000..cdceeab64ab --- /dev/null +++ b/packages/dd-trace/test/llmobs/index.spec.js @@ -0,0 +1,137 @@ +'use strict' + +const proxyquire = require('proxyquire') + +const { channel } = require('dc-polyfill') +const spanProcessCh = channel('dd-trace:span:process') +const evalMetricAppendCh = channel('llmobs:eval-metric:append') +const flushCh = channel('llmobs:writers:flush') +const injectCh = channel('dd-trace:span:inject') + +const LLMObsEvalMetricsWriter = require('../../src/llmobs/writers/evaluations') + +const config = { + llmobs: { + mlApp: 'test' + } +} + +describe('module', () => { + let llmobsModule + let store + let logger + + let LLMObsAgentlessSpanWriter + let LLMObsAgentProxySpanWriter + + before(() => { + sinon.stub(LLMObsEvalMetricsWriter.prototype, 'append') + }) + + beforeEach(() => { + store = {} + logger = { debug: sinon.stub() } + + LLMObsAgentlessSpanWriter = sinon.stub().returns({ + destroy: sinon.stub() + }) + LLMObsAgentProxySpanWriter = sinon.stub().returns({ + destroy: sinon.stub() + }) + + llmobsModule = proxyquire('../../../dd-trace/src/llmobs', { + '../log': logger, + './writers/spans/agentless': LLMObsAgentlessSpanWriter, + './writers/spans/agentProxy': LLMObsAgentProxySpanWriter, + './storage': { + storage: { + getStore () { + return store + } + } + } + }) + + process.removeAllListeners('beforeExit') + }) + + afterEach(() => { + LLMObsAgentProxySpanWriter.resetHistory() + LLMObsAgentlessSpanWriter.resetHistory() + LLMObsEvalMetricsWriter.prototype.append.resetHistory() + llmobsModule.disable() + }) + + after(() => { + LLMObsEvalMetricsWriter.prototype.append.restore() + sinon.restore() + + // get rid of mock stubs for writers + delete require.cache[require.resolve('../../../dd-trace/src/llmobs')] + }) + + describe('handle llmobs info injection', () => { + it('injects LLMObs parent ID when there is a parent LLMObs span', () => { + llmobsModule.enable(config) + store.span = { + context () { + return { + toSpanId () { + return 'parent-id' + } + } + } + } + + const carrier = { + 'x-datadog-tags': '' + } + injectCh.publish({ carrier }) + + expect(carrier['x-datadog-tags']).to.equal(',_dd.p.llmobs_parent_id=parent-id') + }) + + it('does not inject LLMObs parent ID when there is no parent LLMObs span', () => { + llmobsModule.enable(config) + + const carrier = { + 'x-datadog-tags': '' + } + injectCh.publish({ carrier }) + expect(carrier['x-datadog-tags']).to.equal('') + }) + }) + + it('uses the agent proxy span writer', () => { + llmobsModule.enable(config) + expect(LLMObsAgentProxySpanWriter).to.have.been.called + }) + + it('uses the agentless span writer', () => { + config.llmobs.agentlessEnabled = true + llmobsModule.enable(config) + expect(LLMObsAgentlessSpanWriter).to.have.been.called + delete config.llmobs.agentlessEnabled + }) + + it('appends to the eval metric writer', () => { + llmobsModule.enable(config) + + const payload = {} + + evalMetricAppendCh.publish(payload) + + expect(LLMObsEvalMetricsWriter.prototype.append).to.have.been.calledWith(payload) + }) + + it('removes all subscribers when disabling', () => { + llmobsModule.enable(config) + + llmobsModule.disable() + + expect(injectCh.hasSubscribers).to.be.false + expect(evalMetricAppendCh.hasSubscribers).to.be.false + expect(spanProcessCh.hasSubscribers).to.be.false + expect(flushCh.hasSubscribers).to.be.false + }) +}) diff --git a/packages/dd-trace/test/llmobs/noop.spec.js b/packages/dd-trace/test/llmobs/noop.spec.js new file mode 100644 index 00000000000..36dd2279390 --- /dev/null +++ b/packages/dd-trace/test/llmobs/noop.spec.js @@ -0,0 +1,58 @@ +'use strict' + +describe('noop', () => { + let tracer + let llmobs + + before(() => { + tracer = new (require('../../../dd-trace/src/noop/proxy'))() + llmobs = tracer.llmobs + }) + + const nonTracingOps = ['enable', 'disable', 'annotate', 'exportSpan', 'submitEvaluation', 'flush'] + for (const op of nonTracingOps) { + it(`using "${op}" should not throw`, () => { + llmobs[op]() + }) + } + + describe('trace', () => { + it('should not throw with just a span', () => { + const res = llmobs.trace({}, (span) => { + expect(() => span.setTag('foo', 'bar')).does.not.throw + return 1 + }) + + expect(res).to.equal(1) + }) + + it('should not throw with a span and a callback', async () => { + const prom = llmobs.trace({}, (span, cb) => { + expect(() => span.setTag('foo', 'bar')).does.not.throw + expect(() => cb()).does.not.throw + return Promise.resolve(5) + }) + + expect(await prom).to.equal(5) + }) + }) + + describe('wrap', () => { + it('should not throw with just a span', () => { + function fn () { + return 1 + } + + const wrapped = llmobs.wrap({}, fn) + expect(wrapped()).to.equal(1) + }) + + it('should not throw with a span and a callback', async () => { + function fn () { + return Promise.resolve(5) + } + const wrapped = llmobs.wrap({}, fn) + expect(await wrapped()).to.equal(5) + }) + }) +}) diff --git a/packages/dd-trace/test/llmobs/plugins/openai/openaiv3.spec.js b/packages/dd-trace/test/llmobs/plugins/openai/openaiv3.spec.js new file mode 100644 index 00000000000..e78fa298b8c --- /dev/null +++ b/packages/dd-trace/test/llmobs/plugins/openai/openaiv3.spec.js @@ -0,0 +1,382 @@ +'use strict' + +const agent = require('../../../plugins/agent') +const Sampler = require('../../../../src/sampler') +const { DogStatsDClient } = require('../../../../src/dogstatsd') +const { NoopExternalLogger } = require('../../../../src/external-logger/src') + +const nock = require('nock') +const { expectedLLMObsLLMSpanEvent, deepEqualWithMockValues } = require('../../util') +const chai = require('chai') +const semver = require('semver') +const LLMObsAgentProxySpanWriter = require('../../../../src/llmobs/writers/spans/agentProxy') + +const { expect } = chai + +chai.Assertion.addMethod('deepEqualWithMockValues', deepEqualWithMockValues) + +const satisfiesChatCompletion = version => semver.intersects('>=3.2.0', version) + +describe('integrations', () => { + let openai + + describe('openai', () => { + before(() => { + sinon.stub(LLMObsAgentProxySpanWriter.prototype, 'append') + + // reduce errors related to too many listeners + process.removeAllListeners('beforeExit') + + sinon.stub(DogStatsDClient.prototype, '_add') + sinon.stub(NoopExternalLogger.prototype, 'log') + sinon.stub(Sampler.prototype, 'isSampled').returns(true) + + LLMObsAgentProxySpanWriter.prototype.append.reset() + + return agent.load('openai', {}, { + llmobs: { + mlApp: 'test' + } + }) + }) + + afterEach(() => { + nock.cleanAll() + LLMObsAgentProxySpanWriter.prototype.append.reset() + }) + + after(() => { + require('../../../../../dd-trace').llmobs.disable() // unsubscribe from all events + sinon.restore() + return agent.close({ ritmReset: false, wipe: true }) + }) + + withVersions('openai', 'openai', '<4', version => { + const moduleRequirePath = `../../../../../../versions/openai@${version}` + + beforeEach(() => { + const requiredModule = require(moduleRequirePath) + const module = requiredModule.get() + + const { Configuration, OpenAIApi } = module + + const configuration = new Configuration({ + apiKey: 'sk-DATADOG-ACCEPTANCE-TESTS' + }) + + openai = new OpenAIApi(configuration) + }) + + it('submits a completion span', async () => { + nock('https://api.openai.com:443') + .post('/v1/completions') + .reply(200, { + model: 'text-davinci-002', + choices: [{ + text: 'I am doing well, how about you?', + index: 0, + logprobs: null, + finish_reason: 'length' + }], + usage: { prompt_tokens: 3, completion_tokens: 16, total_tokens: 19 } + }, []) + + const checkSpan = agent.use(traces => { + const span = traces[0][0] + const spanEvent = LLMObsAgentProxySpanWriter.prototype.append.getCall(0).args[0] + + const expected = expectedLLMObsLLMSpanEvent({ + span, + spanKind: 'llm', + name: 'openai.createCompletion', + inputMessages: [ + { content: 'How are you?' } + ], + outputMessages: [ + { content: 'I am doing well, how about you?' } + ], + tokenMetrics: { input_tokens: 3, output_tokens: 16, total_tokens: 19 }, + modelName: 'text-davinci-002', + modelProvider: 'openai', + metadata: {}, + tags: { ml_app: 'test', language: 'javascript' } + }) + + expect(spanEvent).to.deepEqualWithMockValues(expected) + }) + + await openai.createCompletion({ + model: 'text-davinci-002', + prompt: 'How are you?' + }) + + await checkSpan + }) + + if (satisfiesChatCompletion(version)) { + it('submits a chat completion span', async () => { + nock('https://api.openai.com:443') + .post('/v1/chat/completions') + .reply(200, { + id: 'chatcmpl-7GaWqyMTD9BLmkmy8SxyjUGX3KSRN', + object: 'chat.completion', + created: 1684188020, + model: 'gpt-3.5-turbo-0301', + usage: { + prompt_tokens: 37, + completion_tokens: 10, + total_tokens: 47 + }, + choices: [{ + message: { + role: 'assistant', + content: 'I am doing well, how about you?' + }, + finish_reason: 'length', + index: 0 + }] + }, []) + + const checkSpan = agent.use(traces => { + const span = traces[0][0] + const spanEvent = LLMObsAgentProxySpanWriter.prototype.append.getCall(0).args[0] + + const expected = expectedLLMObsLLMSpanEvent({ + span, + spanKind: 'llm', + name: 'openai.createChatCompletion', + inputMessages: [ + { role: 'system', content: 'You are a helpful assistant' }, + { role: 'user', content: 'How are you?' } + ], + outputMessages: [ + { role: 'assistant', content: 'I am doing well, how about you?' } + ], + tokenMetrics: { input_tokens: 37, output_tokens: 10, total_tokens: 47 }, + modelName: 'gpt-3.5-turbo-0301', + modelProvider: 'openai', + metadata: {}, + tags: { ml_app: 'test', language: 'javascript' } + }) + + expect(spanEvent).to.deepEqualWithMockValues(expected) + }) + + await openai.createChatCompletion({ + model: 'gpt-3.5-turbo-0301', + messages: [ + { role: 'system', content: 'You are a helpful assistant' }, + { role: 'user', content: 'How are you?' } + ] + }) + + await checkSpan + }) + } + + it('submits an embedding span', async () => { + nock('https://api.openai.com:443') + .post('/v1/embeddings') + .reply(200, { + object: 'list', + data: [{ + object: 'embedding', + index: 0, + embedding: [-0.0034387498, -0.026400521] + }], + model: 'text-embedding-ada-002-v2', + usage: { + prompt_tokens: 2, + total_tokens: 2 + } + }, []) + + const checkSpan = agent.use(traces => { + const span = traces[0][0] + const spanEvent = LLMObsAgentProxySpanWriter.prototype.append.getCall(0).args[0] + + const expected = expectedLLMObsLLMSpanEvent({ + span, + spanKind: 'embedding', + name: 'openai.createEmbedding', + inputDocuments: [ + { text: 'Hello, world!' } + ], + outputValue: '[1 embedding(s) returned with size 2]', + tokenMetrics: { input_tokens: 2, total_tokens: 2 }, + modelName: 'text-embedding-ada-002-v2', + modelProvider: 'openai', + metadata: { encoding_format: 'float' }, + tags: { ml_app: 'test', language: 'javascript' } + }) + + expect(spanEvent).to.deepEqualWithMockValues(expected) + }) + + await openai.createEmbedding({ + model: 'text-embedding-ada-002-v2', + input: 'Hello, world!' + }) + + await checkSpan + }) + + if (satisfiesChatCompletion(version)) { + it('submits a chat completion span with functions', async () => { + nock('https://api.openai.com:443') + .post('/v1/chat/completions') + .reply(200, { + id: 'chatcmpl-7GaWqyMTD9BLmkmy8SxyjUGX3KSRN', + object: 'chat.completion', + created: 1684188020, + model: 'gpt-3.5-turbo-0301', + usage: { + prompt_tokens: 37, + completion_tokens: 10, + total_tokens: 47 + }, + choices: [{ + message: { + role: 'assistant', + content: 'THOUGHT: I will use the "extract_fictional_info" tool', + function_call: { + name: 'extract_fictional_info', + arguments: '{"name":"SpongeBob","origin":"Bikini Bottom"}' + } + }, + finish_reason: 'function_call', + index: 0 + }] + }, []) + + const checkSpan = agent.use(traces => { + const span = traces[0][0] + const spanEvent = LLMObsAgentProxySpanWriter.prototype.append.getCall(0).args[0] + + const expected = expectedLLMObsLLMSpanEvent({ + span, + spanKind: 'llm', + name: 'openai.createChatCompletion', + modelName: 'gpt-3.5-turbo-0301', + modelProvider: 'openai', + inputMessages: [{ role: 'user', content: 'What is SpongeBob SquarePants\'s origin?' }], + outputMessages: [{ + role: 'assistant', + content: 'THOUGHT: I will use the "extract_fictional_info" tool', + tool_calls: [ + { + name: 'extract_fictional_info', + arguments: { + name: 'SpongeBob', + origin: 'Bikini Bottom' + } + } + ] + }], + metadata: { function_call: 'auto' }, + tags: { ml_app: 'test', language: 'javascript' }, + tokenMetrics: { input_tokens: 37, output_tokens: 10, total_tokens: 47 } + }) + + expect(spanEvent).to.deepEqualWithMockValues(expected) + }) + + await openai.createChatCompletion({ + model: 'gpt-3.5-turbo-0301', + messages: [{ role: 'user', content: 'What is SpongeBob SquarePants\'s origin?' }], + functions: [{ type: 'function', functiin: { /* this doesn't matter */} }], + function_call: 'auto' + }) + + await checkSpan + }) + } + + it('submits a completion span with an error', async () => { + nock('https://api.openai.com:443') + .post('/v1/completions') + .reply(400, {}) + + let error + const checkSpan = agent.use(traces => { + const span = traces[0][0] + const spanEvent = LLMObsAgentProxySpanWriter.prototype.append.getCall(0).args[0] + + const expected = expectedLLMObsLLMSpanEvent({ + span, + spanKind: 'llm', + name: 'openai.createCompletion', + inputMessages: [{ content: 'Hello' }], + outputMessages: [{ content: '' }], + modelName: 'gpt-3.5-turbo', + modelProvider: 'openai', + metadata: { max_tokens: 50 }, + tags: { ml_app: 'test', language: 'javascript' }, + error, + errorType: error.type || error.name, + errorMessage: error.message, + errorStack: error.stack + }) + + expect(spanEvent).to.deepEqualWithMockValues(expected) + }) + + try { + await openai.createCompletion({ + model: 'gpt-3.5-turbo', + prompt: 'Hello', + max_tokens: 50 + }) + } catch (e) { + error = e + } + + await checkSpan + }) + + if (satisfiesChatCompletion(version)) { + it('submits a chat completion span with an error', async () => { + nock('https://api.openai.com:443') + .post('/v1/chat/completions') + .reply(400, {}) + + let error + const checkSpan = agent.use(traces => { + const span = traces[0][0] + const spanEvent = LLMObsAgentProxySpanWriter.prototype.append.getCall(0).args[0] + + const expected = expectedLLMObsLLMSpanEvent({ + span, + spanKind: 'llm', + name: 'openai.createChatCompletion', + inputMessages: [{ role: 'user', content: 'Hello' }], + outputMessages: [{ content: '' }], + modelName: 'gpt-3.5-turbo', + modelProvider: 'openai', + metadata: { max_tokens: 50 }, + tags: { ml_app: 'test', language: 'javascript' }, + error, + errorType: error.type || error.name, + errorMessage: error.message, + errorStack: error.stack + }) + + expect(spanEvent).to.deepEqualWithMockValues(expected) + }) + + try { + await openai.createChatCompletion({ + model: 'gpt-3.5-turbo', + messages: [{ role: 'user', content: 'Hello' }], + max_tokens: 50 + }) + } catch (e) { + error = e + } + + await checkSpan + }) + } + }) + }) +}) diff --git a/packages/dd-trace/test/llmobs/plugins/openai/openaiv4.spec.js b/packages/dd-trace/test/llmobs/plugins/openai/openaiv4.spec.js new file mode 100644 index 00000000000..0d4e369525f --- /dev/null +++ b/packages/dd-trace/test/llmobs/plugins/openai/openaiv4.spec.js @@ -0,0 +1,554 @@ +'use strict' + +const fs = require('fs') +const Path = require('path') +const agent = require('../../../plugins/agent') +const Sampler = require('../../../../src/sampler') +const { DogStatsDClient } = require('../../../../src/dogstatsd') +const { NoopExternalLogger } = require('../../../../src/external-logger/src') + +const nock = require('nock') +const { expectedLLMObsLLMSpanEvent, deepEqualWithMockValues } = require('../../util') +const chai = require('chai') +const semver = require('semver') +const LLMObsAgentProxySpanWriter = require('../../../../src/llmobs/writers/spans/agentProxy') + +const { expect } = chai + +chai.Assertion.addMethod('deepEqualWithMockValues', deepEqualWithMockValues) + +const baseOpenAITestsPath = '../../../../../datadog-plugin-openai/test/' + +const satisfiesTools = version => semver.intersects('>4.16.0', version) +const satisfiesStream = version => semver.intersects('>4.1.0', version) + +describe('integrations', () => { + let openai + + describe('openai', () => { + before(() => { + sinon.stub(LLMObsAgentProxySpanWriter.prototype, 'append') + + // reduce errors related to too many listeners + process.removeAllListeners('beforeExit') + + sinon.stub(DogStatsDClient.prototype, '_add') + sinon.stub(NoopExternalLogger.prototype, 'log') + sinon.stub(Sampler.prototype, 'isSampled').returns(true) + + LLMObsAgentProxySpanWriter.prototype.append.reset() + + return agent.load('openai', {}, { + llmobs: { + mlApp: 'test' + } + }) + }) + + afterEach(() => { + nock.cleanAll() + LLMObsAgentProxySpanWriter.prototype.append.reset() + }) + + after(() => { + sinon.restore() + require('../../../../../dd-trace').llmobs.disable() // unsubscribe from all events + // delete require.cache[require.resolve('../../../../dd-trace')] + return agent.close({ ritmReset: false, wipe: true }) + }) + + withVersions('openai', 'openai', '>=4', version => { + const moduleRequirePath = `../../../../../../versions/openai@${version}` + + beforeEach(() => { + const requiredModule = require(moduleRequirePath) + const module = requiredModule.get() + + const OpenAI = module + + openai = new OpenAI({ + apiKey: 'test' + }) + }) + + it('submits a completion span', async () => { + nock('https://api.openai.com:443') + .post('/v1/completions') + .reply(200, { + model: 'text-davinci-002', + choices: [{ + text: 'I am doing well, how about you?', + index: 0, + logprobs: null, + finish_reason: 'length' + }], + usage: { prompt_tokens: 3, completion_tokens: 16, total_tokens: 19 } + }, []) + + const checkSpan = agent.use(traces => { + const span = traces[0][0] + const spanEvent = LLMObsAgentProxySpanWriter.prototype.append.getCall(0).args[0] + + const expected = expectedLLMObsLLMSpanEvent({ + span, + spanKind: 'llm', + name: 'openai.createCompletion', + inputMessages: [ + { content: 'How are you?' } + ], + outputMessages: [ + { content: 'I am doing well, how about you?' } + ], + tokenMetrics: { input_tokens: 3, output_tokens: 16, total_tokens: 19 }, + modelName: 'text-davinci-002', + modelProvider: 'openai', + metadata: {}, + tags: { ml_app: 'test', language: 'javascript' } + }) + + expect(spanEvent).to.deepEqualWithMockValues(expected) + }) + + await openai.completions.create({ + model: 'text-davinci-002', + prompt: 'How are you?' + }) + + await checkSpan + }) + + it('submits a chat completion span', async () => { + nock('https://api.openai.com:443') + .post('/v1/chat/completions') + .reply(200, { + id: 'chatcmpl-7GaWqyMTD9BLmkmy8SxyjUGX3KSRN', + object: 'chat.completion', + created: 1684188020, + model: 'gpt-3.5-turbo-0301', + usage: { + prompt_tokens: 37, + completion_tokens: 10, + total_tokens: 47 + }, + choices: [{ + message: { + role: 'assistant', + content: 'I am doing well, how about you?' + }, + finish_reason: 'length', + index: 0 + }] + }, []) + + const checkSpan = agent.use(traces => { + const span = traces[0][0] + const spanEvent = LLMObsAgentProxySpanWriter.prototype.append.getCall(0).args[0] + + const expected = expectedLLMObsLLMSpanEvent({ + span, + spanKind: 'llm', + name: 'openai.createChatCompletion', + inputMessages: [ + { role: 'system', content: 'You are a helpful assistant' }, + { role: 'user', content: 'How are you?' } + ], + outputMessages: [ + { role: 'assistant', content: 'I am doing well, how about you?' } + ], + tokenMetrics: { input_tokens: 37, output_tokens: 10, total_tokens: 47 }, + modelName: 'gpt-3.5-turbo-0301', + modelProvider: 'openai', + metadata: {}, + tags: { ml_app: 'test', language: 'javascript' } + }) + + expect(spanEvent).to.deepEqualWithMockValues(expected) + }) + + await openai.chat.completions.create({ + model: 'gpt-3.5-turbo-0301', + messages: [ + { role: 'system', content: 'You are a helpful assistant' }, + { role: 'user', content: 'How are you?' } + ] + }) + + await checkSpan + }) + + it('submits an embedding span', async () => { + nock('https://api.openai.com:443') + .post('/v1/embeddings') + .reply(200, { + object: 'list', + data: [{ + object: 'embedding', + index: 0, + embedding: [-0.0034387498, -0.026400521] + }], + model: 'text-embedding-ada-002-v2', + usage: { + prompt_tokens: 2, + total_tokens: 2 + } + }, []) + + const checkSpan = agent.use(traces => { + const span = traces[0][0] + const spanEvent = LLMObsAgentProxySpanWriter.prototype.append.getCall(0).args[0] + + const expected = expectedLLMObsLLMSpanEvent({ + span, + spanKind: 'embedding', + name: 'openai.createEmbedding', + inputDocuments: [ + { text: 'Hello, world!' } + ], + outputValue: '[1 embedding(s) returned with size 2]', + tokenMetrics: { input_tokens: 2, total_tokens: 2 }, + modelName: 'text-embedding-ada-002-v2', + modelProvider: 'openai', + metadata: { encoding_format: 'float' }, + tags: { ml_app: 'test', language: 'javascript' } + }) + + expect(spanEvent).to.deepEqualWithMockValues(expected) + }) + + await openai.embeddings.create({ + model: 'text-embedding-ada-002-v2', + input: 'Hello, world!' + }) + + await checkSpan + }) + + if (satisfiesTools(version)) { + it('submits a chat completion span with tools', async () => { + nock('https://api.openai.com:443') + .post('/v1/chat/completions') + .reply(200, { + id: 'chatcmpl-7GaWqyMTD9BLmkmy8SxyjUGX3KSRN', + object: 'chat.completion', + created: 1684188020, + model: 'gpt-3.5-turbo-0301', + usage: { + prompt_tokens: 37, + completion_tokens: 10, + total_tokens: 47 + }, + choices: [{ + message: { + role: 'assistant', + content: 'THOUGHT: I will use the "extract_fictional_info" tool', + tool_calls: [ + { + id: 'tool-1', + type: 'function', + function: { + name: 'extract_fictional_info', + arguments: '{"name":"SpongeBob","origin":"Bikini Bottom"}' + } + } + ] + }, + finish_reason: 'tool_calls', + index: 0 + }] + }, []) + + const checkSpan = agent.use(traces => { + const span = traces[0][0] + const spanEvent = LLMObsAgentProxySpanWriter.prototype.append.getCall(0).args[0] + + const expected = expectedLLMObsLLMSpanEvent({ + span, + spanKind: 'llm', + name: 'openai.createChatCompletion', + modelName: 'gpt-3.5-turbo-0301', + modelProvider: 'openai', + inputMessages: [{ role: 'user', content: 'What is SpongeBob SquarePants\'s origin?' }], + outputMessages: [{ + role: 'assistant', + content: 'THOUGHT: I will use the "extract_fictional_info" tool', + tool_calls: [ + { + name: 'extract_fictional_info', + arguments: { + name: 'SpongeBob', + origin: 'Bikini Bottom' + }, + tool_id: 'tool-1', + type: 'function' + } + ] + }], + metadata: { tool_choice: 'auto' }, + tags: { ml_app: 'test', language: 'javascript' }, + tokenMetrics: { input_tokens: 37, output_tokens: 10, total_tokens: 47 } + }) + + expect(spanEvent).to.deepEqualWithMockValues(expected) + }) + + await openai.chat.completions.create({ + model: 'gpt-3.5-turbo-0301', + messages: [{ role: 'user', content: 'What is SpongeBob SquarePants\'s origin?' }], + tools: [{ type: 'function', functiin: { /* this doesn't matter */} }], + tool_choice: 'auto' + }) + + await checkSpan + }) + } + + if (satisfiesStream(version)) { + it('submits a streamed completion span', async () => { + nock('https://api.openai.com:443') + .post('/v1/completions') + .reply(200, function () { + return fs.createReadStream(Path.join( + __dirname, baseOpenAITestsPath, 'streamed-responses/completions.simple.txt' + )) + }, { + 'Content-Type': 'text/plain', + 'openai-organization': 'kill-9' + }) + + const checkSpan = agent.use(traces => { + const span = traces[0][0] + const spanEvent = LLMObsAgentProxySpanWriter.prototype.append.getCall(0).args[0] + + const expected = expectedLLMObsLLMSpanEvent({ + span, + spanKind: 'llm', + name: 'openai.createCompletion', + inputMessages: [ + { content: 'Can you say this is a test?' } + ], + outputMessages: [ + { content: ' this is a test.' } + ], + tokenMetrics: { input_tokens: 8, output_tokens: 5, total_tokens: 13 }, + modelName: 'text-davinci-002', + modelProvider: 'openai', + metadata: { temperature: 0.5, stream: true }, + tags: { ml_app: 'test', language: 'javascript' } + }) + + expect(spanEvent).to.deepEqualWithMockValues(expected) + }) + + const stream = await openai.completions.create({ + model: 'text-davinci-002', + prompt: 'Can you say this is a test?', + temperature: 0.5, + stream: true + }) + + for await (const part of stream) { + expect(part).to.have.property('choices') + expect(part.choices[0]).to.have.property('text') + } + + await checkSpan + }) + + it('submits a streamed chat completion span', async () => { + nock('https://api.openai.com:443') + .post('/v1/chat/completions') + .reply(200, function () { + return fs.createReadStream(Path.join( + __dirname, baseOpenAITestsPath, 'streamed-responses/chat.completions.simple.txt' + )) + }, { + 'Content-Type': 'text/plain', + 'openai-organization': 'kill-9' + }) + + const checkSpan = agent.use(traces => { + const span = traces[0][0] + const spanEvent = LLMObsAgentProxySpanWriter.prototype.append.getCall(0).args[0] + + const expected = expectedLLMObsLLMSpanEvent({ + span, + spanKind: 'llm', + name: 'openai.createChatCompletion', + inputMessages: [ + { role: 'user', content: 'Hello' } + ], + outputMessages: [ + { role: 'assistant', content: 'Hello! How can I assist you today?' } + ], + tokenMetrics: { input_tokens: 1, output_tokens: 9, total_tokens: 10 }, + modelName: 'gpt-3.5-turbo-0301', + modelProvider: 'openai', + metadata: { stream: true }, + tags: { ml_app: 'test', language: 'javascript' } + }) + + expect(spanEvent).to.deepEqualWithMockValues(expected) + }) + + const stream = await openai.chat.completions.create({ + model: 'gpt-3.5-turbo-0301', + messages: [{ role: 'user', content: 'Hello' }], + stream: true + }) + + for await (const part of stream) { + expect(part).to.have.property('choices') + expect(part.choices[0]).to.have.property('delta') + } + + await checkSpan + }) + + if (satisfiesTools(version)) { + it('submits a chat completion span with tools stream', async () => { + nock('https://api.openai.com:443') + .post('/v1/chat/completions') + .reply(200, function () { + return fs.createReadStream(Path.join( + __dirname, baseOpenAITestsPath, 'streamed-responses/chat.completions.tool.and.content.txt' + )) + }, { + 'Content-Type': 'text/plain', + 'openai-organization': 'kill-9' + }) + + const checkSpan = agent.use(traces => { + const span = traces[0][0] + const spanEvent = LLMObsAgentProxySpanWriter.prototype.append.getCall(0).args[0] + + const expected = expectedLLMObsLLMSpanEvent({ + span, + spanKind: 'llm', + name: 'openai.createChatCompletion', + modelName: 'gpt-3.5-turbo-0301', + modelProvider: 'openai', + inputMessages: [{ role: 'user', content: 'What function would you call to finish this?' }], + outputMessages: [{ + role: 'assistant', + content: 'THOUGHT: Hi', + tool_calls: [ + { + name: 'finish', + arguments: { answer: '5' }, + type: 'function', + tool_id: 'call_Tg0o5wgoNSKF2iggAPmfWwem' + } + ] + }], + metadata: { tool_choice: 'auto', stream: true }, + tags: { ml_app: 'test', language: 'javascript' }, + tokenMetrics: { input_tokens: 9, output_tokens: 5, total_tokens: 14 } + }) + + expect(spanEvent).to.deepEqualWithMockValues(expected) + }) + + const stream = await openai.chat.completions.create({ + model: 'gpt-3.5-turbo-0301', + messages: [{ role: 'user', content: 'What function would you call to finish this?' }], + tools: [{ type: 'function', function: { /* this doesn't matter */ } }], + tool_choice: 'auto', + stream: true + }) + + for await (const part of stream) { + expect(part).to.have.property('choices') + expect(part.choices[0]).to.have.property('delta') + } + + await checkSpan + }) + } + } + + it('submits a completion span with an error', async () => { + nock('https://api.openai.com:443') + .post('/v1/completions') + .reply(400, {}) + + let error + const checkSpan = agent.use(traces => { + const span = traces[0][0] + const spanEvent = LLMObsAgentProxySpanWriter.prototype.append.getCall(0).args[0] + + const expected = expectedLLMObsLLMSpanEvent({ + span, + spanKind: 'llm', + name: 'openai.createCompletion', + inputMessages: [{ content: 'Hello' }], + outputMessages: [{ content: '' }], + modelName: 'gpt-3.5-turbo', + modelProvider: 'openai', + metadata: { max_tokens: 50 }, + tags: { ml_app: 'test', language: 'javascript' }, + error, + errorType: error.type || error.name, + errorMessage: error.message, + errorStack: error.stack + }) + + expect(spanEvent).to.deepEqualWithMockValues(expected) + }) + + try { + await openai.completions.create({ + model: 'gpt-3.5-turbo', + prompt: 'Hello', + max_tokens: 50 + }) + } catch (e) { + error = e + } + + await checkSpan + }) + + it('submits a chat completion span with an error', async () => { + nock('https://api.openai.com:443') + .post('/v1/chat/completions') + .reply(400, {}) + + let error + const checkSpan = agent.use(traces => { + const span = traces[0][0] + const spanEvent = LLMObsAgentProxySpanWriter.prototype.append.getCall(0).args[0] + + const expected = expectedLLMObsLLMSpanEvent({ + span, + spanKind: 'llm', + name: 'openai.createChatCompletion', + inputMessages: [{ role: 'user', content: 'Hello' }], + outputMessages: [{ content: '' }], + modelName: 'gpt-3.5-turbo', + modelProvider: 'openai', + metadata: { max_tokens: 50 }, + tags: { ml_app: 'test', language: 'javascript' }, + error, + errorType: error.type || error.name, + errorMessage: error.message, + errorStack: error.stack + }) + + expect(spanEvent).to.deepEqualWithMockValues(expected) + }) + + try { + await openai.chat.completions.create({ + model: 'gpt-3.5-turbo', + messages: [{ role: 'user', content: 'Hello' }], + max_tokens: 50 + }) + } catch (e) { + error = e + } + + await checkSpan + }) + }) + }) +}) diff --git a/packages/dd-trace/test/llmobs/sdk/index.spec.js b/packages/dd-trace/test/llmobs/sdk/index.spec.js new file mode 100644 index 00000000000..90415f9bd0b --- /dev/null +++ b/packages/dd-trace/test/llmobs/sdk/index.spec.js @@ -0,0 +1,1027 @@ +'use strict' + +const { expect } = require('chai') +const Config = require('../../../src/config') + +const LLMObsTagger = require('../../../src/llmobs/tagger') +const LLMObsEvalMetricsWriter = require('../../../src/llmobs/writers/evaluations') +const LLMObsAgentProxySpanWriter = require('../../../src/llmobs/writers/spans/agentProxy') +const LLMObsSpanProcessor = require('../../../src/llmobs/span_processor') + +const tracerVersion = require('../../../../../package.json').version + +const { channel } = require('dc-polyfill') +const injectCh = channel('dd-trace:span:inject') + +describe('sdk', () => { + let LLMObsSDK + let llmobs + let tracer + + before(() => { + tracer = require('../../../../dd-trace') + tracer.init({ + service: 'service', + llmobs: { + mlApp: 'mlApp' + } + }) + llmobs = tracer.llmobs + + // spy on properties + sinon.spy(LLMObsSpanProcessor.prototype, 'process') + sinon.spy(LLMObsSpanProcessor.prototype, 'format') + sinon.spy(tracer._tracer._processor, 'process') + + // stub writer functionality + sinon.stub(LLMObsEvalMetricsWriter.prototype, 'append') + sinon.stub(LLMObsEvalMetricsWriter.prototype, 'flush') + sinon.stub(LLMObsAgentProxySpanWriter.prototype, 'append') + sinon.stub(LLMObsAgentProxySpanWriter.prototype, 'flush') + + LLMObsSDK = require('../../../src/llmobs/sdk') + + // remove max listener warnings, we don't care about the writer anyways + process.removeAllListeners('beforeExit') + }) + + afterEach(() => { + LLMObsSpanProcessor.prototype.process.resetHistory() + LLMObsSpanProcessor.prototype.format.resetHistory() + tracer._tracer._processor.process.resetHistory() + + LLMObsEvalMetricsWriter.prototype.append.resetHistory() + LLMObsEvalMetricsWriter.prototype.flush.resetHistory() + + LLMObsAgentProxySpanWriter.prototype.append.resetHistory() + LLMObsAgentProxySpanWriter.prototype.flush.resetHistory() + + process.removeAllListeners('beforeExit') + }) + + after(() => { + sinon.restore() + llmobs.disable() + }) + + describe('enabled', () => { + for (const [value, label] of [ + [true, 'enabled'], + [false, 'disabled'] + ]) { + it(`returns ${value} when llmobs is ${label}`, () => { + const enabledOrDisabledLLMObs = new LLMObsSDK(null, { disable () {} }, { llmobs: { enabled: value } }) + + expect(enabledOrDisabledLLMObs.enabled).to.equal(value) + enabledOrDisabledLLMObs.disable() // unsubscribe + }) + } + }) + + describe('enable', () => { + it('enables llmobs if it is disabled', () => { + const config = new Config({}) + const llmobsModule = { + enable: sinon.stub(), + disable () {} + } + + // do not fully enable a disabled llmobs + const disabledLLMObs = new LLMObsSDK(tracer._tracer, llmobsModule, config) + + disabledLLMObs.enable({ + mlApp: 'mlApp' + }) + + expect(disabledLLMObs.enabled).to.be.true + expect(disabledLLMObs._config.llmobs.mlApp).to.equal('mlApp') + expect(disabledLLMObs._config.llmobs.agentlessEnabled).to.be.false + + expect(llmobsModule.enable).to.have.been.called + + disabledLLMObs.disable() // unsubscribe + }) + + it('does not enable llmobs if it is already enabled', () => { + sinon.spy(llmobs._llmobsModule, 'enable') + llmobs.enable({}) + + expect(llmobs.enabled).to.be.true + expect(llmobs._llmobsModule.enable).to.not.have.been.called + llmobs._llmobsModule.enable.restore() + }) + + it('does not enable llmobs if env var conflicts', () => { + const config = new Config({}) + const llmobsModule = { + enable: sinon.stub() + } + + // do not fully enable a disabled llmobs + const disabledLLMObs = new LLMObsSDK(tracer._tracer, llmobsModule, config) + process.env.DD_LLMOBS_ENABLED = 'false' + + disabledLLMObs.enable({}) + + expect(disabledLLMObs.enabled).to.be.false + delete process.env.DD_LLMOBS_ENABLED + disabledLLMObs.disable() // unsubscribe + }) + }) + + describe('disable', () => { + it('disables llmobs if it is enabled', () => { + const llmobsModule = { + disable: sinon.stub() + } + + const config = new Config({ + llmobs: {} + }) + + const enabledLLMObs = new LLMObsSDK(tracer._tracer, llmobsModule, config) + + expect(enabledLLMObs.enabled).to.be.true + enabledLLMObs.disable() + + expect(enabledLLMObs.enabled).to.be.false + expect(llmobsModule.disable).to.have.been.called + }) + + it('does not disable llmobs if it is already disabled', () => { + // do not fully enable a disabled llmobs + const disabledLLMObs = new LLMObsSDK(null, { disable () {} }, { llmobs: { enabled: false } }) + sinon.spy(disabledLLMObs._llmobsModule, 'disable') + + disabledLLMObs.disable() + + expect(disabledLLMObs.enabled).to.be.false + expect(disabledLLMObs._llmobsModule.disable).to.not.have.been.called + }) + }) + + describe('tracing', () => { + describe('trace', () => { + describe('tracing behavior', () => { + it('starts a span if llmobs is disabled but does not process it in the LLMObs span processor', () => { + tracer._tracer._config.llmobs.enabled = false + + llmobs.trace({ kind: 'workflow', name: 'myWorkflow' }, (span, cb) => { + expect(LLMObsTagger.tagMap.get(span)).to.not.exist + expect(() => span.setTag('k', 'v')).to.not.throw() + expect(() => cb()).to.not.throw() + }) + + expect(llmobs._tracer._processor.process).to.have.been.called + expect(LLMObsSpanProcessor.prototype.format).to.not.have.been.called + + tracer._tracer._config.llmobs.enabled = true + }) + + it('throws if the kind is invalid', () => { + expect(() => llmobs.trace({ kind: 'invalid' }, () => {})).to.throw() + + expect(llmobs._tracer._processor.process).to.not.have.been.called + expect(LLMObsSpanProcessor.prototype.format).to.not.have.been.called + }) + + // TODO: need span kind optional for this + it.skip('throws if no name is provided', () => { + expect(() => llmobs.trace({ kind: 'workflow' }, () => {})).to.throw() + + expect(llmobs._tracer._processor.process).to.not.have.been.called + expect(LLMObsSpanProcessor.prototype.format).to.not.have.been.called + }) + + it('traces a block', () => { + let span + + llmobs.trace({ kind: 'workflow' }, _span => { + span = _span + sinon.spy(span, 'finish') + }) + + expect(span.finish).to.have.been.called + }) + + it('traces a block with a callback', () => { + let span + let done + + llmobs.trace({ kind: 'workflow' }, (_span, _done) => { + span = _span + sinon.spy(span, 'finish') + done = _done + }) + + expect(span.finish).to.not.have.been.called + + done() + + expect(span.finish).to.have.been.called + }) + + it('traces a promise', done => { + const deferred = {} + const promise = new Promise(resolve => { + deferred.resolve = resolve + }) + + let span + + llmobs + .trace({ kind: 'workflow' }, _span => { + span = _span + sinon.spy(span, 'finish') + return promise + }) + .then(() => { + expect(span.finish).to.have.been.called + done() + }) + .catch(done) + + expect(span.finish).to.not.have.been.called + + deferred.resolve() + }) + }) + + describe('parentage', () => { + // TODO: need to implement custom trace IDs + it.skip('starts a span with a distinct trace id', () => { + llmobs.trace({ kind: 'workflow', name: 'test' }, span => { + expect(LLMObsTagger.tagMap.get(span)['_ml_obs.trace_id']) + .to.exist.and.to.not.equal(span.context().toTraceId(true)) + }) + }) + + it('sets span parentage correctly', () => { + llmobs.trace({ kind: 'workflow', name: 'test' }, outerLLMSpan => { + llmobs.trace({ kind: 'task', name: 'test' }, innerLLMSpan => { + expect(LLMObsTagger.tagMap.get(innerLLMSpan)['_ml_obs.llmobs_parent_id']) + .to.equal(outerLLMSpan.context().toSpanId()) + // TODO: need to implement custom trace IDs + // expect(innerLLMSpan.context()._tags['_ml_obs.trace_id']) + // .to.equal(outerLLMSpan.context()._tags['_ml_obs.trace_id']) + }) + }) + }) + + it('maintains llmobs parentage separately from apm spans', () => { + llmobs.trace({ kind: 'workflow', name: 'outer-llm' }, outerLLMSpan => { + expect(llmobs._active()).to.equal(outerLLMSpan) + tracer.trace('apmSpan', apmSpan => { + expect(llmobs._active()).to.equal(outerLLMSpan) + llmobs.trace({ kind: 'workflow', name: 'inner-llm' }, innerLLMSpan => { + expect(llmobs._active()).to.equal(innerLLMSpan) + + // llmobs span linkage + expect(LLMObsTagger.tagMap.get(innerLLMSpan)['_ml_obs.llmobs_parent_id']) + .to.equal(outerLLMSpan.context().toSpanId()) + + // apm span linkage + expect(innerLLMSpan.context()._parentId.toString(10)).to.equal(apmSpan.context().toSpanId()) + expect(apmSpan.context()._parentId.toString(10)).to.equal(outerLLMSpan.context().toSpanId()) + }) + }) + }) + }) + + // TODO: need to implement custom trace IDs + it.skip('starts different traces for llmobs spans as child spans of an apm root span', () => { + let apmTraceId, traceId1, traceId2 + tracer.trace('apmRootSpan', apmRootSpan => { + apmTraceId = apmRootSpan.context().toTraceId(true) + llmobs.trace('workflow', llmobsSpan1 => { + traceId1 = llmobsSpan1.context()._tags['_ml_obs.trace_id'] + }) + + llmobs.trace('workflow', llmobsSpan2 => { + traceId2 = llmobsSpan2.context()._tags['_ml_obs.trace_id'] + }) + }) + + expect(traceId1).to.not.equal(traceId2) + expect(traceId1).to.not.equal(apmTraceId) + expect(traceId2).to.not.equal(apmTraceId) + }) + + it('maintains the llmobs parentage when error callbacks are used', () => { + llmobs.trace({ kind: 'workflow' }, outer => { + llmobs.trace({ kind: 'task' }, (inner, cb) => { + expect(llmobs._active()).to.equal(inner) + expect(LLMObsTagger.tagMap.get(inner)['_ml_obs.llmobs_parent_id']).to.equal(outer.context().toSpanId()) + cb() // finish the span + }) + + expect(llmobs._active()).to.equal(outer) + + llmobs.trace({ kind: 'task' }, (inner) => { + expect(llmobs._active()).to.equal(inner) + expect(LLMObsTagger.tagMap.get(inner)['_ml_obs.llmobs_parent_id']).to.equal(outer.context().toSpanId()) + }) + }) + }) + }) + }) + + describe('wrap', () => { + describe('tracing behavior', () => { + it('starts a span if llmobs is disabled but does not process it in the LLMObs span processor', () => { + tracer._tracer._config.llmobs.enabled = false + + const fn = llmobs.wrap({ kind: 'workflow' }, (a) => { + expect(a).to.equal(1) + expect(LLMObsTagger.tagMap.get(llmobs._active())).to.not.exist + }) + + expect(() => fn(1)).to.not.throw() + + expect(llmobs._tracer._processor.process).to.have.been.called + expect(LLMObsSpanProcessor.prototype.format).to.not.have.been.called + + tracer._tracer._config.llmobs.enabled = true + }) + + it('throws if the kind is invalid', () => { + expect(() => llmobs.wrap({ kind: 'invalid' }, () => {})).to.throw() + }) + + it('wraps a function', () => { + let span + const fn = llmobs.wrap({ kind: 'workflow' }, () => { + span = tracer.scope().active() + sinon.spy(span, 'finish') + }) + + fn() + + expect(span.finish).to.have.been.called + }) + + it('wraps a function with a callback', () => { + let span + let next + + const fn = llmobs.wrap({ kind: 'workflow' }, (_next) => { + span = tracer.scope().active() + sinon.spy(span, 'finish') + next = _next + }) + + fn(() => {}) + + expect(span.finish).to.not.have.been.called + + next() + + expect(span.finish).to.have.been.called + }) + + it('does not auto-annotate llm spans', () => { + let span + function myLLM (input) { + span = llmobs._active() + return '' + } + + const wrappedMyLLM = llmobs.wrap({ kind: 'llm' }, myLLM) + + wrappedMyLLM('input') + + expect(LLMObsTagger.tagMap.get(span)).to.deep.equal({ + '_ml_obs.meta.span.kind': 'llm', + '_ml_obs.meta.ml_app': 'mlApp', + '_ml_obs.llmobs_parent_id': 'undefined' + }) + }) + + it('does not auto-annotate embedding spans input', () => { + let span + function myEmbedding (input) { + span = llmobs._active() + return 'output' + } + + const wrappedMyEmbedding = llmobs.wrap({ kind: 'embedding' }, myEmbedding) + + wrappedMyEmbedding('input') + + expect(LLMObsTagger.tagMap.get(span)).to.deep.equal({ + '_ml_obs.meta.span.kind': 'embedding', + '_ml_obs.meta.ml_app': 'mlApp', + '_ml_obs.llmobs_parent_id': 'undefined', + '_ml_obs.meta.output.value': 'output' + }) + }) + + it('does not auto-annotate retrieval spans output', () => { + let span + function myRetrieval (input) { + span = llmobs._active() + return 'output' + } + + const wrappedMyRetrieval = llmobs.wrap({ kind: 'retrieval' }, myRetrieval) + + wrappedMyRetrieval('input') + + expect(LLMObsTagger.tagMap.get(span)).to.deep.equal({ + '_ml_obs.meta.span.kind': 'retrieval', + '_ml_obs.meta.ml_app': 'mlApp', + '_ml_obs.llmobs_parent_id': 'undefined', + '_ml_obs.meta.input.value': 'input' + }) + }) + + // TODO: need span kind optional for this test + it.skip('sets the span name to "unnamed-anonymous-function" if no name is provided', () => { + let span + const fn = llmobs.wrap({ kind: 'workflow' }, () => { + span = llmobs._active() + }) + + fn() + + expect(span.context()._name).to.equal('unnamed-anonymous-function') + }) + }) + + describe('parentage', () => { + // TODO: need to implement custom trace IDs + it.skip('starts a span with a distinct trace id', () => { + const fn = llmobs.wrap('workflow', { name: 'test' }, () => { + const span = llmobs._active() + expect(span.context()._tags['_ml_obs.trace_id']) + .to.exist.and.to.not.equal(span.context().toTraceId(true)) + }) + + fn() + }) + + it('sets span parentage correctly', () => { + let outerLLMSpan, innerLLMSpan + + function outer () { + outerLLMSpan = llmobs._active() + innerWrapped() + } + + function inner () { + innerLLMSpan = llmobs._active() + expect(LLMObsTagger.tagMap.get(innerLLMSpan)['_ml_obs.llmobs_parent_id']) + .to.equal(outerLLMSpan.context().toSpanId()) + // TODO: need to implement custom trace IDs + // expect(innerLLMSpan.context()._tags['_ml_obs.trace_id']) + // .to.equal(outerLLMSpan.context()._tags['_ml_obs.trace_id']) + } + + const outerWrapped = llmobs.wrap({ kind: 'workflow' }, outer) + const innerWrapped = llmobs.wrap({ kind: 'task' }, inner) + + outerWrapped() + }) + + it('maintains llmobs parentage separately from apm spans', () => { + let outerLLMObsSpan, innerLLMObsSpan + + function outerLLMObs () { + outerLLMObsSpan = llmobs._active() + expect(outerLLMObsSpan).to.equal(tracer.scope().active()) + + apmWrapped() + } + function apm () { + expect(llmobs._active()).to.equal(outerLLMObsSpan) + innerWrapped() + } + function innerLLMObs () { + innerLLMObsSpan = llmobs._active() + expect(innerLLMObsSpan).to.equal(tracer.scope().active()) + expect(LLMObsTagger.tagMap.get(innerLLMObsSpan)['_ml_obs.llmobs_parent_id']) + .to.equal(outerLLMObsSpan.context().toSpanId()) + // TODO: need to implement custom trace IDs + // expect(innerLLMObsSpan.context()._tags['_ml_obs.trace_id']) + // .to.equal(outerLLMObsSpan.context()._tags['_ml_obs.trace_id']) + } + + const outerWrapped = llmobs.wrap({ kind: 'workflow' }, outerLLMObs) + const apmWrapped = tracer.wrap('workflow', apm) + const innerWrapped = llmobs.wrap({ kind: 'workflow' }, innerLLMObs) + + outerWrapped() + }) + + // TODO: need to implement custom trace IDs + it.skip('starts different traces for llmobs spans as child spans of an apm root span', () => { + let traceId1, traceId2, apmTraceId + function apm () { + apmTraceId = tracer.scope().active().context().toTraceId(true) + llmObsWrapped1() + llmObsWrapped2() + } + function llmObs1 () { + traceId1 = LLMObsTagger.tagMap.get(llmobs._active())['_ml_obs.trace_id'] + } + function llmObs2 () { + traceId2 = LLMObsTagger.tagMap.get(llmobs._active())['_ml_obs.trace_id'] + } + + const apmWrapped = tracer.wrap('workflow', apm) + const llmObsWrapped1 = llmobs.wrap({ kind: 'workflow' }, llmObs1) + const llmObsWrapped2 = llmobs.wrap({ kind: 'workflow' }, llmObs2) + + apmWrapped() + + expect(traceId1).to.not.equal(traceId2) + expect(traceId1).to.not.equal(apmTraceId) + expect(traceId2).to.not.equal(apmTraceId) + }) + + it('maintains the llmobs parentage when callbacks are used', () => { + let outerSpan + function outer () { + outerSpan = llmobs._active() + wrappedInner1(() => {}) + expect(outerSpan).to.equal(tracer.scope().active()) + wrappedInner2() + } + + function inner1 (cb) { + const inner = tracer.scope().active() + expect(llmobs._active()).to.equal(inner) + expect(LLMObsTagger.tagMap.get(inner)['_ml_obs.llmobs_parent_id']).to.equal(outerSpan.context().toSpanId()) + cb() + } + + function inner2 () { + const inner = tracer.scope().active() + expect(llmobs._active()).to.equal(inner) + expect(LLMObsTagger.tagMap.get(inner)['_ml_obs.llmobs_parent_id']).to.equal(outerSpan.context().toSpanId()) + } + + const wrappedOuter = llmobs.wrap({ kind: 'workflow' }, outer) + const wrappedInner1 = llmobs.wrap({ kind: 'task' }, inner1) + const wrappedInner2 = llmobs.wrap({ kind: 'task' }, inner2) + + wrappedOuter() + }) + }) + }) + }) + + describe('annotate', () => { + it('returns if llmobs is disabled', () => { + tracer._tracer._config.llmobs.enabled = false + sinon.spy(llmobs, '_active') + llmobs.annotate() + + expect(llmobs._active).to.not.have.been.called + llmobs._active.restore() + + tracer._tracer._config.llmobs.enabled = true + }) + + it('throws if no arguments are provided', () => { + expect(() => llmobs.annotate()).to.throw() + }) + + it('throws if there are no options given', () => { + llmobs.trace({ kind: 'llm', name: 'test' }, span => { + expect(() => llmobs.annotate(span)).to.throw() + + // span should still exist in the registry, just with no annotations + expect(LLMObsTagger.tagMap.get(span)).to.deep.equal({ + '_ml_obs.meta.span.kind': 'llm', + '_ml_obs.meta.ml_app': 'mlApp', + '_ml_obs.llmobs_parent_id': 'undefined' + }) + }) + }) + + it('throws if the provided span is not an LLMObs span', () => { + tracer.trace('test', span => { + expect(() => llmobs.annotate(span, {})).to.throw() + + // no span in registry, should not throw + expect(LLMObsTagger.tagMap.get(span)).to.not.exist + }) + }) + + it('throws if the span is finished', () => { + sinon.spy(llmobs._tagger, 'tagTextIO') + llmobs.trace({ kind: 'workflow', name: 'outer' }, () => { + let innerLLMSpan + llmobs.trace({ kind: 'task', name: 'inner' }, _span => { + innerLLMSpan = _span + }) + + expect(() => llmobs.annotate(innerLLMSpan, {})).to.throw() + expect(llmobs._tagger.tagTextIO).to.not.have.been.called + }) + llmobs._tagger.tagTextIO.restore() + }) + + it('throws for an llmobs span with an invalid kind', () => { + // TODO this might end up being obsolete with llmobs span kind as optional + sinon.spy(llmobs._tagger, 'tagLLMIO') + llmobs.trace({ kind: 'llm', name: 'test' }, span => { + LLMObsTagger.tagMap.get(span)['_ml_obs.meta.span.kind'] = undefined // somehow this is set + expect(() => llmobs.annotate(span, {})).to.throw() + }) + + expect(llmobs._tagger.tagLLMIO).to.not.have.been.called + llmobs._tagger.tagLLMIO.restore() + }) + + it('annotates the current active llmobs span in an llmobs scope', () => { + sinon.spy(llmobs._tagger, 'tagTextIO') + + llmobs.trace({ kind: 'workflow', name: 'test' }, span => { + const inputData = {} + llmobs.annotate({ inputData }) + + expect(llmobs._tagger.tagTextIO).to.have.been.calledWith(span, inputData, undefined) + }) + + llmobs._tagger.tagTextIO.restore() + }) + + it('annotates the current active llmobs span in an apm scope', () => { + sinon.spy(llmobs._tagger, 'tagTextIO') + + llmobs.trace({ kind: 'workflow', name: 'test' }, llmobsSpan => { + tracer.trace('apmSpan', () => { + const inputData = {} + llmobs.annotate({ inputData }) + + expect(llmobs._tagger.tagTextIO).to.have.been.calledWith(llmobsSpan, inputData, undefined) + }) + }) + + llmobs._tagger.tagTextIO.restore() + }) + + it('annotates llm io for an llm span', () => { + const inputData = [{ role: 'system', content: 'system prompt' }] + const outputData = [{ role: 'ai', content: 'no question was asked' }] + + llmobs.trace({ kind: 'llm', name: 'test' }, span => { + llmobs.annotate({ inputData, outputData }) + + expect(LLMObsTagger.tagMap.get(span)).to.deep.equal({ + '_ml_obs.meta.span.kind': 'llm', + '_ml_obs.meta.ml_app': 'mlApp', + '_ml_obs.llmobs_parent_id': 'undefined', + '_ml_obs.meta.input.messages': inputData, + '_ml_obs.meta.output.messages': outputData + }) + }) + }) + + it('annotates embedding io for an embedding span', () => { + const inputData = [{ text: 'input text' }] + const outputData = 'documents embedded' + + llmobs.trace({ kind: 'embedding', name: 'test' }, span => { + llmobs.annotate({ inputData, outputData }) + + expect(LLMObsTagger.tagMap.get(span)).to.deep.equal({ + '_ml_obs.meta.span.kind': 'embedding', + '_ml_obs.meta.ml_app': 'mlApp', + '_ml_obs.llmobs_parent_id': 'undefined', + '_ml_obs.meta.input.documents': inputData, + '_ml_obs.meta.output.value': outputData + }) + }) + }) + + it('annotates retrieval io for a retrieval span', () => { + const inputData = 'input text' + const outputData = [{ text: 'output text' }] + + llmobs.trace({ kind: 'retrieval', name: 'test' }, span => { + llmobs.annotate({ inputData, outputData }) + + expect(LLMObsTagger.tagMap.get(span)).to.deep.equal({ + '_ml_obs.meta.span.kind': 'retrieval', + '_ml_obs.meta.ml_app': 'mlApp', + '_ml_obs.llmobs_parent_id': 'undefined', + '_ml_obs.meta.input.value': inputData, + '_ml_obs.meta.output.documents': outputData + }) + }) + }) + + it('annotates metadata if present', () => { + const metadata = { response_type: 'json' } + + llmobs.trace({ kind: 'llm', name: 'test' }, span => { + llmobs.annotate({ metadata }) + + expect(LLMObsTagger.tagMap.get(span)).to.deep.equal({ + '_ml_obs.meta.span.kind': 'llm', + '_ml_obs.meta.ml_app': 'mlApp', + '_ml_obs.llmobs_parent_id': 'undefined', + '_ml_obs.meta.metadata': metadata + }) + }) + }) + + it('annotates metrics if present', () => { + const metrics = { score: 0.6 } + + llmobs.trace({ kind: 'llm', name: 'test' }, span => { + llmobs.annotate({ metrics }) + + expect(LLMObsTagger.tagMap.get(span)).to.deep.equal({ + '_ml_obs.meta.span.kind': 'llm', + '_ml_obs.meta.ml_app': 'mlApp', + '_ml_obs.llmobs_parent_id': 'undefined', + '_ml_obs.metrics': metrics + }) + }) + }) + + it('annotates tags if present', () => { + const tags = { 'custom.tag': 'value' } + + llmobs.trace({ kind: 'llm', name: 'test' }, span => { + llmobs.annotate({ tags }) + + expect(LLMObsTagger.tagMap.get(span)).to.deep.equal({ + '_ml_obs.meta.span.kind': 'llm', + '_ml_obs.meta.ml_app': 'mlApp', + '_ml_obs.llmobs_parent_id': 'undefined', + '_ml_obs.tags': tags + }) + }) + }) + }) + + describe('exportSpan', () => { + it('throws if no span is provided', () => { + expect(() => llmobs.exportSpan()).to.throw() + }) + + it('throws if the provided span is not an LLMObs span', () => { + tracer.trace('test', span => { + expect(() => llmobs.exportSpan(span)).to.throw() + }) + }) + + it('uses the provided span', () => { + llmobs.trace({ kind: 'workflow', name: 'test' }, span => { + const spanCtx = llmobs.exportSpan(span) + + const traceId = span.context().toTraceId(true) + const spanId = span.context().toSpanId() + + expect(spanCtx).to.deep.equal({ traceId, spanId }) + }) + }) + + it('uses the active span in an llmobs scope', () => { + llmobs.trace({ kind: 'workflow', name: 'test' }, span => { + const spanCtx = llmobs.exportSpan() + + const traceId = span.context().toTraceId(true) + const spanId = span.context().toSpanId() + + expect(spanCtx).to.deep.equal({ traceId, spanId }) + }) + }) + + it('uses the active span in an apm scope', () => { + llmobs.trace({ kind: 'workflow', name: 'test' }, llmobsSpan => { + tracer.trace('apmSpan', () => { + const spanCtx = llmobs.exportSpan() + + const traceId = llmobsSpan.context().toTraceId(true) + const spanId = llmobsSpan.context().toSpanId() + + expect(spanCtx).to.deep.equal({ traceId, spanId }) + }) + }) + }) + + it('returns undefined if the provided span is not a span', () => { + llmobs.trace({ kind: 'workflow', name: 'test' }, fakeSpan => { + fakeSpan.context().toTraceId = undefined // something that would throw + LLMObsTagger.tagMap.set(fakeSpan, {}) + const spanCtx = llmobs.exportSpan(fakeSpan) + + expect(spanCtx).to.be.undefined + }) + }) + }) + + describe('submitEvaluation', () => { + let spanCtx + let originalApiKey + + before(() => { + originalApiKey = tracer._tracer._config.apiKey + tracer._tracer._config.apiKey = 'test' + }) + + beforeEach(() => { + spanCtx = { + traceId: '1234', + spanId: '5678' + } + }) + + after(() => { + tracer._tracer._config.apiKey = originalApiKey + }) + + it('does not submit an evaluation if llmobs is disabled', () => { + tracer._tracer._config.llmobs.enabled = false + llmobs.submitEvaluation() + + expect(LLMObsEvalMetricsWriter.prototype.append).to.not.have.been.called + + tracer._tracer._config.llmobs.enabled = true + }) + + it('throws for a missing API key', () => { + const apiKey = tracer._tracer._config.apiKey + delete tracer._tracer._config.apiKey + + expect(() => llmobs.submitEvaluation(spanCtx)).to.throw() + expect(LLMObsEvalMetricsWriter.prototype.append).to.not.have.been.called + + tracer._tracer._config.apiKey = apiKey + }) + + it('throws for an invalid span context', () => { + const invalid = {} + + expect(() => llmobs.submitEvaluation(invalid, {})).to.throw() + expect(LLMObsEvalMetricsWriter.prototype.append).to.not.have.been.called + }) + + it('throws for a missing mlApp', () => { + const mlApp = tracer._tracer._config.llmobs.mlApp + delete tracer._tracer._config.llmobs.mlApp + + expect(() => llmobs.submitEvaluation(spanCtx)).to.throw() + expect(LLMObsEvalMetricsWriter.prototype.append).to.not.have.been.called + + tracer._tracer._config.llmobs.mlApp = mlApp + }) + + it('throws for an invalid timestamp', () => { + expect(() => { + llmobs.submitEvaluation(spanCtx, { + mlApp: 'test', + timestampMs: 'invalid' + }) + }).to.throw() + expect(LLMObsEvalMetricsWriter.prototype.append).to.not.have.been.called + }) + + it('throws for a missing label', () => { + expect(() => { + llmobs.submitEvaluation(spanCtx, { + mlApp: 'test', + timestampMs: 1234 + }) + }).to.throw() + expect(LLMObsEvalMetricsWriter.prototype.append).to.not.have.been.called + }) + + it('throws for an invalid metric type', () => { + expect(() => { + llmobs.submitEvaluation(spanCtx, { + mlApp: 'test', + timestampMs: 1234, + label: 'test', + metricType: 'invalid' + }) + }).to.throw() + expect(LLMObsEvalMetricsWriter.prototype.append).to.not.have.been.called + }) + + it('throws for a mismatched value for a categorical metric', () => { + expect(() => { + llmobs.submitEvaluation(spanCtx, { + mlApp: 'test', + timestampMs: 1234, + label: 'test', + metricType: 'categorical', + value: 1 + }) + }).to.throw() + expect(LLMObsEvalMetricsWriter.prototype.append).to.not.have.been.called + }) + + it('throws for a mismatched value for a score metric', () => { + expect(() => { + llmobs.submitEvaluation(spanCtx, { + mlApp: 'test', + timestampMs: 1234, + label: 'test', + metricType: 'score', + value: 'string' + }) + }).to.throw() + + expect(LLMObsEvalMetricsWriter.prototype.append).to.not.have.been.called + }) + + it('submits an evaluation metric', () => { + llmobs.submitEvaluation(spanCtx, { + mlApp: 'test', + timestampMs: 1234, + label: 'test', + metricType: 'score', + value: 0.6, + tags: { + host: 'localhost' + } + }) + + expect(LLMObsEvalMetricsWriter.prototype.append.getCall(0).args[0]).to.deep.equal({ + trace_id: spanCtx.traceId, + span_id: spanCtx.spanId, + ml_app: 'test', + timestamp_ms: 1234, + label: 'test', + metric_type: 'score', + score_value: 0.6, + tags: [`dd-trace.version:${tracerVersion}`, 'ml_app:test', 'host:localhost'] + }) + }) + + it('sets `categorical_value` for categorical metrics', () => { + llmobs.submitEvaluation(spanCtx, { + mlApp: 'test', + timestampMs: 1234, + label: 'test', + metricType: 'categorical', + value: 'foo', + tags: { + host: 'localhost' + } + }) + + expect(LLMObsEvalMetricsWriter.prototype.append.getCall(0).args[0]).to.have.property('categorical_value', 'foo') + }) + + it('defaults to the current time if no timestamp is provided', () => { + sinon.stub(Date, 'now').returns(1234) + llmobs.submitEvaluation(spanCtx, { + mlApp: 'test', + label: 'test', + metricType: 'score', + value: 0.6 + }) + + expect(LLMObsEvalMetricsWriter.prototype.append.getCall(0).args[0]).to.have.property('timestamp_ms', 1234) + Date.now.restore() + }) + }) + + describe('flush', () => { + it('does not flush if llmobs is disabled', () => { + tracer._tracer._config.llmobs.enabled = false + llmobs.flush() + + expect(LLMObsEvalMetricsWriter.prototype.flush).to.not.have.been.called + expect(LLMObsAgentProxySpanWriter.prototype.flush).to.not.have.been.called + tracer._tracer._config.llmobs.enabled = true + }) + + it('flushes the evaluation writer and span writer', () => { + llmobs.flush() + + expect(LLMObsEvalMetricsWriter.prototype.flush).to.have.been.called + expect(LLMObsAgentProxySpanWriter.prototype.flush).to.have.been.called + }) + + it('logs if there was an error flushing', () => { + LLMObsEvalMetricsWriter.prototype.flush.throws(new Error('boom')) + + expect(() => llmobs.flush()).to.not.throw() + }) + }) + + describe('distributed', () => { + it('adds the current llmobs span id to the injection context', () => { + const carrier = { 'x-datadog-tags': '' } + let parentId + llmobs.trace({ kind: 'workflow', name: 'myWorkflow' }, span => { + parentId = span.context().toSpanId() + + // simulate injection from http integration or from tracer + // something that triggers the text_map injection + injectCh.publish({ carrier }) + }) + + expect(carrier['x-datadog-tags']).to.equal(`,_dd.p.llmobs_parent_id=${parentId}`) + }) + }) +}) diff --git a/packages/dd-trace/test/llmobs/sdk/integration.spec.js b/packages/dd-trace/test/llmobs/sdk/integration.spec.js new file mode 100644 index 00000000000..acba94d8f71 --- /dev/null +++ b/packages/dd-trace/test/llmobs/sdk/integration.spec.js @@ -0,0 +1,256 @@ +'use strict' + +const { expectedLLMObsNonLLMSpanEvent, deepEqualWithMockValues } = require('../util') +const chai = require('chai') + +chai.Assertion.addMethod('deepEqualWithMockValues', deepEqualWithMockValues) + +const tags = { + ml_app: 'test', + language: 'javascript' +} + +const AgentProxyWriter = require('../../../src/llmobs/writers/spans/agentProxy') +const EvalMetricsWriter = require('../../../src/llmobs/writers/evaluations') + +const tracerVersion = require('../../../../../package.json').version + +describe('end to end sdk integration tests', () => { + let tracer + let llmobs + let payloadGenerator + + function run (payloadGenerator) { + payloadGenerator() + return { + spans: tracer._tracer._processor.process.args.map(args => args[0]).reverse(), // spans finish in reverse order + llmobsSpans: AgentProxyWriter.prototype.append.args?.map(args => args[0]), + evaluationMetrics: EvalMetricsWriter.prototype.append.args?.map(args => args[0]) + } + } + + function check (expected, actual) { + for (const expectedLLMObsSpanIdx in expected) { + const expectedLLMObsSpan = expected[expectedLLMObsSpanIdx] + const actualLLMObsSpan = actual[expectedLLMObsSpanIdx] + expect(actualLLMObsSpan).to.deep.deepEqualWithMockValues(expectedLLMObsSpan) + } + } + + before(() => { + tracer = require('../../../../dd-trace') + tracer.init({ + llmobs: { + mlApp: 'test' + } + }) + + // another test suite may have disabled LLMObs + // to clear the intervals and unsubscribe + // in that case, the `init` call above won't have re-enabled it + // we'll re-enable it here + llmobs = tracer.llmobs + if (!llmobs.enabled) { + llmobs.enable({ + mlApp: 'test' + }) + } + + tracer._tracer._config.apiKey = 'test' + + sinon.spy(tracer._tracer._processor, 'process') + sinon.stub(AgentProxyWriter.prototype, 'append') + sinon.stub(EvalMetricsWriter.prototype, 'append') + }) + + afterEach(() => { + tracer._tracer._processor.process.resetHistory() + AgentProxyWriter.prototype.append.resetHistory() + EvalMetricsWriter.prototype.append.resetHistory() + + process.removeAllListeners('beforeExit') + + llmobs.disable() + llmobs.enable({ mlApp: 'test', apiKey: 'test' }) + }) + + after(() => { + sinon.restore() + llmobs.disable() + delete global._ddtrace + delete require.cache[require.resolve('../../../../dd-trace')] + }) + + it('uses trace correctly', () => { + payloadGenerator = function () { + const result = llmobs.trace({ kind: 'agent' }, () => { + llmobs.annotate({ inputData: 'hello', outputData: 'world', metadata: { foo: 'bar' } }) + return tracer.trace('apmSpan', () => { + llmobs.annotate({ tags: { bar: 'baz' } }) // should use the current active llmobs span + return llmobs.trace({ kind: 'workflow', name: 'myWorkflow' }, () => { + llmobs.annotate({ inputData: 'world', outputData: 'hello' }) + return 'boom' + }) + }) + }) + + expect(result).to.equal('boom') + } + + const { spans, llmobsSpans } = run(payloadGenerator) + expect(spans).to.have.lengthOf(3) + expect(llmobsSpans).to.have.lengthOf(2) + + const expected = [ + expectedLLMObsNonLLMSpanEvent({ + span: spans[0], + spanKind: 'agent', + tags: { ...tags, bar: 'baz' }, + metadata: { foo: 'bar' }, + inputValue: 'hello', + outputValue: 'world' + }), + expectedLLMObsNonLLMSpanEvent({ + span: spans[2], + spanKind: 'workflow', + parentId: spans[0].context().toSpanId(), + tags, + name: 'myWorkflow', + inputValue: 'world', + outputValue: 'hello' + }) + ] + + check(expected, llmobsSpans) + }) + + it('uses wrap correctly', () => { + payloadGenerator = function () { + function agent (input) { + llmobs.annotate({ inputData: 'hello' }) + return apm(input) + } + // eslint-disable-next-line no-func-assign + agent = llmobs.wrap({ kind: 'agent' }, agent) + + function apm (input) { + llmobs.annotate({ metadata: { foo: 'bar' } }) // should annotate the agent span + return workflow(input) + } + // eslint-disable-next-line no-func-assign + apm = tracer.wrap('apm', apm) + + function workflow () { + llmobs.annotate({ outputData: 'custom' }) + return 'world' + } + // eslint-disable-next-line no-func-assign + workflow = llmobs.wrap({ kind: 'workflow', name: 'myWorkflow' }, workflow) + + agent('my custom input') + } + + const { spans, llmobsSpans } = run(payloadGenerator) + expect(spans).to.have.lengthOf(3) + expect(llmobsSpans).to.have.lengthOf(2) + + const expected = [ + expectedLLMObsNonLLMSpanEvent({ + span: spans[0], + spanKind: 'agent', + tags, + inputValue: 'hello', + outputValue: 'world', + metadata: { foo: 'bar' } + }), + expectedLLMObsNonLLMSpanEvent({ + span: spans[2], + spanKind: 'workflow', + parentId: spans[0].context().toSpanId(), + tags, + name: 'myWorkflow', + inputValue: 'my custom input', + outputValue: 'custom' + }) + ] + + check(expected, llmobsSpans) + }) + + it('instruments and uninstruments as needed', () => { + payloadGenerator = function () { + llmobs.disable() + llmobs.trace({ kind: 'agent', name: 'llmobsParent' }, () => { + llmobs.annotate({ inputData: 'hello', outputData: 'world' }) + llmobs.enable({ mlApp: 'test1' }) + llmobs.trace({ kind: 'workflow', name: 'child1' }, () => { + llmobs.disable() + llmobs.trace({ kind: 'workflow', name: 'child2' }, () => { + llmobs.enable({ mlApp: 'test2' }) + llmobs.trace({ kind: 'workflow', name: 'child3' }, () => {}) + }) + }) + }) + } + + const { spans, llmobsSpans } = run(payloadGenerator) + expect(spans).to.have.lengthOf(4) + expect(llmobsSpans).to.have.lengthOf(2) + + const expected = [ + expectedLLMObsNonLLMSpanEvent({ + span: spans[1], + spanKind: 'workflow', + tags: { ...tags, ml_app: 'test1' }, + name: 'child1' + }), + expectedLLMObsNonLLMSpanEvent({ + span: spans[3], + spanKind: 'workflow', + tags: { ...tags, ml_app: 'test2' }, + name: 'child3', + parentId: spans[1].context().toSpanId() + }) + ] + + check(expected, llmobsSpans) + }) + + it('submits evaluations', () => { + sinon.stub(Date, 'now').returns(1234567890) + payloadGenerator = function () { + llmobs.trace({ kind: 'agent', name: 'myAgent' }, () => { + llmobs.annotate({ inputData: 'hello', outputData: 'world' }) + const spanCtx = llmobs.exportSpan() + llmobs.submitEvaluation(spanCtx, { + label: 'foo', + metricType: 'categorical', + value: 'bar' + }) + }) + } + + const { spans, llmobsSpans, evaluationMetrics } = run(payloadGenerator) + expect(spans).to.have.lengthOf(1) + expect(llmobsSpans).to.have.lengthOf(1) + expect(evaluationMetrics).to.have.lengthOf(1) + + // check eval metrics content + const exptected = [ + { + trace_id: spans[0].context().toTraceId(true), + span_id: spans[0].context().toSpanId(), + label: 'foo', + metric_type: 'categorical', + categorical_value: 'bar', + ml_app: 'test', + timestamp_ms: 1234567890, + tags: [`dd-trace.version:${tracerVersion}`, 'ml_app:test'] + } + ] + + check(exptected, evaluationMetrics) + + Date.now.restore() + }) +}) diff --git a/packages/dd-trace/test/llmobs/sdk/typescript/index.spec.js b/packages/dd-trace/test/llmobs/sdk/typescript/index.spec.js new file mode 100644 index 00000000000..b792a4fbdb7 --- /dev/null +++ b/packages/dd-trace/test/llmobs/sdk/typescript/index.spec.js @@ -0,0 +1,133 @@ +'use strict' + +const { execSync } = require('child_process') +const { + FakeAgent, + createSandbox, + spawnProc +} = require('../../../../../../integration-tests/helpers') +const chai = require('chai') +const path = require('path') +const { expectedLLMObsNonLLMSpanEvent, deepEqualWithMockValues } = require('../../util') + +chai.Assertion.addMethod('deepEqualWithMockValues', deepEqualWithMockValues) + +function check (expected, actual) { + for (const expectedLLMObsSpanIdx in expected) { + const expectedLLMObsSpan = expected[expectedLLMObsSpanIdx] + const actualLLMObsSpan = actual[expectedLLMObsSpanIdx] + expect(actualLLMObsSpan).to.deep.deepEqualWithMockValues(expectedLLMObsSpan) + } +} + +const testVersions = [ + '^1', + '^2', + '^3', + '^4', + '^5' +] + +const testCases = [ + { + name: 'not initialized', + file: 'noop' + }, + { + name: 'instruments an application with decorators', + file: 'index', + setup: (agent, results = {}) => { + const llmobsRes = agent.assertLlmObsPayloadReceived(({ payload }) => { + results.llmobsSpans = payload.spans + }) + + const apmRes = agent.assertMessageReceived(({ payload }) => { + results.apmSpans = payload + }) + + return [llmobsRes, apmRes] + }, + runTest: ({ llmobsSpans, apmSpans }) => { + const actual = llmobsSpans + const expected = [ + expectedLLMObsNonLLMSpanEvent({ + span: apmSpans[0][0], + spanKind: 'agent', + tags: { + ml_app: 'test', + language: 'javascript' + }, + inputValue: 'this is a', + outputValue: 'test' + }) + ] + + check(expected, actual) + } + } +] + +// a bit of devex to show the version we're actually testing +// so we don't need to know ahead of time +function getLatestVersion (range) { + const command = `npm show typescript@${range} version` + const output = execSync(command, { encoding: 'utf-8' }).trim() + const versions = output.split('\n').map(line => line.split(' ')[1].replace(/'/g, '')) + return versions[versions.length - 1] +} + +describe('typescript', () => { + let agent + let proc + let sandbox + + for (const version of testVersions) { + context(`with version ${getLatestVersion(version)}`, () => { + before(async function () { + this.timeout(20000) + sandbox = await createSandbox( + [`typescript@${version}`], false, ['./packages/dd-trace/test/llmobs/sdk/typescript/*'] + ) + }) + + after(async () => { + await sandbox.remove() + }) + + beforeEach(async () => { + agent = await new FakeAgent().start() + }) + + afterEach(async () => { + proc && proc.kill() + await agent.stop() + }) + + for (const test of testCases) { + const { name, file } = test + it(name, async () => { + const cwd = sandbox.folder + + const results = {} + const waiters = test.setup ? test.setup(agent, results) : [] + + // compile typescript + execSync( + `tsc --target ES6 --experimentalDecorators --module commonjs --sourceMap ${file}.ts`, + { cwd, stdio: 'inherit' } + ) + + proc = await spawnProc( + path.join(cwd, `${file}.js`), + { cwd, env: { DD_TRACE_AGENT_PORT: agent.port } } + ) + + await Promise.all(waiters) + + // some tests just need the file to run, not assert payloads + test.runTest && test.runTest(results) + }) + } + }) + } +}) diff --git a/packages/dd-trace/test/llmobs/sdk/typescript/index.ts b/packages/dd-trace/test/llmobs/sdk/typescript/index.ts new file mode 100644 index 00000000000..9aa320fd92c --- /dev/null +++ b/packages/dd-trace/test/llmobs/sdk/typescript/index.ts @@ -0,0 +1,23 @@ +// @ts-ignore +import tracer from 'dd-trace'; + +const llmobs = tracer.init({ + llmobs: { + mlApp: 'test', + } +}).llmobs; + +class Test { + @llmobs.decorate({ kind: 'agent' }) + runChain (input: string) { + llmobs.annotate({ + inputData: 'this is a', + outputData: 'test' + }) + + return 'world' + } +} + +const test: Test = new Test(); +test.runChain('hello'); diff --git a/packages/dd-trace/test/llmobs/sdk/typescript/noop.ts b/packages/dd-trace/test/llmobs/sdk/typescript/noop.ts new file mode 100644 index 00000000000..e1b7c00837b --- /dev/null +++ b/packages/dd-trace/test/llmobs/sdk/typescript/noop.ts @@ -0,0 +1,19 @@ +// @ts-ignore +import tracer from 'dd-trace'; +import * as assert from 'assert'; +const llmobs = tracer.llmobs; + +class Test { + @llmobs.decorate({ kind: 'agent' }) + runChain (input: string) { + llmobs.annotate({ + inputData: 'this is a', + outputData: 'test' + }) + + return 'world' + } +} + +const test: Test = new Test(); +assert.equal(test.runChain('hello'), 'world') \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/span_processor.spec.js b/packages/dd-trace/test/llmobs/span_processor.spec.js new file mode 100644 index 00000000000..ae73c4a9677 --- /dev/null +++ b/packages/dd-trace/test/llmobs/span_processor.spec.js @@ -0,0 +1,360 @@ +'use strict' + +const { expect } = require('chai') +const proxyquire = require('proxyquire') + +// we will use this to populate the span-tags map +const LLMObsTagger = require('../../src/llmobs/tagger') + +describe('span processor', () => { + let LLMObsSpanProcessor + let processor + let writer + let log + + beforeEach(() => { + writer = { + append: sinon.stub() + } + + log = { + warn: sinon.stub() + } + + LLMObsSpanProcessor = proxyquire('../../src/llmobs/span_processor', { + '../../../../package.json': { version: 'x.y.z' }, + '../log': log + }) + + processor = new LLMObsSpanProcessor({ llmobs: { enabled: true } }) + processor.setWriter(writer) + }) + + describe('process', () => { + let span + + it('should do nothing if llmobs is not enabled', () => { + processor = new LLMObsSpanProcessor({ llmobs: { enabled: false } }) + + expect(() => processor.process({ span })).not.to.throw() + }) + + it('should do nothing if the span is not an llm obs span', () => { + span = { context: () => ({ _tags: {} }) } + + expect(processor._writer.append).to.not.have.been.called + }) + + it('should format the span event for the writer', () => { + span = { + _name: 'test', + _startTime: 0, // this is in ms, will be converted to ns + _duration: 1, // this is in ms, will be converted to ns + context () { + return { + _tags: {}, + toTraceId () { return '123' }, // should not use this + toSpanId () { return '456' } + } + } + } + LLMObsTagger.tagMap.set(span, { + '_ml_obs.meta.span.kind': 'llm', + '_ml_obs.meta.model_name': 'myModel', + '_ml_obs.meta.model_provider': 'myProvider', + '_ml_obs.meta.metadata': { foo: 'bar' }, + '_ml_obs.meta.ml_app': 'myApp', + '_ml_obs.meta.input.value': 'input-value', + '_ml_obs.meta.output.value': 'output-value', + '_ml_obs.meta.input.messages': [{ role: 'user', content: 'hello' }], + '_ml_obs.meta.output.messages': [{ role: 'assistant', content: 'world' }], + '_ml_obs.llmobs_parent_id': '1234' + }) + + processor.process({ span }) + const payload = writer.append.getCall(0).firstArg + + expect(payload).to.deep.equal({ + trace_id: '123', + span_id: '456', + parent_id: '1234', + name: 'test', + tags: [ + 'version:', + 'env:', + 'service:', + 'source:integration', + 'ml_app:myApp', + 'dd-trace.version:x.y.z', + 'error:0', + 'language:javascript' + ], + start_ns: 0, + duration: 1000000, + status: 'ok', + meta: { + 'span.kind': 'llm', + model_name: 'myModel', + model_provider: 'myprovider', // should be lowercase + input: { + value: 'input-value', + messages: [{ role: 'user', content: 'hello' }] + }, + output: { + value: 'output-value', + messages: [{ role: 'assistant', content: 'world' }] + }, + metadata: { foo: 'bar' } + }, + metrics: {}, + _dd: { + trace_id: '123', + span_id: '456' + } + }) + + expect(writer.append).to.have.been.calledOnce + }) + + it('removes problematic fields from the metadata', () => { + // problematic fields are circular references or bigints + const metadata = { + bigint: BigInt(1), + deep: { + foo: 'bar' + }, + bar: 'baz' + } + metadata.circular = metadata + metadata.deep.circular = metadata.deep + span = { + context () { + return { + _tags: {}, + toTraceId () { return '123' }, + toSpanId () { return '456' } + } + } + } + + LLMObsTagger.tagMap.set(span, { + '_ml_obs.meta.span.kind': 'llm', + '_ml_obs.meta.metadata': metadata + }) + + processor.process({ span }) + const payload = writer.append.getCall(0).firstArg + + expect(payload.meta.metadata).to.deep.equal({ + bar: 'baz', + bigint: 'Unserializable value', + circular: 'Unserializable value', + deep: { foo: 'bar', circular: 'Unserializable value' } + }) + }) + + it('tags output documents for a retrieval span', () => { + span = { + context () { + return { + _tags: {}, + toTraceId () { return '123' }, + toSpanId () { return '456' } + } + } + } + + LLMObsTagger.tagMap.set(span, { + '_ml_obs.meta.span.kind': 'retrieval', + '_ml_obs.meta.output.documents': [{ text: 'hello', name: 'myDoc', id: '1', score: 0.6 }] + }) + + processor.process({ span }) + const payload = writer.append.getCall(0).firstArg + + expect(payload.meta.output.documents).to.deep.equal([{ + text: 'hello', + name: 'myDoc', + id: '1', + score: 0.6 + }]) + }) + + it('tags input documents for an embedding span', () => { + span = { + context () { + return { + _tags: {}, + toTraceId () { return '123' }, + toSpanId () { return '456' } + } + } + } + + LLMObsTagger.tagMap.set(span, { + '_ml_obs.meta.span.kind': 'embedding', + '_ml_obs.meta.input.documents': [{ text: 'hello', name: 'myDoc', id: '1', score: 0.6 }] + }) + + processor.process({ span }) + const payload = writer.append.getCall(0).firstArg + + expect(payload.meta.input.documents).to.deep.equal([{ + text: 'hello', + name: 'myDoc', + id: '1', + score: 0.6 + }]) + }) + + it('defaults model provider to custom', () => { + span = { + context () { + return { + _tags: {}, + toTraceId () { return '123' }, + toSpanId () { return '456' } + } + } + } + + LLMObsTagger.tagMap.set(span, { + '_ml_obs.meta.span.kind': 'llm', + '_ml_obs.meta.model_name': 'myModel' + }) + + processor.process({ span }) + const payload = writer.append.getCall(0).firstArg + + expect(payload.meta.model_provider).to.equal('custom') + }) + + it('sets an error appropriately', () => { + span = { + context () { + return { + _tags: { + 'error.message': 'error message', + 'error.type': 'error type', + 'error.stack': 'error stack' + }, + toTraceId () { return '123' }, + toSpanId () { return '456' } + } + } + } + + LLMObsTagger.tagMap.set(span, { + '_ml_obs.meta.span.kind': 'llm' + }) + + processor.process({ span }) + const payload = writer.append.getCall(0).firstArg + + expect(payload.meta['error.message']).to.equal('error message') + expect(payload.meta['error.type']).to.equal('error type') + expect(payload.meta['error.stack']).to.equal('error stack') + expect(payload.status).to.equal('error') + + expect(payload.tags).to.include('error_type:error type') + }) + + it('uses the error itself if the span does not have specific error fields', () => { + span = { + context () { + return { + _tags: { + error: new Error('error message') + }, + toTraceId () { return '123' }, + toSpanId () { return '456' } + } + } + } + + LLMObsTagger.tagMap.set(span, { + '_ml_obs.meta.span.kind': 'llm' + }) + + processor.process({ span }) + const payload = writer.append.getCall(0).firstArg + + expect(payload.meta['error.message']).to.equal('error message') + expect(payload.meta['error.type']).to.equal('Error') + expect(payload.meta['error.stack']).to.exist + expect(payload.status).to.equal('error') + + expect(payload.tags).to.include('error_type:Error') + }) + + it('uses the span name from the tag if provided', () => { + span = { + _name: 'test', + context () { + return { + _tags: {}, + toTraceId () { return '123' }, + toSpanId () { return '456' } + } + } + } + + LLMObsTagger.tagMap.set(span, { + '_ml_obs.meta.span.kind': 'llm', + '_ml_obs.name': 'mySpan' + }) + + processor.process({ span }) + const payload = writer.append.getCall(0).firstArg + + expect(payload.name).to.equal('mySpan') + }) + + it('attaches session id if provided', () => { + span = { + context () { + return { + _tags: {}, + toTraceId () { return '123' }, + toSpanId () { return '456' } + } + } + } + + LLMObsTagger.tagMap.set(span, { + '_ml_obs.meta.span.kind': 'llm', + '_ml_obs.session_id': '1234' + }) + + processor.process({ span }) + const payload = writer.append.getCall(0).firstArg + + expect(payload.session_id).to.equal('1234') + expect(payload.tags).to.include('session_id:1234') + }) + + it('sets span tags appropriately', () => { + span = { + context () { + return { + _tags: {}, + toTraceId () { return '123' }, + toSpanId () { return '456' } + } + } + } + + LLMObsTagger.tagMap.set(span, { + '_ml_obs.meta.span.kind': 'llm', + '_ml_obs.tags': { hostname: 'localhost', foo: 'bar', source: 'mySource' } + }) + + processor.process({ span }) + const payload = writer.append.getCall(0).firstArg + + expect(payload.tags).to.include('foo:bar') + expect(payload.tags).to.include('source:mySource') + expect(payload.tags).to.include('hostname:localhost') + }) + }) +}) diff --git a/packages/dd-trace/test/llmobs/tagger.spec.js b/packages/dd-trace/test/llmobs/tagger.spec.js new file mode 100644 index 00000000000..783ce91bdae --- /dev/null +++ b/packages/dd-trace/test/llmobs/tagger.spec.js @@ -0,0 +1,576 @@ +'use strict' + +const { expect } = require('chai') +const proxyquire = require('proxyquire') + +function unserializbleObject () { + const obj = {} + obj.obj = obj + return obj +} + +describe('tagger', () => { + let span + let spanContext + let Tagger + let tagger + let logger + let util + + beforeEach(() => { + spanContext = { + _tags: {}, + _trace: { tags: {} } + } + + span = { + context () { return spanContext }, + setTag (k, v) { + this.context()._tags[k] = v + } + } + + util = { + generateTraceId: sinon.stub().returns('0123') + } + + logger = { + warn: sinon.stub() + } + + Tagger = proxyquire('../../src/llmobs/tagger', { + '../log': logger, + './util': util + }) + }) + + describe('without softFail', () => { + beforeEach(() => { + tagger = new Tagger({ llmobs: { enabled: true, mlApp: 'my-default-ml-app' } }) + }) + + describe('registerLLMObsSpan', () => { + it('will not set tags if llmobs is not enabled', () => { + tagger = new Tagger({ llmobs: { enabled: false } }) + tagger.registerLLMObsSpan(span, 'llm') + + expect(Tagger.tagMap.get(span)).to.deep.equal(undefined) + }) + + it('tags an llm obs span with basic and default properties', () => { + tagger.registerLLMObsSpan(span, { kind: 'workflow' }) + + expect(Tagger.tagMap.get(span)).to.deep.equal({ + '_ml_obs.meta.span.kind': 'workflow', + '_ml_obs.meta.ml_app': 'my-default-ml-app', + '_ml_obs.llmobs_parent_id': 'undefined' // no parent id provided + }) + }) + + it('uses options passed in to set tags', () => { + tagger.registerLLMObsSpan(span, { + kind: 'llm', + modelName: 'my-model', + modelProvider: 'my-provider', + sessionId: 'my-session', + mlApp: 'my-app' + }) + + expect(Tagger.tagMap.get(span)).to.deep.equal({ + '_ml_obs.meta.span.kind': 'llm', + '_ml_obs.meta.model_name': 'my-model', + '_ml_obs.meta.model_provider': 'my-provider', + '_ml_obs.session_id': 'my-session', + '_ml_obs.meta.ml_app': 'my-app', + '_ml_obs.llmobs_parent_id': 'undefined' + }) + }) + + it('uses the name if provided', () => { + tagger.registerLLMObsSpan(span, { kind: 'llm', name: 'my-span-name' }) + + expect(Tagger.tagMap.get(span)).to.deep.equal({ + '_ml_obs.meta.span.kind': 'llm', + '_ml_obs.meta.ml_app': 'my-default-ml-app', + '_ml_obs.llmobs_parent_id': 'undefined', + '_ml_obs.name': 'my-span-name' + }) + }) + + it('defaults parent id to undefined', () => { + tagger.registerLLMObsSpan(span, { kind: 'llm' }) + + expect(Tagger.tagMap.get(span)).to.deep.equal({ + '_ml_obs.meta.span.kind': 'llm', + '_ml_obs.meta.ml_app': 'my-default-ml-app', + '_ml_obs.llmobs_parent_id': 'undefined' + }) + }) + + it('uses the parent span if provided to populate fields', () => { + const parentSpan = { + context () { + return { + _tags: { + '_ml_obs.meta.ml_app': 'my-ml-app', + '_ml_obs.session_id': 'my-session' + }, + toSpanId () { return '5678' } + } + } + } + tagger.registerLLMObsSpan(span, { kind: 'llm', parent: parentSpan }) + + expect(Tagger.tagMap.get(span)).to.deep.equal({ + '_ml_obs.meta.span.kind': 'llm', + '_ml_obs.meta.ml_app': 'my-ml-app', + '_ml_obs.session_id': 'my-session', + '_ml_obs.llmobs_parent_id': '5678' + }) + }) + + it('uses the propagated trace id if provided', () => { + tagger.registerLLMObsSpan(span, { kind: 'llm' }) + + expect(Tagger.tagMap.get(span)).to.deep.equal({ + '_ml_obs.meta.span.kind': 'llm', + '_ml_obs.meta.ml_app': 'my-default-ml-app', + '_ml_obs.llmobs_parent_id': 'undefined' + }) + }) + + it('uses the propagated parent id if provided', () => { + spanContext._trace.tags['_dd.p.llmobs_parent_id'] = '-567' + + tagger.registerLLMObsSpan(span, { kind: 'llm' }) + + expect(Tagger.tagMap.get(span)).to.deep.equal({ + '_ml_obs.meta.span.kind': 'llm', + '_ml_obs.meta.ml_app': 'my-default-ml-app', + '_ml_obs.llmobs_parent_id': '-567' + }) + }) + + it('does not set span type if the LLMObs span kind is falsy', () => { + tagger.registerLLMObsSpan(span, { kind: false }) + + expect(Tagger.tagMap.get(span)).to.be.undefined + }) + }) + + describe('tagMetadata', () => { + it('tags a span with metadata', () => { + tagger._register(span) + tagger.tagMetadata(span, { a: 'foo', b: 'bar' }) + expect(Tagger.tagMap.get(span)).to.deep.equal({ + '_ml_obs.meta.metadata': { a: 'foo', b: 'bar' } + }) + }) + }) + + describe('tagMetrics', () => { + it('tags a span with metrics', () => { + tagger._register(span) + tagger.tagMetrics(span, { a: 1, b: 2 }) + expect(Tagger.tagMap.get(span)).to.deep.equal({ + '_ml_obs.metrics': { a: 1, b: 2 } + }) + }) + + it('tags maps token metric names appropriately', () => { + tagger._register(span) + tagger.tagMetrics(span, { + inputTokens: 1, + outputTokens: 2, + totalTokens: 3, + foo: 10 + }) + expect(Tagger.tagMap.get(span)).to.deep.equal({ + '_ml_obs.metrics': { input_tokens: 1, output_tokens: 2, total_tokens: 3, foo: 10 } + }) + }) + + it('throws for non-number entries', () => { + const metrics = { + a: 1, + b: 'foo', + c: { depth: 1 }, + d: undefined + } + tagger._register(span) + expect(() => tagger.tagMetrics(span, metrics)).to.throw() + }) + }) + + describe('tagSpanTags', () => { + it('sets tags on a span', () => { + const tags = { foo: 'bar' } + tagger._register(span) + tagger.tagSpanTags(span, tags) + expect(Tagger.tagMap.get(span)).to.deep.equal({ + '_ml_obs.tags': { foo: 'bar' } + }) + }) + + it('merges tags so they do not overwrite', () => { + Tagger.tagMap.set(span, { '_ml_obs.tags': { a: 1 } }) + const tags = { a: 2, b: 1 } + tagger.tagSpanTags(span, tags) + expect(Tagger.tagMap.get(span)).to.deep.equal({ + '_ml_obs.tags': { a: 1, b: 1 } + }) + }) + }) + + describe('tagLLMIO', () => { + it('tags a span with llm io', () => { + const inputData = [ + 'you are an amazing assistant', + { content: 'hello! my name is foobar' }, + { content: 'I am a robot', role: 'assistant' }, + { content: 'I am a human', role: 'user' }, + {} + ] + + const outputData = 'Nice to meet you, human!' + + tagger._register(span) + tagger.tagLLMIO(span, inputData, outputData) + expect(Tagger.tagMap.get(span)).to.deep.equal({ + '_ml_obs.meta.input.messages': [ + { content: 'you are an amazing assistant' }, + { content: 'hello! my name is foobar' }, + { content: 'I am a robot', role: 'assistant' }, + { content: 'I am a human', role: 'user' }, + { content: '' } + ], + '_ml_obs.meta.output.messages': [{ content: 'Nice to meet you, human!' }] + }) + }) + + it('throws for a non-object message', () => { + const messages = [ + 5 + ] + + expect(() => tagger.tagLLMIO(span, messages, undefined)).to.throw() + }) + + it('throws for a non-string message content', () => { + const messages = [ + { content: 5 } + ] + + expect(() => tagger.tagLLMIO(span, messages, undefined)).to.throw() + }) + + it('throws for a non-string message role', () => { + const messages = [ + { content: 'a', role: 5 } + ] + + expect(() => tagger.tagLLMIO(span, messages, undefined)).to.throw() + }) + + describe('tagging tool calls appropriately', () => { + it('tags a span with tool calls', () => { + const inputData = [ + { content: 'hello', toolCalls: [{ name: 'tool1' }, { name: 'tool2', arguments: { a: 1, b: 2 } }] }, + { content: 'goodbye', toolCalls: [{ name: 'tool3' }] } + ] + const outputData = [ + { content: 'hi', toolCalls: [{ name: 'tool4' }] } + ] + + tagger._register(span) + tagger.tagLLMIO(span, inputData, outputData) + expect(Tagger.tagMap.get(span)).to.deep.equal({ + '_ml_obs.meta.input.messages': [ + { + content: 'hello', + tool_calls: [{ name: 'tool1' }, { name: 'tool2', arguments: { a: 1, b: 2 } }] + }, { + content: 'goodbye', + tool_calls: [{ name: 'tool3' }] + }], + '_ml_obs.meta.output.messages': [{ content: 'hi', tool_calls: [{ name: 'tool4' }] }] + }) + }) + + it('throws for a non-object tool call', () => { + const messages = [ + { content: 'a', toolCalls: 5 } + ] + + expect(() => tagger.tagLLMIO(span, messages, undefined)).to.throw() + }) + + it('throws for a non-string tool name', () => { + const messages = [ + { content: 'a', toolCalls: [{ name: 5 }] } + ] + + expect(() => tagger.tagLLMIO(span, messages, undefined)).to.throw() + }) + + it('throws for a non-object tool arguments', () => { + const messages = [ + { content: 'a', toolCalls: [{ name: 'tool1', arguments: 5 }] } + ] + + expect(() => tagger.tagLLMIO(span, messages, undefined)).to.throw() + }) + + it('throws for a non-string tool id', () => { + const messages = [ + { content: 'a', toolCalls: [{ name: 'tool1', toolId: 5 }] } + ] + + expect(() => tagger.tagLLMIO(span, messages, undefined)).to.throw() + }) + + it('throws for a non-string tool type', () => { + const messages = [ + { content: 'a', toolCalls: [{ name: 'tool1', type: 5 }] } + ] + + expect(() => tagger.tagLLMIO(span, messages, undefined)).to.throw() + }) + + it('logs multiple errors if there are multiple errors for a message and filters it out', () => { + const messages = [ + { content: 'a', toolCalls: [5, { name: 5, type: 7 }], role: 7 } + ] + + expect(() => tagger.tagLLMIO(span, messages, undefined)).to.throw() + }) + }) + }) + + describe('tagEmbeddingIO', () => { + it('tags a span with embedding io', () => { + const inputData = [ + 'my string document', + { text: 'my object document' }, + { text: 'foo', name: 'bar' }, + { text: 'baz', id: 'qux' }, + { text: 'quux', score: 5 }, + { text: 'foo', name: 'bar', id: 'qux', score: 5 } + ] + const outputData = 'embedded documents' + tagger._register(span) + tagger.tagEmbeddingIO(span, inputData, outputData) + expect(Tagger.tagMap.get(span)).to.deep.equal({ + '_ml_obs.meta.input.documents': [ + { text: 'my string document' }, + { text: 'my object document' }, + { text: 'foo', name: 'bar' }, + { text: 'baz', id: 'qux' }, + { text: 'quux', score: 5 }, + { text: 'foo', name: 'bar', id: 'qux', score: 5 }], + '_ml_obs.meta.output.value': 'embedded documents' + }) + }) + + it('throws for a non-object document', () => { + const documents = [ + 5 + ] + + expect(() => tagger.tagEmbeddingIO(span, documents, undefined)).to.throw() + }) + + it('throws for a non-string document text', () => { + const documents = [ + { text: 5 } + ] + + expect(() => tagger.tagEmbeddingIO(span, documents, undefined)).to.throw() + }) + + it('throws for a non-string document name', () => { + const documents = [ + { text: 'a', name: 5 } + ] + + expect(() => tagger.tagEmbeddingIO(span, documents, undefined)).to.throw() + }) + + it('throws for a non-string document id', () => { + const documents = [ + { text: 'a', id: 5 } + ] + + expect(() => tagger.tagEmbeddingIO(span, documents, undefined)).to.throw() + }) + + it('throws for a non-number document score', () => { + const documents = [ + { text: 'a', score: '5' } + ] + + expect(() => tagger.tagEmbeddingIO(span, documents, undefined)).to.throw() + }) + }) + + describe('tagRetrievalIO', () => { + it('tags a span with retrieval io', () => { + const inputData = 'some query' + const outputData = [ + 'result 1', + { text: 'result 2' }, + { text: 'foo', name: 'bar' }, + { text: 'baz', id: 'qux' }, + { text: 'quux', score: 5 }, + { text: 'foo', name: 'bar', id: 'qux', score: 5 } + ] + + tagger._register(span) + tagger.tagRetrievalIO(span, inputData, outputData) + expect(Tagger.tagMap.get(span)).to.deep.equal({ + '_ml_obs.meta.input.value': 'some query', + '_ml_obs.meta.output.documents': [ + { text: 'result 1' }, + { text: 'result 2' }, + { text: 'foo', name: 'bar' }, + { text: 'baz', id: 'qux' }, + { text: 'quux', score: 5 }, + { text: 'foo', name: 'bar', id: 'qux', score: 5 }] + }) + }) + + it('throws for malformed properties on documents', () => { + const inputData = 'some query' + const outputData = [ + true, + { text: 5 }, + { text: 'foo', name: 5 }, + 'hi', + null, + undefined + ] + + // specific cases of throwing tested with embedding inputs + expect(() => tagger.tagRetrievalIO(span, inputData, outputData)).to.throw() + }) + }) + + describe('tagTextIO', () => { + it('tags a span with text io', () => { + const inputData = { some: 'object' } + const outputData = 'some text' + tagger._register(span) + tagger.tagTextIO(span, inputData, outputData) + expect(Tagger.tagMap.get(span)).to.deep.equal({ + '_ml_obs.meta.input.value': '{"some":"object"}', + '_ml_obs.meta.output.value': 'some text' + }) + }) + + it('throws when the value is not JSON serializable', () => { + const data = unserializbleObject() + expect(() => tagger.tagTextIO(span, data, 'output')).to.throw() + }) + }) + }) + + describe('with softFail', () => { + beforeEach(() => { + tagger = new Tagger({ llmobs: { enabled: true, mlApp: 'my-default-ml-app' } }, true) + }) + + it('logs a warning when an unexpected value is encountered for text tagging', () => { + const data = unserializbleObject() + tagger._register(span) + tagger.tagTextIO(span, data, 'input') + expect(logger.warn).to.have.been.calledOnce + }) + + it('logs a warning when an unexpected value is encountered for metrics tagging', () => { + const metrics = { + a: 1, + b: 'foo' + } + + tagger._register(span) + tagger.tagMetrics(span, metrics) + expect(logger.warn).to.have.been.calledOnce + }) + + describe('tagDocuments', () => { + it('logs a warning when a document is not an object', () => { + const data = [undefined] + tagger._register(span) + tagger.tagEmbeddingIO(span, data, undefined) + expect(logger.warn).to.have.been.calledOnce + }) + + it('logs multiple warnings otherwise', () => { + const documents = [ + { + text: 'a', + name: 5, + id: 7, + score: '5' + } + ] + + tagger._register(span) + tagger.tagEmbeddingIO(span, documents, undefined) + expect(logger.warn.callCount).to.equal(3) + }) + }) + + describe('tagMessages', () => { + it('logs a warning when a message is not an object', () => { + const messages = [5] + tagger._register(span) + tagger.tagLLMIO(span, messages, undefined) + expect(logger.warn).to.have.been.calledOnce + }) + + it('logs multiple warnings otherwise', () => { + const messages = [ + { content: 5, role: 5 } + ] + + tagger._register(span) + tagger.tagLLMIO(span, messages, undefined) + expect(logger.warn.callCount).to.equal(2) + }) + + describe('tool call tagging', () => { + it('logs a warning when a message tool call is not an object', () => { + const messages = [ + { content: 'a', toolCalls: 5 } + ] + + tagger._register(span) + tagger.tagLLMIO(span, messages, undefined) + expect(logger.warn).to.have.been.calledOnce + }) + + it('logs multiple warnings otherwise', () => { + const messages = [ + { + content: 'a', + toolCalls: [ + { + name: 5, + arguments: 'not an object', + toolId: 5, + type: 5 + } + ], + role: 7 + } + ] + + tagger._register(span) + tagger.tagLLMIO(span, messages, undefined) + expect(logger.warn.callCount).to.equal(5) // 4 for tool call + 1 for role + }) + }) + }) + }) +}) diff --git a/packages/dd-trace/test/llmobs/util.js b/packages/dd-trace/test/llmobs/util.js new file mode 100644 index 00000000000..4c3b76da090 --- /dev/null +++ b/packages/dd-trace/test/llmobs/util.js @@ -0,0 +1,201 @@ +'use strict' + +const chai = require('chai') + +const tracerVersion = require('../../../../package.json').version + +const MOCK_STRING = Symbol('string') +const MOCK_NUMBER = Symbol('number') +const MOCK_ANY = Symbol('any') + +function deepEqualWithMockValues (expected) { + const actual = this._obj + + for (const key in actual) { + if (expected[key] === MOCK_STRING) { + new chai.Assertion(typeof actual[key], `key ${key}`).to.equal('string') + } else if (expected[key] === MOCK_NUMBER) { + new chai.Assertion(typeof actual[key], `key ${key}`).to.equal('number') + } else if (expected[key] === MOCK_ANY) { + new chai.Assertion(actual[key], `key ${key}`).to.exist + } else if (Array.isArray(expected[key])) { + const sortedExpected = [...expected[key].sort()] + const sortedActual = [...actual[key].sort()] + new chai.Assertion(sortedActual, `key: ${key}`).to.deep.equal(sortedExpected) + } else if (typeof expected[key] === 'object') { + new chai.Assertion(actual[key], `key: ${key}`).to.deepEqualWithMockValues(expected[key]) + } else { + new chai.Assertion(actual[key], `key: ${key}`).to.equal(expected[key]) + } + } +} + +function expectedLLMObsLLMSpanEvent (options) { + const spanEvent = expectedLLMObsBaseEvent(options) + + const meta = { input: {}, output: {} } + const { + spanKind, + modelName, + modelProvider, + inputMessages, + inputDocuments, + outputMessages, + outputValue, + metadata, + tokenMetrics + } = options + + if (spanKind === 'llm') { + if (inputMessages) meta.input.messages = inputMessages + if (outputMessages) meta.output.messages = outputMessages + } else if (spanKind === 'embedding') { + if (inputDocuments) meta.input.documents = inputDocuments + if (outputValue) meta.output.value = outputValue + } + + if (!spanEvent.meta.input) delete spanEvent.meta.input + if (!spanEvent.meta.output) delete spanEvent.meta.output + + if (modelName) meta.model_name = modelName + if (modelProvider) meta.model_provider = modelProvider + if (metadata) meta.metadata = metadata + + Object.assign(spanEvent.meta, meta) + + if (tokenMetrics) spanEvent.metrics = tokenMetrics + + return spanEvent +} + +function expectedLLMObsNonLLMSpanEvent (options) { + const spanEvent = expectedLLMObsBaseEvent(options) + const { + spanKind, + inputValue, + outputValue, + outputDocuments, + metadata, + tokenMetrics + } = options + + const meta = { input: {}, output: {} } + if (spanKind === 'retrieval') { + if (inputValue) meta.input.value = inputValue + if (outputDocuments) meta.output.documents = outputDocuments + if (outputValue) meta.output.value = outputValue + } + if (inputValue) meta.input.value = inputValue + if (metadata) meta.metadata = metadata + if (outputValue) meta.output.value = outputValue + + if (!spanEvent.meta.input) delete spanEvent.meta.input + if (!spanEvent.meta.output) delete spanEvent.meta.output + + Object.assign(spanEvent.meta, meta) + + if (tokenMetrics) spanEvent.metrics = tokenMetrics + + return spanEvent +} + +function expectedLLMObsBaseEvent ({ + span, + parentId, + name, + spanKind, + tags, + sessionId, + error, + errorType, + errorMessage, + errorStack +} = {}) { + // the `span` could be a raw DatadogSpan or formatted span + const spanName = name || span.name || span._name + const spanId = span.span_id ? fromBuffer(span.span_id) : span.context().toSpanId() + const startNs = span.start ? fromBuffer(span.start, true) : Math.round(span._startTime * 1e6) + const duration = span.duration ? fromBuffer(span.duration, true) : Math.round(span._duration * 1e6) + + const spanEvent = { + trace_id: MOCK_STRING, + span_id: spanId, + parent_id: parentId || 'undefined', + name: spanName, + tags: expectedLLMObsTags({ span, tags, error, errorType, sessionId }), + start_ns: startNs, + duration, + status: error ? 'error' : 'ok', + meta: { 'span.kind': spanKind }, + metrics: {}, + _dd: { + trace_id: MOCK_STRING, + span_id: spanId + } + } + + if (sessionId) spanEvent.session_id = sessionId + + if (error) { + spanEvent.meta['error.type'] = errorType + spanEvent.meta['error.message'] = errorMessage + spanEvent.meta['error.stack'] = errorStack + } + + return spanEvent +} + +function expectedLLMObsTags ({ + span, + error, + errorType, + tags, + sessionId +}) { + tags = tags || {} + + const version = span.meta?.version || span._parentTracer?._version + const env = span.meta?.env || span._parentTracer?._env + const service = span.meta?.service || span._parentTracer?._service + + const spanTags = [ + `version:${version ?? ''}`, + `env:${env ?? ''}`, + `service:${service ?? ''}`, + 'source:integration', + `ml_app:${tags.ml_app}`, + `dd-trace.version:${tracerVersion}` + ] + + if (sessionId) spanTags.push(`session_id:${sessionId}`) + + if (error) { + spanTags.push('error:1') + if (errorType) spanTags.push(`error_type:${errorType}`) + } else { + spanTags.push('error:0') + } + + for (const [key, value] of Object.entries(tags)) { + if (!['version', 'env', 'service', 'ml_app'].includes(key)) { + spanTags.push(`${key}:${value}`) + } + } + + return spanTags +} + +function fromBuffer (spanProperty, isNumber = false) { + const { buffer, offset } = spanProperty + const strVal = buffer.readBigInt64BE(offset).toString() + return isNumber ? Number(strVal) : strVal +} + +module.exports = { + expectedLLMObsLLMSpanEvent, + expectedLLMObsNonLLMSpanEvent, + deepEqualWithMockValues, + MOCK_ANY, + MOCK_NUMBER, + MOCK_STRING +} diff --git a/packages/dd-trace/test/llmobs/util.spec.js b/packages/dd-trace/test/llmobs/util.spec.js new file mode 100644 index 00000000000..063e618c1ef --- /dev/null +++ b/packages/dd-trace/test/llmobs/util.spec.js @@ -0,0 +1,142 @@ +'use strict' + +const { + encodeUnicode, + getFunctionArguments, + validateKind +} = require('../../src/llmobs/util') + +describe('util', () => { + describe('encodeUnicode', () => { + it('should encode unicode characters', () => { + expect(encodeUnicode('😀')).to.equal('\\ud83d\\ude00') + }) + + it('should encode only unicode characters in a string', () => { + expect(encodeUnicode('test 😀')).to.equal('test \\ud83d\\ude00') + }) + }) + + describe('validateKind', () => { + for (const kind of ['llm', 'agent', 'task', 'tool', 'workflow', 'retrieval', 'embedding']) { + it(`should return true for valid kind: ${kind}`, () => { + expect(validateKind(kind)).to.equal(kind) + }) + } + + it('should throw for an empty string', () => { + expect(() => validateKind('')).to.throw() + }) + + it('should throw for an invalid kind', () => { + expect(() => validateKind('invalid')).to.throw() + }) + + it('should throw for an undefined kind', () => { + expect(() => validateKind()).to.throw() + }) + }) + + describe('getFunctionArguments', () => { + describe('functionality', () => { + it('should return undefined for a function without arguments', () => { + expect(getFunctionArguments(() => {})).to.deep.equal(undefined) + }) + + it('should capture a single argument only by its value', () => { + expect(getFunctionArguments((arg) => {}, ['bar'])).to.deep.equal('bar') + }) + + it('should capture multiple arguments by name', () => { + expect(getFunctionArguments((foo, bar) => {}, ['foo', 'bar'])).to.deep.equal({ foo: 'foo', bar: 'bar' }) + }) + + it('should ignore arguments not passed in', () => { + expect(getFunctionArguments((foo, bar, baz) => {}, ['foo', 'bar'])).to.deep.equal({ foo: 'foo', bar: 'bar' }) + }) + + it('should capture spread arguments', () => { + expect( + getFunctionArguments((foo, bar, ...args) => {}, ['foo', 'bar', 1, 2, 3]) + ).to.deep.equal({ foo: 'foo', bar: 'bar', args: [1, 2, 3] }) + }) + }) + + describe('parsing configurations', () => { + it('should parse multiple arguments with single-line comments', () => { + function foo ( + bar, // bar comment + baz // baz comment + ) {} + + expect(getFunctionArguments(foo, ['bar', 'baz'])).to.deep.equal({ bar: 'bar', baz: 'baz' }) + }) + + it('should parse multiple arguments with multi-line comments', () => { + function foo ( + bar, /* bar comment */ + baz /* baz comment */ + ) {} + + expect(getFunctionArguments(foo, ['bar', 'baz'])).to.deep.equal({ bar: 'bar', baz: 'baz' }) + }) + + it('should parse multiple arguments with stacked multi-line comments', () => { + function foo ( + /** + * hello + */ + bar, + /** + * world + */ + baz + ) {} + + expect(getFunctionArguments(foo, ['bar', 'baz'])).to.deep.equal({ bar: 'bar', baz: 'baz' }) + }) + + it('parses when simple default values are present', () => { + function foo (bar = 'baz') {} + + expect(getFunctionArguments(foo, ['bar'])).to.deep.equal('bar') + }) + + it('should ignore the default value when no argument is passed', () => { + function foo (bar = 'baz') {} + + expect(getFunctionArguments(foo, [])).to.deep.equal(undefined) + }) + + it('parses when a default value is a function', () => { + function foo (bar = () => {}, baz = 4) {} + + expect(getFunctionArguments(foo, ['bar'])).to.deep.equal('bar') + }) + + it('parses when a simple object is passed in', () => { + function foo (bar = { baz: 4 }) {} + + expect(getFunctionArguments(foo, ['bar'])).to.deep.equal('bar') + }) + + it('parses when a complex object is passed in', () => { + function foo (bar = { baz: { a: 5, b: { c: 4 } }, bat: 0 }, baz) {} + + expect(getFunctionArguments(foo, [{ bar: 'baz' }, 'baz'])).to.deep.equal({ bar: { bar: 'baz' }, baz: 'baz' }) + }) + + it('parses when one of the arguments is an arrow function', () => { + function foo (fn = (a, b, c) => {}, ctx) {} + + expect(getFunctionArguments(foo, ['fn', 'ctx'])).to.deep.equal({ fn: 'fn', ctx: 'ctx' }) + }) + + it('parses when one of the arguments is a function', () => { + function foo (fn = function (a, b, c) {}, ctx) {} + + expect(getFunctionArguments(foo, ['fn', 'ctx'])).to.deep.equal({ fn: 'fn', ctx: 'ctx' }) + }) + }) + }) +}) diff --git a/packages/dd-trace/test/llmobs/writers/base.spec.js b/packages/dd-trace/test/llmobs/writers/base.spec.js new file mode 100644 index 00000000000..8b971b2748a --- /dev/null +++ b/packages/dd-trace/test/llmobs/writers/base.spec.js @@ -0,0 +1,179 @@ +'use strict' +const { expect } = require('chai') +const proxyquire = require('proxyquire') + +describe('BaseLLMObsWriter', () => { + let BaseLLMObsWriter + let writer + let request + let clock + let options + let logger + + beforeEach(() => { + request = sinon.stub() + logger = { + debug: sinon.stub(), + warn: sinon.stub(), + error: sinon.stub() + } + BaseLLMObsWriter = proxyquire('../../../src/llmobs/writers/base', { + '../../exporters/common/request': request, + '../../log': logger + }) + + clock = sinon.useFakeTimers() + + options = { + endpoint: '/api/v2/llmobs', + intake: 'llmobs-intake.datadoghq.com' + } + }) + + afterEach(() => { + clock.restore() + process.removeAllListeners('beforeExit') + }) + + it('constructs a writer with a url', () => { + writer = new BaseLLMObsWriter(options) + + expect(writer._url.href).to.equal('https://llmobs-intake.datadoghq.com/api/v2/llmobs') + expect(logger.debug).to.have.been.calledWith( + 'Started BaseLLMObsWriter to https://llmobs-intake.datadoghq.com/api/v2/llmobs' + ) + }) + + it('calls flush before the process exits', () => { + writer = new BaseLLMObsWriter(options) + writer.flush = sinon.spy() + + process.emit('beforeExit') + + expect(writer.flush).to.have.been.calledOnce + }) + + it('calls flush at the correct interval', async () => { + writer = new BaseLLMObsWriter(options) + + writer.flush = sinon.spy() + + clock.tick(1000) + + expect(writer.flush).to.have.been.calledOnce + }) + + it('appends an event to the buffer', () => { + writer = new BaseLLMObsWriter(options) + const event = { foo: 'bar–' } + writer.append(event) + + expect(writer._buffer).to.have.lengthOf(1) + expect(writer._buffer[0]).to.deep.equal(event) + expect(writer._bufferSize).to.equal(16) + }) + + it('does not append an event if the buffer is full', () => { + writer = new BaseLLMObsWriter(options) + + for (let i = 0; i < 1000; i++) { + writer.append({ foo: 'bar' }) + } + + writer.append({ foo: 'bar' }) + expect(writer._buffer).to.have.lengthOf(1000) + expect(logger.warn).to.have.been.calledWith('BaseLLMObsWriter event buffer full (limit is 1000), dropping event') + }) + + it('flushes the buffer', () => { + writer = new BaseLLMObsWriter(options) + + const event1 = { foo: 'bar' } + const event2 = { foo: 'baz' } + + writer.append(event1) + writer.append(event2) + + writer.makePayload = (events) => ({ events }) + + // Stub the request function to call its third argument + request.callsFake((url, options, callback) => { + callback(null, null, 202) + }) + + writer.flush() + + expect(request).to.have.been.calledOnce + const calledArgs = request.getCall(0).args + + expect(calledArgs[0]).to.deep.equal(JSON.stringify({ events: [event1, event2] })) + expect(calledArgs[1]).to.deep.equal({ + headers: { + 'Content-Type': 'application/json' + }, + method: 'POST', + url: writer._url, + timeout: 5000 + }) + + expect(logger.debug).to.have.been.calledWith( + 'Sent 2 LLMObs undefined events to https://llmobs-intake.datadoghq.com/api/v2/llmobs' + ) + + expect(writer._buffer).to.have.lengthOf(0) + expect(writer._bufferSize).to.equal(0) + }) + + it('does not flush an empty buffer', () => { + writer = new BaseLLMObsWriter(options) + writer.flush() + + expect(request).to.not.have.been.called + }) + + it('logs errors from the request', () => { + writer = new BaseLLMObsWriter(options) + writer.makePayload = (events) => ({ events }) + + writer.append({ foo: 'bar' }) + + const error = new Error('boom') + request.callsFake((url, options, callback) => { + callback(error) + }) + + writer.flush() + + expect(logger.error).to.have.been.calledWith( + 'Error sending 1 LLMObs undefined events to https://llmobs-intake.datadoghq.com/api/v2/llmobs: boom' + ) + }) + + describe('destroy', () => { + it('destroys the writer', () => { + sinon.spy(global, 'clearInterval') + sinon.spy(process, 'removeListener') + writer = new BaseLLMObsWriter(options) + writer.flush = sinon.stub() + + writer.destroy() + + expect(writer._destroyed).to.be.true + expect(clearInterval).to.have.been.calledWith(writer._periodic) + expect(process.removeListener).to.have.been.calledWith('beforeExit', writer.destroy) + expect(writer.flush).to.have.been.calledOnce + expect(logger.debug) + .to.have.been.calledWith('Stopping BaseLLMObsWriter') + }) + + it('does not destroy more than once', () => { + writer = new BaseLLMObsWriter(options) + + logger.debug.reset() // ignore log from constructor + writer.destroy() + writer.destroy() + + expect(logger.debug).to.have.been.calledOnce + }) + }) +}) diff --git a/packages/dd-trace/test/llmobs/writers/evaluations.spec.js b/packages/dd-trace/test/llmobs/writers/evaluations.spec.js new file mode 100644 index 00000000000..e81955450c4 --- /dev/null +++ b/packages/dd-trace/test/llmobs/writers/evaluations.spec.js @@ -0,0 +1,46 @@ +'use strict' + +describe('LLMObsEvalMetricsWriter', () => { + let LLMObsEvalMetricsWriter + let writer + let flush + + beforeEach(() => { + LLMObsEvalMetricsWriter = require('../../../src/llmobs/writers/evaluations') + flush = sinon.stub() + }) + + afterEach(() => { + process.removeAllListeners('beforeExit') + }) + + it('constructs the writer with the correct values', () => { + writer = new LLMObsEvalMetricsWriter({ + site: 'datadoghq.com', + llmobs: {}, + apiKey: '1234' + }) + + writer.flush = flush // just to stop the beforeExit flush call + + expect(writer._url.href).to.equal('https://api.datadoghq.com/api/intake/llm-obs/v1/eval-metric') + expect(writer._headers['DD-API-KEY']).to.equal('1234') + expect(writer._eventType).to.equal('evaluation_metric') + }) + + it('builds the payload correctly', () => { + writer = new LLMObsEvalMetricsWriter({ + site: 'datadoghq.com', + apiKey: 'test' + }) + + const events = [ + { name: 'test', value: 1 } + ] + + const payload = writer.makePayload(events) + + expect(payload.data.type).to.equal('evaluation_metric') + expect(payload.data.attributes.metrics).to.deep.equal(events) + }) +}) diff --git a/packages/dd-trace/test/llmobs/writers/spans/agentProxy.spec.js b/packages/dd-trace/test/llmobs/writers/spans/agentProxy.spec.js new file mode 100644 index 00000000000..6ed0f150885 --- /dev/null +++ b/packages/dd-trace/test/llmobs/writers/spans/agentProxy.spec.js @@ -0,0 +1,28 @@ +'use stict' + +describe('LLMObsAgentProxySpanWriter', () => { + let LLMObsAgentProxySpanWriter + let writer + + beforeEach(() => { + LLMObsAgentProxySpanWriter = require('../../../../src/llmobs/writers/spans/agentProxy') + }) + + it('is initialized correctly', () => { + writer = new LLMObsAgentProxySpanWriter({ + hostname: '127.0.0.1', + port: 8126 + }) + + expect(writer._url.href).to.equal('http://127.0.0.1:8126/evp_proxy/v2/api/v2/llmobs') + expect(writer._headers['X-Datadog-EVP-Subdomain']).to.equal('llmobs-intake') + }) + + it('is initialized correctly with default hostname', () => { + writer = new LLMObsAgentProxySpanWriter({ + port: 8126 // port will always be defaulted by config + }) + + expect(writer._url.href).to.equal('http://localhost:8126/evp_proxy/v2/api/v2/llmobs') + }) +}) diff --git a/packages/dd-trace/test/llmobs/writers/spans/agentless.spec.js b/packages/dd-trace/test/llmobs/writers/spans/agentless.spec.js new file mode 100644 index 00000000000..e3cf421a3ed --- /dev/null +++ b/packages/dd-trace/test/llmobs/writers/spans/agentless.spec.js @@ -0,0 +1,21 @@ +'use stict' + +describe('LLMObsAgentlessSpanWriter', () => { + let LLMObsAgentlessSpanWriter + let writer + + beforeEach(() => { + LLMObsAgentlessSpanWriter = require('../../../../src/llmobs/writers/spans/agentless') + }) + + it('is initialized correctly', () => { + writer = new LLMObsAgentlessSpanWriter({ + site: 'datadoghq.com', + llmobs: {}, + apiKey: '1234' + }) + + expect(writer._url.href).to.equal('https://llmobs-intake.datadoghq.com/api/v2/llmobs') + expect(writer._headers['DD-API-KEY']).to.equal('1234') + }) +}) diff --git a/packages/dd-trace/test/llmobs/writers/spans/base.spec.js b/packages/dd-trace/test/llmobs/writers/spans/base.spec.js new file mode 100644 index 00000000000..1c9965cd9c2 --- /dev/null +++ b/packages/dd-trace/test/llmobs/writers/spans/base.spec.js @@ -0,0 +1,99 @@ +'use strict' + +const proxyquire = require('proxyquire') + +describe('LLMObsSpanWriter', () => { + let LLMObsSpanWriter + let writer + let options + let logger + + beforeEach(() => { + logger = { + warn: sinon.stub(), + debug: sinon.stub() + } + LLMObsSpanWriter = proxyquire('../../../../src/llmobs/writers/spans/base', { + '../../../log': logger + }) + options = { + endpoint: '/api/v2/llmobs', + intake: 'llmobs-intake.datadoghq.com' + } + }) + + afterEach(() => { + process.removeAllListeners('beforeExit') + }) + + it('is initialized correctly', () => { + writer = new LLMObsSpanWriter(options) + + expect(writer._eventType).to.equal('span') + }) + + it('computes the number of bytes of the appended event', () => { + writer = new LLMObsSpanWriter(options) + + const event = { name: 'test', value: 1 } + const eventSizeBytes = Buffer.from(JSON.stringify(event)).byteLength + + writer.append(event) + + expect(writer._bufferSize).to.equal(eventSizeBytes) + }) + + it('truncates the event if it exceeds the size limit', () => { + writer = new LLMObsSpanWriter(options) + + const event = { + name: 'test', + meta: { + input: { value: 'a'.repeat(1024 * 1024) }, + output: { value: 'a'.repeat(1024 * 1024) } + } + } + + writer.append(event) + + const bufferEvent = writer._buffer[0] + expect(bufferEvent).to.deep.equal({ + name: 'test', + meta: { + input: { value: "[This value has been dropped because this span's size exceeds the 1MB size limit.]" }, + output: { value: "[This value has been dropped because this span's size exceeds the 1MB size limit.]" } + }, + collection_errors: ['dropped_io'] + }) + }) + + it('flushes the queue if the next event will exceed the payload limit', () => { + writer = new LLMObsSpanWriter(options) + writer.flush = sinon.stub() + + writer._bufferSize = (5 << 20) - 1 + writer._buffer = Array.from({ length: 10 }) + const event = { name: 'test', value: 'a'.repeat(1024) } + + writer.append(event) + + expect(writer.flush).to.have.been.calledOnce + expect(logger.debug).to.have.been.calledWith( + 'Flusing queue because queing next event will exceed EvP payload limit' + ) + }) + + it('creates the payload correctly', () => { + writer = new LLMObsSpanWriter(options) + + const events = [ + { name: 'test', value: 1 } + ] + + const payload = writer.makePayload(events) + + expect(payload['_dd.stage']).to.equal('raw') + expect(payload.event_type).to.equal('span') + expect(payload.spans).to.deep.equal(events) + }) +}) diff --git a/packages/dd-trace/test/opentracing/propagation/text_map.spec.js b/packages/dd-trace/test/opentracing/propagation/text_map.spec.js index 58ee69047ba..5b7fef68092 100644 --- a/packages/dd-trace/test/opentracing/propagation/text_map.spec.js +++ b/packages/dd-trace/test/opentracing/propagation/text_map.spec.js @@ -58,6 +58,16 @@ describe('TextMapPropagator', () => { } }) + it('should not crash without spanContext', () => { + const carrier = {} + propagator.inject(null, carrier) + }) + + it('should not crash without carrier', () => { + const spanContext = createContext() + propagator.inject(spanContext, null) + }) + it('should inject the span context into the carrier', () => { const carrier = {} const spanContext = createContext() @@ -492,6 +502,12 @@ describe('TextMapPropagator', () => { expect(first._spanId.toString(16)).to.equal(spanId) }) + it('should not crash with invalid traceparent', () => { + textMap.traceparent = 'invalid' + + propagator.extract(textMap) + }) + it('should always extract tracestate from tracecontext when trace IDs match', () => { textMap.traceparent = '00-0000000000000000000000000000007B-0000000000000456-01' textMap.tracestate = 'other=bleh,dd=t.foo_bar_baz_:abc_!@#$%^&*()_+`-~;s:2;o:foo;t.dm:-4' diff --git a/packages/dd-trace/test/opentracing/tracer.spec.js b/packages/dd-trace/test/opentracing/tracer.spec.js index 1a6ae261f0b..31e3df79a33 100644 --- a/packages/dd-trace/test/opentracing/tracer.spec.js +++ b/packages/dd-trace/test/opentracing/tracer.spec.js @@ -245,6 +245,40 @@ describe('Tracer', () => { expect(span.addTags).to.have.been.calledWith(fields.tags) }) + it('If span is granted a service name that differs from the global service name' + + 'ensure spans `version` tag is undefined.', () => { + config.tags = { + foo: 'tracer', + bar: 'tracer' + } + + fields.tags = { + bar: 'span', + baz: 'span', + service: 'new-service' + + } + + tracer = new Tracer(config) + const testSpan = tracer.startSpan('name', fields) + + expect(span.addTags).to.have.been.calledWith(config.tags) + expect(span.addTags).to.have.been.calledWith({ ...fields.tags, version: undefined }) + expect(Span).to.have.been.calledWith(tracer, processor, prioritySampler, { + operationName: 'name', + parent: null, + tags: { + 'service.name': 'new-service' + }, + startTime: fields.startTime, + hostname: undefined, + traceId128BitGenerationEnabled: undefined, + integrationName: undefined, + links: undefined + }) + expect(testSpan).to.equal(span) + }) + it('should start a span with the trace ID generation configuration', () => { config.traceId128BitGenerationEnabled = true tracer = new Tracer(config) diff --git a/packages/dd-trace/test/plugins/externals.json b/packages/dd-trace/test/plugins/externals.json index 78373b16daa..5b00aa6061c 100644 --- a/packages/dd-trace/test/plugins/externals.json +++ b/packages/dd-trace/test/plugins/externals.json @@ -47,6 +47,18 @@ "versions": [">=3"] } ], + "body-parser": [ + { + "name": "express", + "versions": ["^4"] + } + ], + "cookie-parser": [ + { + "name": "express", + "versions": ["^4"] + } + ], "cypress": [ { "name": "cypress", @@ -65,6 +77,14 @@ { "name": "cookie-parser", "versions": [">=1.4.6"] + }, + { + "name": "request", + "versions": ["2.88.2"] + }, + { + "name": "multer", + "versions": ["^1.4.4-lts.1"] } ], "express-mongo-sanitize": [ diff --git a/packages/dd-trace/test/plugins/helpers.js b/packages/dd-trace/test/plugins/helpers.js index b35793b6664..add1361e167 100644 --- a/packages/dd-trace/test/plugins/helpers.js +++ b/packages/dd-trace/test/plugins/helpers.js @@ -117,11 +117,16 @@ function unbreakThen (promise) { } } +function getNextLineNumber () { + return Number(new Error().stack.split('\n')[2].match(/:(\d+):/)[1]) + 1 +} + module.exports = { breakThen, compare, deepInclude, expectSomeSpan, + getNextLineNumber, resolveNaming, unbreakThen, withDefaults diff --git a/packages/dd-trace/test/plugins/outbound.spec.js b/packages/dd-trace/test/plugins/outbound.spec.js index 5709c789575..2d801cd1f4c 100644 --- a/packages/dd-trace/test/plugins/outbound.spec.js +++ b/packages/dd-trace/test/plugins/outbound.spec.js @@ -3,7 +3,9 @@ require('../setup/tap') const { expect } = require('chai') +const { getNextLineNumber } = require('./helpers') const OutboundPlugin = require('../../src/plugins/outbound') +const parseTags = require('../../../datadog-core/src/utils/src/parse-tags') describe('OuboundPlugin', () => { describe('peer service decision', () => { @@ -157,4 +159,50 @@ describe('OuboundPlugin', () => { }) }) }) + + describe('code origin tags', () => { + let instance = null + + beforeEach(() => { + const tracerStub = { + _tracer: { + startSpan: sinon.stub().returns({ + addTags: sinon.spy() + }) + } + } + instance = new OutboundPlugin(tracerStub) + }) + + it('should not add exit tags to span if codeOriginForSpans.enabled is false', () => { + sinon.stub(instance, '_tracerConfig').value({ codeOriginForSpans: { enabled: false } }) + const span = instance.startSpan('test') + expect(span.addTags).to.not.have.been.called + }) + + it('should add exit tags to span if codeOriginForSpans.enabled is true', () => { + sinon.stub(instance, '_tracerConfig').value({ codeOriginForSpans: { enabled: true } }) + + const lineNumber = String(getNextLineNumber()) + const span = instance.startSpan('test') + + expect(span.addTags).to.have.been.calledOnce + const args = span.addTags.args[0] + expect(args).to.have.property('length', 1) + const tags = parseTags(args[0]) + + expect(tags).to.nested.include({ '_dd.code_origin.type': 'exit' }) + expect(tags._dd.code_origin).to.have.property('frames').to.be.an('array').with.length.above(0) + + for (const frame of tags._dd.code_origin.frames) { + expect(frame).to.have.property('file', __filename) + expect(frame).to.have.property('line').to.match(/^\d+$/) + expect(frame).to.have.property('column').to.match(/^\d+$/) + expect(frame).to.have.property('type').to.a('string') + } + + const topFrame = tags._dd.code_origin.frames[0] + expect(topFrame).to.have.property('line', lineNumber) + }) + }) }) diff --git a/packages/dd-trace/test/plugins/util/stacktrace.spec.js b/packages/dd-trace/test/plugins/util/stacktrace.spec.js index 3fefc2b29ef..a96ed87f965 100644 --- a/packages/dd-trace/test/plugins/util/stacktrace.spec.js +++ b/packages/dd-trace/test/plugins/util/stacktrace.spec.js @@ -1,6 +1,7 @@ 'use strict' const { isAbsolute } = require('path') +const { getNextLineNumber } = require('../helpers') require('../../setup/tap') @@ -62,7 +63,3 @@ describe('stacktrace utils', () => { }) }) }) - -function getNextLineNumber () { - return Number(new Error().stack.split('\n')[2].match(/:(\d+):/)[1]) + 1 -} diff --git a/packages/dd-trace/test/priority_sampler.spec.js b/packages/dd-trace/test/priority_sampler.spec.js index 5000d81ff09..88c134a5758 100644 --- a/packages/dd-trace/test/priority_sampler.spec.js +++ b/packages/dd-trace/test/priority_sampler.spec.js @@ -11,7 +11,8 @@ const { SAMPLING_MECHANISM_MANUAL, SAMPLING_MECHANISM_REMOTE_USER, SAMPLING_MECHANISM_REMOTE_DYNAMIC, - DECISION_MAKER_KEY + DECISION_MAKER_KEY, + SAMPLING_MECHANISM_APPSEC } = require('../src/constants') const SERVICE_NAME = ext.tags.SERVICE_NAME @@ -451,4 +452,61 @@ describe('PrioritySampler', () => { expect(context._sampling.mechanism).to.equal(SAMPLING_MECHANISM_AGENT) }) }) + + describe('setPriority', () => { + it('should set sampling priority and default mechanism', () => { + prioritySampler.setPriority(span, USER_KEEP) + + expect(context._sampling.priority).to.equal(USER_KEEP) + expect(context._sampling.mechanism).to.equal(SAMPLING_MECHANISM_MANUAL) + }) + + it('should set sampling priority and mechanism', () => { + prioritySampler.setPriority(span, USER_KEEP, SAMPLING_MECHANISM_APPSEC) + + expect(context._sampling.priority).to.equal(USER_KEEP) + expect(context._sampling.mechanism).to.equal(SAMPLING_MECHANISM_APPSEC) + }) + + it('should filter out invalid priorities', () => { + prioritySampler.setPriority(span, 42) + + expect(context._sampling.priority).to.be.undefined + expect(context._sampling.mechanism).to.be.undefined + }) + + it('should add decision maker tag if not set before', () => { + prioritySampler.setPriority(span, USER_KEEP, SAMPLING_MECHANISM_APPSEC) + + expect(context._trace.tags[DECISION_MAKER_KEY]).to.equal('-5') + }) + + it('should override previous priority but mantain previous decision maker tag', () => { + prioritySampler.sample(span) + + prioritySampler.setPriority(span, USER_KEEP, SAMPLING_MECHANISM_APPSEC) + + expect(context._sampling.priority).to.equal(USER_KEEP) + expect(context._sampling.mechanism).to.equal(SAMPLING_MECHANISM_APPSEC) + expect(context._trace.tags[DECISION_MAKER_KEY]).to.equal('-0') + }) + }) + + describe('keepTrace', () => { + it('should not fail if no _prioritySampler', () => { + expect(() => { + PrioritySampler.keepTrace(span, SAMPLING_MECHANISM_APPSEC) + }).to.not.throw() + }) + + it('should call setPriority with span USER_KEEP and mechanism', () => { + const setPriority = sinon.stub(prioritySampler, 'setPriority') + + span._prioritySampler = prioritySampler + + PrioritySampler.keepTrace(span, SAMPLING_MECHANISM_APPSEC) + + expect(setPriority).to.be.calledOnceWithExactly(span, USER_KEEP, SAMPLING_MECHANISM_APPSEC) + }) + }) }) diff --git a/packages/dd-trace/test/profiling/exporters/agent.spec.js b/packages/dd-trace/test/profiling/exporters/agent.spec.js index b318456eebd..8391e14d613 100644 --- a/packages/dd-trace/test/profiling/exporters/agent.spec.js +++ b/packages/dd-trace/test/profiling/exporters/agent.spec.js @@ -303,7 +303,7 @@ describe('exporters/agent', function () { /^Adding wall profile to agent export:( [0-9a-f]{2})+$/, /^Adding space profile to agent export:( [0-9a-f]{2})+$/, /^Submitting profiler agent report attempt #1 to:/i, - /^Error from the agent: HTTP Error 400$/, + /^Error from the agent: HTTP Error 500$/, /^Submitting profiler agent report attempt #2 to:/i, /^Agent export response: ([0-9a-f]{2}( |$))*/ ] @@ -344,7 +344,7 @@ describe('exporters/agent', function () { return } const data = Buffer.from(json) - res.writeHead(400, { + res.writeHead(500, { 'content-type': 'application/json', 'content-length': data.length }) @@ -356,6 +356,43 @@ describe('exporters/agent', function () { waitForResponse ]) }) + + it('should not retry on 4xx errors', async function () { + const exporter = newAgentExporter({ url, logger: { debug: () => {}, error: () => {} } }) + const start = new Date() + const end = new Date() + const tags = { foo: 'bar' } + + const [wall, space] = await Promise.all([ + createProfile(['wall', 'microseconds']), + createProfile(['space', 'bytes']) + ]) + + const profiles = { + wall, + space + } + + let tries = 0 + const json = JSON.stringify({ error: 'some error' }) + app.post('/profiling/v1/input', upload.any(), (_, res) => { + tries++ + const data = Buffer.from(json) + res.writeHead(400, { + 'content-type': 'application/json', + 'content-length': data.length + }) + res.end(data) + }) + + try { + await exporter.export({ profiles, start, end, tags }) + throw new Error('should have thrown') + } catch (err) { + expect(err.message).to.equal('HTTP Error 400') + } + expect(tries).to.equal(1) + }) }) describe('using ipv6', () => { diff --git a/packages/dd-trace/test/proxy.spec.js b/packages/dd-trace/test/proxy.spec.js index a21e2f4226a..3d7ebbc5a2a 100644 --- a/packages/dd-trace/test/proxy.spec.js +++ b/packages/dd-trace/test/proxy.spec.js @@ -131,7 +131,8 @@ describe('TracerProxy', () => { remoteConfig: { enabled: true }, - configure: sinon.spy() + configure: sinon.spy(), + llmobs: {} } Config = sinon.stub().returns(config) diff --git a/requirements.json b/requirements.json new file mode 100644 index 00000000000..85fc7c33894 --- /dev/null +++ b/requirements.json @@ -0,0 +1,85 @@ +{ + "$schema": "https://raw.githubusercontent.com/DataDog/auto_inject/refs/heads/main/preload_go/cmd/library_requirements_tester/testdata/requirements_schema.json", + "version": 1, + "native_deps": { + "glibc": [{ + "arch": "arm", + "supported": true, + "description": "From ubuntu xenial (16.04)", + "min": "2.23" + },{ + "arch": "arm64", + "supported": true, + "description": "From centOS 7", + "min": "2.17" + },{ + "arch": "x64", + "supported": true, + "description": "From centOS 7", + "min": "2.17" + },{ + "arch": "x86", + "supported": true, + "description": "From debian jessie (8)", + "min": "2.19" + }], + "musl": [{ + "arch": "arm", + "supported": true, + "description": "From alpine 3.13" + },{ + "arch": "arm64", + "supported": true, + "description": "From alpine 3.13" + },{ + "arch": "x64", + "supported": true, + "description": "From alpine 3.13" + },{ + "arch": "x86", + "supported": true, + "description": "From alpine 3.13" + }] + }, + "deny": [ + { + "id": "npm", + "description": "Ignore the npm CLI", + "os": null, + "cmds": [ + "**/node", + "**/nodejs", + "**/ts-node", + "**/ts-node-*" + ], + "args": [{ "args": ["*/npm-cli.js"], "position": 1}], + "envars": null + }, + { + "id": "yarn", + "description": "Ignore the yarn CLI", + "os": null, + "cmds": [ + "**/node", + "**/nodejs", + "**/ts-node", + "**/ts-node-*" + ], + "args": [{ "args": ["*/yarn.js"], "position": 1}], + "envars": null + }, + { + "id": "pnpm", + "description": "Ignore the pnpm CLI", + "os": null, + "cmds": [ + "**/node", + "**/nodejs", + "**/ts-node", + "**/ts-node-*" + ], + "args": [{ "args": ["*/pnpm.cjs"], "position": 1}], + "envars": null + } + ] +} diff --git a/yarn.lock b/yarn.lock index e5c88856acd..87bbc2ecc63 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10,6 +10,151 @@ "@jridgewell/gen-mapping" "^0.3.0" "@jridgewell/trace-mapping" "^0.3.9" +"@apollo/cache-control-types@^1.0.3": + version "1.0.3" + resolved "https://registry.yarnpkg.com/@apollo/cache-control-types/-/cache-control-types-1.0.3.tgz#5da62cf64c3b4419dabfef4536b57a40c8ff0b47" + integrity sha512-F17/vCp7QVwom9eG7ToauIKdAxpSoadsJnqIfyryLFSkLSOEqu+eC5Z3N8OXcUVStuOMcNHlyraRsA6rRICu4g== + +"@apollo/protobufjs@1.2.7": + version "1.2.7" + resolved "https://registry.yarnpkg.com/@apollo/protobufjs/-/protobufjs-1.2.7.tgz#3a8675512817e4a046a897e5f4f16415f16a7d8a" + integrity sha512-Lahx5zntHPZia35myYDBRuF58tlwPskwHc5CWBZC/4bMKB6siTBWwtMrkqXcsNwQiFSzSx5hKdRPUmemrEp3Gg== + dependencies: + "@protobufjs/aspromise" "^1.1.2" + "@protobufjs/base64" "^1.1.2" + "@protobufjs/codegen" "^2.0.4" + "@protobufjs/eventemitter" "^1.1.0" + "@protobufjs/fetch" "^1.1.0" + "@protobufjs/float" "^1.0.2" + "@protobufjs/inquire" "^1.1.0" + "@protobufjs/path" "^1.1.2" + "@protobufjs/pool" "^1.1.0" + "@protobufjs/utf8" "^1.1.0" + "@types/long" "^4.0.0" + long "^4.0.0" + +"@apollo/server-gateway-interface@^1.1.1": + version "1.1.1" + resolved "https://registry.yarnpkg.com/@apollo/server-gateway-interface/-/server-gateway-interface-1.1.1.tgz#a79632aa921edefcd532589943f6b97c96fa4d3c" + integrity sha512-pGwCl/po6+rxRmDMFgozKQo2pbsSwE91TpsDBAOgf74CRDPXHHtM88wbwjab0wMMZh95QfR45GGyDIdhY24bkQ== + dependencies: + "@apollo/usage-reporting-protobuf" "^4.1.1" + "@apollo/utils.fetcher" "^2.0.0" + "@apollo/utils.keyvaluecache" "^2.1.0" + "@apollo/utils.logger" "^2.0.0" + +"@apollo/server@^4.11.0": + version "4.11.0" + resolved "https://registry.yarnpkg.com/@apollo/server/-/server-4.11.0.tgz#21c0f10ad805192a5485e58ed5c5b3dbe2243174" + integrity sha512-SWDvbbs0wl2zYhKG6aGLxwTJ72xpqp0awb2lotNpfezd9VcAvzaUizzKQqocephin2uMoaA8MguoyBmgtPzNWw== + dependencies: + "@apollo/cache-control-types" "^1.0.3" + "@apollo/server-gateway-interface" "^1.1.1" + "@apollo/usage-reporting-protobuf" "^4.1.1" + "@apollo/utils.createhash" "^2.0.0" + "@apollo/utils.fetcher" "^2.0.0" + "@apollo/utils.isnodelike" "^2.0.0" + "@apollo/utils.keyvaluecache" "^2.1.0" + "@apollo/utils.logger" "^2.0.0" + "@apollo/utils.usagereporting" "^2.1.0" + "@apollo/utils.withrequired" "^2.0.0" + "@graphql-tools/schema" "^9.0.0" + "@types/express" "^4.17.13" + "@types/express-serve-static-core" "^4.17.30" + "@types/node-fetch" "^2.6.1" + async-retry "^1.2.1" + cors "^2.8.5" + express "^4.17.1" + loglevel "^1.6.8" + lru-cache "^7.10.1" + negotiator "^0.6.3" + node-abort-controller "^3.1.1" + node-fetch "^2.6.7" + uuid "^9.0.0" + whatwg-mimetype "^3.0.0" + +"@apollo/usage-reporting-protobuf@^4.1.0", "@apollo/usage-reporting-protobuf@^4.1.1": + version "4.1.1" + resolved "https://registry.yarnpkg.com/@apollo/usage-reporting-protobuf/-/usage-reporting-protobuf-4.1.1.tgz#407c3d18c7fbed7a264f3b9a3812620b93499de1" + integrity sha512-u40dIUePHaSKVshcedO7Wp+mPiZsaU6xjv9J+VyxpoU/zL6Jle+9zWeG98tr/+SZ0nZ4OXhrbb8SNr0rAPpIDA== + dependencies: + "@apollo/protobufjs" "1.2.7" + +"@apollo/utils.createhash@^2.0.0": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@apollo/utils.createhash/-/utils.createhash-2.0.1.tgz#9d982a166833ce08265ff70f8ef781d65109bdaa" + integrity sha512-fQO4/ZOP8LcXWvMNhKiee+2KuKyqIcfHrICA+M4lj/h/Lh1H10ICcUtk6N/chnEo5HXu0yejg64wshdaiFitJg== + dependencies: + "@apollo/utils.isnodelike" "^2.0.1" + sha.js "^2.4.11" + +"@apollo/utils.dropunuseddefinitions@^2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@apollo/utils.dropunuseddefinitions/-/utils.dropunuseddefinitions-2.0.1.tgz#916cd912cbd88769d3b0eab2d24f4674eeda8124" + integrity sha512-EsPIBqsSt2BwDsv8Wu76LK5R1KtsVkNoO4b0M5aK0hx+dGg9xJXuqlr7Fo34Dl+y83jmzn+UvEW+t1/GP2melA== + +"@apollo/utils.fetcher@^2.0.0": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@apollo/utils.fetcher/-/utils.fetcher-2.0.1.tgz#2f6e3edc8ce79fbe916110d9baaddad7e13d955f" + integrity sha512-jvvon885hEyWXd4H6zpWeN3tl88QcWnHp5gWF5OPF34uhvoR+DFqcNxs9vrRaBBSY3qda3Qe0bdud7tz2zGx1A== + +"@apollo/utils.isnodelike@^2.0.0", "@apollo/utils.isnodelike@^2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@apollo/utils.isnodelike/-/utils.isnodelike-2.0.1.tgz#08a7e50f08d2031122efa25af089d1c6ee609f31" + integrity sha512-w41XyepR+jBEuVpoRM715N2ZD0xMD413UiJx8w5xnAZD2ZkSJnMJBoIzauK83kJpSgNuR6ywbV29jG9NmxjK0Q== + +"@apollo/utils.keyvaluecache@^2.1.0": + version "2.1.1" + resolved "https://registry.yarnpkg.com/@apollo/utils.keyvaluecache/-/utils.keyvaluecache-2.1.1.tgz#f3f79a2f00520c6ab7a77a680a4e1fec4d19e1a6" + integrity sha512-qVo5PvUUMD8oB9oYvq4ViCjYAMWnZ5zZwEjNF37L2m1u528x5mueMlU+Cr1UinupCgdB78g+egA1G98rbJ03Vw== + dependencies: + "@apollo/utils.logger" "^2.0.1" + lru-cache "^7.14.1" + +"@apollo/utils.logger@^2.0.0", "@apollo/utils.logger@^2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@apollo/utils.logger/-/utils.logger-2.0.1.tgz#74faeb97d7ad9f22282dfb465bcb2e6873b8a625" + integrity sha512-YuplwLHaHf1oviidB7MxnCXAdHp3IqYV8n0momZ3JfLniae92eYqMIx+j5qJFX6WKJPs6q7bczmV4lXIsTu5Pg== + +"@apollo/utils.printwithreducedwhitespace@^2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@apollo/utils.printwithreducedwhitespace/-/utils.printwithreducedwhitespace-2.0.1.tgz#f4fadea0ae849af2c19c339cc5420d1ddfaa905e" + integrity sha512-9M4LUXV/fQBh8vZWlLvb/HyyhjJ77/I5ZKu+NBWV/BmYGyRmoEP9EVAy7LCVoY3t8BDcyCAGfxJaLFCSuQkPUg== + +"@apollo/utils.removealiases@2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@apollo/utils.removealiases/-/utils.removealiases-2.0.1.tgz#2873c93d72d086c60fc0d77e23d0f75e66a2598f" + integrity sha512-0joRc2HBO4u594Op1nev+mUF6yRnxoUH64xw8x3bX7n8QBDYdeYgY4tF0vJReTy+zdn2xv6fMsquATSgC722FA== + +"@apollo/utils.sortast@^2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@apollo/utils.sortast/-/utils.sortast-2.0.1.tgz#58c90bb8bd24726346b61fa51ba7fcf06e922ef7" + integrity sha512-eciIavsWpJ09za1pn37wpsCGrQNXUhM0TktnZmHwO+Zy9O4fu/WdB4+5BvVhFiZYOXvfjzJUcc+hsIV8RUOtMw== + dependencies: + lodash.sortby "^4.7.0" + +"@apollo/utils.stripsensitiveliterals@^2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@apollo/utils.stripsensitiveliterals/-/utils.stripsensitiveliterals-2.0.1.tgz#2f3350483be376a98229f90185eaf19888323132" + integrity sha512-QJs7HtzXS/JIPMKWimFnUMK7VjkGQTzqD9bKD1h3iuPAqLsxd0mUNVbkYOPTsDhUKgcvUOfOqOJWYohAKMvcSA== + +"@apollo/utils.usagereporting@^2.1.0": + version "2.1.0" + resolved "https://registry.yarnpkg.com/@apollo/utils.usagereporting/-/utils.usagereporting-2.1.0.tgz#11bca6a61fcbc6e6d812004503b38916e74313f4" + integrity sha512-LPSlBrn+S17oBy5eWkrRSGb98sWmnEzo3DPTZgp8IQc8sJe0prDgDuppGq4NeQlpoqEHz0hQeYHAOA0Z3aQsxQ== + dependencies: + "@apollo/usage-reporting-protobuf" "^4.1.0" + "@apollo/utils.dropunuseddefinitions" "^2.0.1" + "@apollo/utils.printwithreducedwhitespace" "^2.0.1" + "@apollo/utils.removealiases" "2.0.1" + "@apollo/utils.sortast" "^2.0.1" + "@apollo/utils.stripsensitiveliterals" "^2.0.1" + +"@apollo/utils.withrequired@^2.0.0": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@apollo/utils.withrequired/-/utils.withrequired-2.0.1.tgz#e72bc512582a6f26af150439f7eb7473b46ba874" + integrity sha512-YBDiuAX9i1lLc6GeTy1m7DGLFn/gMnvXqlalOIMjM7DeOgIacEjjfwPqb0M1CQ2v11HhR15d1NmxJoRCfrNqcA== + "@babel/code-frame@^7.22.13", "@babel/code-frame@^7.23.5": version "7.23.5" resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.23.5.tgz#9009b69a8c602293476ad598ff53e4562e15c244" @@ -256,10 +401,10 @@ resolved "https://registry.npmjs.org/@colors/colors/-/colors-1.5.0.tgz" integrity "sha1-u1BFecHK6SPmV2pPXaQ9Jfl729k= sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==" -"@datadog/native-appsec@8.1.1": - version "8.1.1" - resolved "https://registry.yarnpkg.com/@datadog/native-appsec/-/native-appsec-8.1.1.tgz#76aa34697e6ecbd3d9ef7e6938d3cdcfa689b1f3" - integrity sha512-mf+Ym/AzET4FeUTXOs8hz0uLOSsVIUnavZPUx8YoKWK5lKgR2L+CLfEzOpjBwgFpDgbV8I1/vyoGelgGpsMKHA== +"@datadog/native-appsec@8.2.1": + version "8.2.1" + resolved "https://registry.yarnpkg.com/@datadog/native-appsec/-/native-appsec-8.2.1.tgz#e84f9ec7e5dddea2531970117744264a685da15a" + integrity sha512-PnSlb4DC+EngEfXvZLYVBUueMnxxQV0dTpwbRQmyC6rcIFBzBCPxUl6O0hZaxCNmT1dgllpif+P1efrSi85e0Q== dependencies: node-gyp-build "^3.9.0" @@ -271,25 +416,25 @@ lru-cache "^7.14.0" node-gyp-build "^4.5.0" -"@datadog/native-iast-taint-tracking@3.1.0": - version "3.1.0" - resolved "https://registry.yarnpkg.com/@datadog/native-iast-taint-tracking/-/native-iast-taint-tracking-3.1.0.tgz#7b2ed7f8fad212d65e5ab03bcdea8b42a3051b2e" - integrity sha512-rw6qSjmxmu1yFHVvZLXFt/rVq2tUZXocNogPLB8n7MPpA0jijNGb109WokWw5ITImiW91GcGDuBW6elJDVKouQ== +"@datadog/native-iast-taint-tracking@3.2.0": + version "3.2.0" + resolved "https://registry.yarnpkg.com/@datadog/native-iast-taint-tracking/-/native-iast-taint-tracking-3.2.0.tgz#9fb6823d82f934e12c06ea1baa7399ca80deb2ec" + integrity sha512-Mc6FzCoyvU5yXLMsMS9yKnEqJMWoImAukJXolNWCTm+JQYCMf2yMsJ8pBAm7KyZKliamM9rCn7h7Tr2H3lXwjA== dependencies: node-gyp-build "^3.9.0" -"@datadog/native-metrics@^2.0.0": - version "2.0.0" - resolved "https://registry.npmjs.org/@datadog/native-metrics/-/native-metrics-2.0.0.tgz" - integrity sha512-YklGVwUtmKGYqFf1MNZuOHvTYdKuR4+Af1XkWcMD8BwOAjxmd9Z+97328rCOY8TFUJzlGUPaXzB8j2qgG/BMwA== +"@datadog/native-metrics@^3.0.1": + version "3.0.1" + resolved "https://registry.yarnpkg.com/@datadog/native-metrics/-/native-metrics-3.0.1.tgz#dc276c93785c0377a048e316f23b7c8ff3acfa84" + integrity sha512-0GuMyYyXf+Qpb/F+Fcekz58f2mO37lit9U3jMbWY/m8kac44gCPABzL5q3gWbdH+hWgqYfQoEYsdNDGSrKfwoQ== dependencies: node-addon-api "^6.1.0" node-gyp-build "^3.9.0" -"@datadog/pprof@5.3.0": - version "5.3.0" - resolved "https://registry.yarnpkg.com/@datadog/pprof/-/pprof-5.3.0.tgz#c2f58d328ecced7f99887f1a559d7fe3aecb9219" - integrity sha512-53z2Q3K92T6Pf4vz4Ezh8kfkVEvLzbnVqacZGgcbkP//q0joFzO8q00Etw1S6NdnCX0XmX08ULaF4rUI5r14mw== +"@datadog/pprof@5.4.1": + version "5.4.1" + resolved "https://registry.yarnpkg.com/@datadog/pprof/-/pprof-5.4.1.tgz#08c9bcf5d8efb2eeafdfc9f5bb5402f79fb41266" + integrity sha512-IvpL96e/cuh8ugP5O8Czdup7XQOLHeIDgM5pac5W7Lc1YzGe5zTtebKFpitvb1CPw1YY+1qFx0pWGgKP2kOfHg== dependencies: delay "^5.0.0" node-gyp-build "<4.0" @@ -444,6 +589,37 @@ resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.57.0.tgz#a5417ae8427873f1dd08b70b3574b453e67b5f7f" integrity sha512-Ys+3g2TaW7gADOJzPt83SJtCDhMjndcDMFVQ/Tj9iA1BfJzFKD9mAUXT3OenpuPHbI6P/myECxRJrofUsDx/5g== +"@graphql-tools/merge@^8.4.1": + version "8.4.2" + resolved "https://registry.yarnpkg.com/@graphql-tools/merge/-/merge-8.4.2.tgz#95778bbe26b635e8d2f60ce9856b388f11fe8288" + integrity sha512-XbrHAaj8yDuINph+sAfuq3QCZ/tKblrTLOpirK0+CAgNlZUCHs0Fa+xtMUURgwCVThLle1AF7svJCxFizygLsw== + dependencies: + "@graphql-tools/utils" "^9.2.1" + tslib "^2.4.0" + +"@graphql-tools/schema@^9.0.0": + version "9.0.19" + resolved "https://registry.yarnpkg.com/@graphql-tools/schema/-/schema-9.0.19.tgz#c4ad373b5e1b8a0cf365163435b7d236ebdd06e7" + integrity sha512-oBRPoNBtCkk0zbUsyP4GaIzCt8C0aCI4ycIRUL67KK5pOHljKLBBtGT+Jr6hkzA74C8Gco8bpZPe7aWFjiaK2w== + dependencies: + "@graphql-tools/merge" "^8.4.1" + "@graphql-tools/utils" "^9.2.1" + tslib "^2.4.0" + value-or-promise "^1.0.12" + +"@graphql-tools/utils@^9.2.1": + version "9.2.1" + resolved "https://registry.yarnpkg.com/@graphql-tools/utils/-/utils-9.2.1.tgz#1b3df0ef166cfa3eae706e3518b17d5922721c57" + integrity sha512-WUw506Ql6xzmOORlriNrD6Ugx+HjVgYxt9KCXD9mHAak+eaXSwuGGPyE60hy9xaDEoXKBsG7SkG69ybitaVl6A== + dependencies: + "@graphql-typed-document-node/core" "^3.1.1" + tslib "^2.4.0" + +"@graphql-typed-document-node/core@^3.1.1": + version "3.2.0" + resolved "https://registry.yarnpkg.com/@graphql-typed-document-node/core/-/core-3.2.0.tgz#5f3d96ec6b2354ad6d8a28bf216a1d97b5426861" + integrity sha512-mB9oAsNCm9aM3/SOv4YtBMqZbYj10R7dkq8byBqxGY/ncFwhf2oQzMV+LCRlWoDSEBJ3COiR1yeDvMtsoOsuFQ== + "@humanwhocodes/config-array@^0.11.14": version "0.11.14" resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.11.14.tgz#d78e481a039f7566ecc9660b4ea7fe6b1fec442b" @@ -660,11 +836,76 @@ resolved "https://registry.yarnpkg.com/@sinonjs/text-encoding/-/text-encoding-0.7.3.tgz#282046f03e886e352b2d5f5da5eb755e01457f3f" integrity sha512-DE427ROAphMQzU4ENbliGYrBSYPXF+TtLg9S8vzeA+OF4ZKzoDdzfL8sxuMUGS/lgRhM6j1URSk9ghf7Xo1tyA== +"@types/body-parser@*": + version "1.19.5" + resolved "https://registry.yarnpkg.com/@types/body-parser/-/body-parser-1.19.5.tgz#04ce9a3b677dc8bd681a17da1ab9835dc9d3ede4" + integrity sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg== + dependencies: + "@types/connect" "*" + "@types/node" "*" + +"@types/connect@*": + version "3.4.38" + resolved "https://registry.yarnpkg.com/@types/connect/-/connect-3.4.38.tgz#5ba7f3bc4fbbdeaff8dded952e5ff2cc53f8d858" + integrity sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug== + dependencies: + "@types/node" "*" + +"@types/express-serve-static-core@^4.17.30", "@types/express-serve-static-core@^4.17.33": + version "4.19.6" + resolved "https://registry.yarnpkg.com/@types/express-serve-static-core/-/express-serve-static-core-4.19.6.tgz#e01324c2a024ff367d92c66f48553ced0ab50267" + integrity sha512-N4LZ2xG7DatVqhCZzOGb1Yi5lMbXSZcmdLDe9EzSndPV2HpWYWzRbaerl2n27irrm94EPpprqa8KpskPT085+A== + dependencies: + "@types/node" "*" + "@types/qs" "*" + "@types/range-parser" "*" + "@types/send" "*" + +"@types/express@^4.17.13": + version "4.17.21" + resolved "https://registry.yarnpkg.com/@types/express/-/express-4.17.21.tgz#c26d4a151e60efe0084b23dc3369ebc631ed192d" + integrity sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ== + dependencies: + "@types/body-parser" "*" + "@types/express-serve-static-core" "^4.17.33" + "@types/qs" "*" + "@types/serve-static" "*" + +"@types/http-errors@*": + version "2.0.4" + resolved "https://registry.yarnpkg.com/@types/http-errors/-/http-errors-2.0.4.tgz#7eb47726c391b7345a6ec35ad7f4de469cf5ba4f" + integrity sha512-D0CFMMtydbJAegzOyHjtiKPLlvnm3iTZyZRSZoLq2mRhDdmLfIWOCYPfQJ4cu2erKghU++QvjcUjp/5h7hESpA== + "@types/json5@^0.0.29": version "0.0.29" resolved "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz" integrity sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ== +"@types/long@^4.0.0": + version "4.0.2" + resolved "https://registry.yarnpkg.com/@types/long/-/long-4.0.2.tgz#b74129719fc8d11c01868010082d483b7545591a" + integrity sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA== + +"@types/mime@^1": + version "1.3.5" + resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.5.tgz#1ef302e01cf7d2b5a0fa526790c9123bf1d06690" + integrity sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w== + +"@types/node-fetch@^2.6.1": + version "2.6.11" + resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.11.tgz#9b39b78665dae0e82a08f02f4967d62c66f95d24" + integrity sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g== + dependencies: + "@types/node" "*" + form-data "^4.0.0" + +"@types/node@*": + version "22.7.8" + resolved "https://registry.yarnpkg.com/@types/node/-/node-22.7.8.tgz#a1dbf0dc5f71bdd2642fc89caef65d58747ce825" + integrity sha512-a922jJy31vqR5sk+kAdIENJjHblqcZ4RmERviFsER4WJcEONqxKcjNOlk0q7OUfrF5sddT+vng070cdfMlrPLg== + dependencies: + undici-types "~6.19.2" + "@types/node@>=13.7.0": version "20.10.6" resolved "https://registry.yarnpkg.com/@types/node/-/node-20.10.6.tgz#a3ec84c22965802bf763da55b2394424f22bfbb5" @@ -682,6 +923,16 @@ resolved "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.5.tgz" integrity "sha1-XxnSuFqY6VWANvajysyIGUIPBc8= sha512-JCB8C6SnDoQf0cNycqd/35A7MjcnK+ZTqE7judS6o7utxUCg6imJg3QK2qzHKszlTjcj2cn+NwMB2i96ubpj7w==" +"@types/qs@*": + version "6.9.16" + resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.16.tgz#52bba125a07c0482d26747d5d4947a64daf8f794" + integrity sha512-7i+zxXdPD0T4cKDuxCUXJ4wHcsJLwENa6Z3dCu8cfCK743OGy5Nu1RmAGqDPsoTDINVEcdXKRvR/zre+P2Ku1A== + +"@types/range-parser@*": + version "1.2.7" + resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.7.tgz#50ae4353eaaddc04044279812f52c8c65857dbcb" + integrity sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ== + "@types/react@^17.0.52": version "17.0.71" resolved "https://registry.yarnpkg.com/@types/react/-/react-17.0.71.tgz#3673d446ad482b1564e44bf853b3ab5bcbc942c4" @@ -696,6 +947,23 @@ resolved "https://registry.yarnpkg.com/@types/scheduler/-/scheduler-0.16.8.tgz#ce5ace04cfeabe7ef87c0091e50752e36707deff" integrity "sha1-zlrOBM/qvn74fACR5QdS42cH3v8= sha512-WZLiwShhwLRmeV6zH+GkbOFT6Z6VklCItrDioxUnv+u4Ll+8vKeFySoFyK/0ctcRpOmwAicELfmys1sDc/Rw+A==" +"@types/send@*": + version "0.17.4" + resolved "https://registry.yarnpkg.com/@types/send/-/send-0.17.4.tgz#6619cd24e7270793702e4e6a4b958a9010cfc57a" + integrity sha512-x2EM6TJOybec7c52BX0ZspPodMsQUd5L6PRwOunVyVUhXiBSKf3AezDL8Dgvgt5o0UfKNfuA0eMLr2wLT4AiBA== + dependencies: + "@types/mime" "^1" + "@types/node" "*" + +"@types/serve-static@*": + version "1.15.7" + resolved "https://registry.yarnpkg.com/@types/serve-static/-/serve-static-1.15.7.tgz#22174bbd74fb97fe303109738e9b5c2f3064f714" + integrity sha512-W8Ym+h8nhuRwaKPaDw34QUkwsGi6Rc4yYqvKFo5rm2FUEhCFbzVWrxXUxuKK8TASjWsysJY0nsmNCGhCOIsrOw== + dependencies: + "@types/http-errors" "*" + "@types/node" "*" + "@types/send" "*" + "@types/yoga-layout@1.9.2": version "1.9.2" resolved "https://registry.npmjs.org/@types/yoga-layout/-/yoga-layout-1.9.2.tgz" @@ -916,6 +1184,13 @@ async-hook-domain@^2.0.4: resolved "https://registry.npmjs.org/async-hook-domain/-/async-hook-domain-2.0.4.tgz" integrity sha512-14LjCmlK1PK8eDtTezR6WX8TMaYNIzBIsd2D1sGoGjgx0BuNMMoSdk7i/drlbtamy0AWv9yv2tkB+ASdmeqFIw== +async-retry@^1.2.1: + version "1.3.3" + resolved "https://registry.yarnpkg.com/async-retry/-/async-retry-1.3.3.tgz#0e7f36c04d8478e7a58bdbed80cedf977785f280" + integrity sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw== + dependencies: + retry "0.13.1" + asynckit@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" @@ -1429,6 +1704,14 @@ core-util-is@~1.0.0: resolved "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz" integrity sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ== +cors@^2.8.5: + version "2.8.5" + resolved "https://registry.yarnpkg.com/cors/-/cors-2.8.5.tgz#eac11da51592dd86b9f06f6e7ac293b3df875d29" + integrity sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g== + dependencies: + object-assign "^4" + vary "^1" + cross-argv@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/cross-argv/-/cross-argv-1.0.0.tgz" @@ -2028,7 +2311,7 @@ events@1.1.1: resolved "https://registry.npmjs.org/events/-/events-1.1.1.tgz" integrity "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ= sha512-kEcvvCBByWXGnZy6JUlgAp2gBIUjfCAV6P6TgT1/aaQKcmuAEC4OZTV1I4EWQLz2gxZw76atuVyvHhTxvi0Flw==" -express@^4.18.2: +express@^4.17.1, express@^4.18.2: version "4.21.1" resolved "https://registry.yarnpkg.com/express/-/express-4.21.1.tgz#9dae5dda832f16b4eec941a4e44aa89ec481b281" integrity sha512-YSFlK1Ee0/GC8QaO91tHcDxJiE/X4FbpAyQWkxAvG6AXCuR65YzK8ua6D9hvi/TzUfZMpc+BwuM1IPw8fmQBiQ== @@ -2578,7 +2861,7 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" -inherits@2, inherits@2.0.4, inherits@^2.0.3, inherits@~2.0.3: +inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@~2.0.3: version "2.0.4" resolved "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== @@ -3115,6 +3398,16 @@ log-symbols@4.1.0: chalk "^4.1.0" is-unicode-supported "^0.1.0" +loglevel@^1.6.8: + version "1.9.2" + resolved "https://registry.yarnpkg.com/loglevel/-/loglevel-1.9.2.tgz#c2e028d6c757720107df4e64508530db6621ba08" + integrity sha512-HgMmCqIJSAKqo68l0rS2AanEWfkxaZ5wNiEFb5ggm08lDs9Xl2KxBlX3PTcaD2chBM1gXAYf491/M2Rv8Jwayg== + +long@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/long/-/long-4.0.0.tgz#9a7b71cfb7d361a194ea555241c92f7468d5bf28" + integrity sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA== + long@^5.0.0: version "5.2.0" resolved "https://registry.npmjs.org/long/-/long-5.2.0.tgz" @@ -3141,10 +3434,10 @@ lru-cache@^5.1.1: dependencies: yallist "^3.0.2" -lru-cache@^7.14.0: - version "7.14.0" - resolved "https://registry.npmjs.org/lru-cache/-/lru-cache-7.14.0.tgz" - integrity sha512-EIRtP1GrSJny0dqb50QXRUNBxHJhcpxHC++M5tD7RYbvLLn5KVWKsbyswSSqDuU15UFi3bgTQIY8nhDMeF6aDQ== +lru-cache@^7.10.1, lru-cache@^7.14.0, lru-cache@^7.14.1: + version "7.18.3" + resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-7.18.3.tgz#f793896e0fd0e954a59dfdd82f0773808df6aa89" + integrity sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA== make-dir@^3.0.0, make-dir@^3.0.2: version "3.1.0" @@ -3336,6 +3629,11 @@ negotiator@0.6.3: resolved "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz" integrity sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg== +negotiator@^0.6.3: + version "0.6.4" + resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.4.tgz#777948e2452651c570b712dd01c23e262713fff7" + integrity sha512-myRT3DiWPHqho5PrJaIRyaMv2kgYf0mUVgBNOYMuCH5Ki1yEiQaf/ZJuQ62nvpc44wL5WDbTX7yGJi1Neevw8w== + nise@^5.1.4: version "5.1.9" resolved "https://registry.yarnpkg.com/nise/-/nise-5.1.9.tgz#0cb73b5e4499d738231a473cd89bd8afbb618139" @@ -3358,11 +3656,23 @@ nock@^11.3.3: mkdirp "^0.5.0" propagate "^2.0.0" +node-abort-controller@^3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/node-abort-controller/-/node-abort-controller-3.1.1.tgz#a94377e964a9a37ac3976d848cb5c765833b8548" + integrity sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ== + node-addon-api@^6.1.0: version "6.1.0" resolved "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz" integrity sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA== +node-fetch@^2.6.7: + version "2.7.0" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" + integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== + dependencies: + whatwg-url "^5.0.0" + node-gyp-build@<4.0, node-gyp-build@^3.9.0: version "3.9.0" resolved "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-3.9.0.tgz" @@ -3423,9 +3733,9 @@ nyc@^15.1.0: test-exclude "^6.0.0" yargs "^15.0.2" -object-assign@^4.1.0, object-assign@^4.1.1: +object-assign@^4, object-assign@^4.1.0, object-assign@^4.1.1: version "4.1.1" - resolved "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz" + resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" integrity sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg== object-inspect@^1.13.1, object-inspect@^1.9.0: @@ -3951,7 +4261,7 @@ retimer@^2.0.0: resolved "https://registry.npmjs.org/retimer/-/retimer-2.0.0.tgz" integrity sha512-KLXY85WkEq2V2bKex/LOO1ViXVn2KGYe4PYysAdYdjmraYIUsVkXu8O4am+8+5UbaaGl1qho4aqAAPHNQ4GSbg== -retry@^0.13.1: +retry@0.13.1, retry@^0.13.1: version "0.13.1" resolved "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz" integrity "sha1-GFsVh6z2eRnWOzVzSeA1N7JIRlg= sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==" @@ -3990,7 +4300,7 @@ safe-array-concat@^1.1.2: has-symbols "^1.0.3" isarray "^2.0.5" -safe-buffer@5.2.1, safe-buffer@^5.1.0: +safe-buffer@5.2.1, safe-buffer@^5.0.1, safe-buffer@^5.1.0: version "5.2.1" resolved "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz" integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== @@ -4115,6 +4425,14 @@ setprototypeof@1.2.0: resolved "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz" integrity sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw== +sha.js@^2.4.11: + version "2.4.11" + resolved "https://registry.yarnpkg.com/sha.js/-/sha.js-2.4.11.tgz#37a5cf0b81ecbc6943de109ba2960d1b26584ae7" + integrity sha512-QMEp5B7cftE7APOjk5Y6xgrbWu+WkLVQwk8JNjZ8nKRciZaByEW6MubieAiToS7+dwvrjGhH8jRXz3MVd0AYqQ== + dependencies: + inherits "^2.0.1" + safe-buffer "^5.0.1" + shebang-command@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz" @@ -4466,6 +4784,11 @@ toidentifier@1.0.1: resolved "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz" integrity sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA== +tr46@~0.0.3: + version "0.0.3" + resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" + integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw== + treport@^3.0.4: version "3.0.4" resolved "https://registry.npmjs.org/treport/-/treport-3.0.4.tgz" @@ -4495,6 +4818,11 @@ tsconfig-paths@^3.15.0: minimist "^1.2.6" strip-bom "^3.0.0" +tslib@^2.4.0: + version "2.8.0" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.8.0.tgz#d124c86c3c05a40a91e6fdea4021bd31d377971b" + integrity sha512-jWVzBLplnCmoaTr13V9dYbiQ99wvZRd0vNWaDRg+aVYRcjDF3nDksxFDE/+fkXnKhpnUUkmx5pK/v8mCtLVqZA== + type-check@^0.4.0, type-check@~0.4.0: version "0.4.0" resolved "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz" @@ -4611,6 +4939,11 @@ undici-types@~5.26.4: resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617" integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA== +undici-types@~6.19.2: + version "6.19.8" + resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-6.19.8.tgz#35111c9d1437ab83a7cdc0abae2f26d88eda0a02" + integrity sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw== + unicode-length@^2.0.2: version "2.1.0" resolved "https://registry.npmjs.org/unicode-length/-/unicode-length-2.1.0.tgz" @@ -4682,11 +5015,39 @@ uuid@^8.3.2: resolved "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz" integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== -vary@~1.1.2: +uuid@^9.0.0: + version "9.0.1" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.1.tgz#e188d4c8853cc722220392c424cd637f32293f30" + integrity sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA== + +value-or-promise@^1.0.12: + version "1.0.12" + resolved "https://registry.yarnpkg.com/value-or-promise/-/value-or-promise-1.0.12.tgz#0e5abfeec70148c78460a849f6b003ea7986f15c" + integrity sha512-Z6Uz+TYwEqE7ZN50gwn+1LCVo9ZVrpxRPOhOLnncYkY1ZzOYtrX8Fwf/rFktZ8R5mJms6EZf5TqNOMeZmnPq9Q== + +vary@^1, vary@~1.1.2: version "1.1.2" resolved "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz" integrity sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg== +webidl-conversions@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" + integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ== + +whatwg-mimetype@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/whatwg-mimetype/-/whatwg-mimetype-3.0.0.tgz#5fa1a7623867ff1af6ca3dc72ad6b8a4208beba7" + integrity sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q== + +whatwg-url@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" + integrity sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw== + dependencies: + tr46 "~0.0.3" + webidl-conversions "^3.0.0" + which-boxed-primitive@^1.0.2: version "1.0.2" resolved "https://registry.npmjs.org/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz"