From b14cb7c0d47150e573bf325f19c2f8e7d175420f Mon Sep 17 00:00:00 2001 From: Din Date: Tue, 17 Dec 2024 10:36:22 -0800 Subject: [PATCH] add file runner to the CLI --- package.json | 3 +- pnpm-lock.yaml | 48 +++++++++++++++++ src/cli.ts | 99 +++++++++++++++++++++++++--------- src/laminar.ts | 12 +---- src/sdk/configuration/index.ts | 2 +- src/sdk/node-server-sdk.ts | 2 +- test/tracing.test.ts | 37 +++++++++---- 7 files changed, 153 insertions(+), 50 deletions(-) diff --git a/package.json b/package.json index cd4800c..b4f2044 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@lmnr-ai/lmnr", - "version": "0.4.26", + "version": "0.4.27", "description": "TypeScript SDK for Laminar AI", "main": "dist/index.js", "types": "dist/index.d.ts", @@ -77,6 +77,7 @@ "argparse": "^2.0.1", "cli-progress": "^3.12.0", "esbuild": "^0.24.0", + "glob": "^11.0.0", "uuid": "^11.0.3" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 94ea959..cb887d4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -83,6 +83,9 @@ importers: esbuild: specifier: ^0.24.0 version: 0.24.0 + glob: + specifier: ^11.0.0 + version: 11.0.0 uuid: specifier: ^11.0.3 version: 11.0.3 @@ -2120,6 +2123,11 @@ packages: resolution: {integrity: sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==} hasBin: true + glob@11.0.0: + resolution: {integrity: sha512-9UiX/Bl6J2yaBbxKoEBRm4Cipxgok8kQYcOPEhScPwebu2I0HoQOuYdIO6S3hLuWoZgpDpwQZMzTFxgpkyT76g==} + engines: {node: 20 || >=22} + hasBin: true + glob@7.2.3: resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==} deprecated: Glob versions prior to v9 are no longer supported @@ -2263,6 +2271,10 @@ packages: jackspeak@3.4.3: resolution: {integrity: sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==} + jackspeak@4.0.2: + resolution: {integrity: sha512-bZsjR/iRjl1Nk1UkjGpAzLNfQtzuijhn2g+pbZb98HQ1Gk8vM9hfbxeMBP+M2/UUdwj0RqGG3mlvk2MsAqwvEw==} + engines: {node: 20 || >=22} + joycon@3.1.1: resolution: {integrity: sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==} engines: {node: '>=10'} @@ -2451,6 +2463,10 @@ packages: lru-cache@10.4.3: resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} + lru-cache@11.0.2: + resolution: {integrity: sha512-123qHRfJBmo2jXDbo/a5YOQrJoHF/GNQTLzQ5+IdK5pWpceK17yRc6ozlWd25FxvGKQbIUs91fDFkXmDHTKcyA==} + engines: {node: 20 || >=22} + lru-cache@9.1.2: resolution: {integrity: sha512-ERJq3FOzJTxBbFjZ7iDs+NiK4VI9Wz+RdrrAB8dio1oV+YvdPzUEE4QNiT2VD51DkIbCYRUUzCRkssXCHqSnKQ==} engines: {node: 14 || >=16.14} @@ -2489,6 +2505,10 @@ packages: resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} engines: {node: '>=10'} + minimatch@10.0.1: + resolution: {integrity: sha512-ethXTt3SGGR+95gudmqJ1eNhRO7eGEGIgYA9vnPatK4/etz2MEVDno5GMCibdMTuBMyElzIlgxMna3K94XDIDQ==} + engines: {node: 20 || >=22} + minimatch@3.1.2: resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==} @@ -2718,6 +2738,10 @@ packages: resolution: {integrity: sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==} engines: {node: '>=16 || 14 >=14.18'} + path-scurry@2.0.0: + resolution: {integrity: sha512-ypGJsmGtdXUOeM5u93TyeIEfEhM6s+ljAhrk5vAvSx8uyY/02OvrZnA0YNGUrPXfpJMgI1ODd3nwz8Npx4O4cg==} + engines: {node: 20 || >=22} + pathe@1.1.2: resolution: {integrity: sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ==} @@ -6123,6 +6147,15 @@ snapshots: package-json-from-dist: 1.0.1 path-scurry: 1.11.1 + glob@11.0.0: + dependencies: + foreground-child: 3.3.0 + jackspeak: 4.0.2 + minimatch: 10.0.1 + minipass: 7.1.2 + package-json-from-dist: 1.0.1 + path-scurry: 2.0.0 + glob@7.2.3: dependencies: fs.realpath: 1.0.0 @@ -6317,6 +6350,10 @@ snapshots: optionalDependencies: '@pkgjs/parseargs': 0.11.0 + jackspeak@4.0.2: + dependencies: + '@isaacs/cliui': 8.0.2 + joycon@3.1.1: {} js-base64@3.7.2: {} @@ -6553,6 +6590,8 @@ snapshots: lru-cache@10.4.3: {} + lru-cache@11.0.2: {} + lru-cache@9.1.2: {} magic-bytes.js@1.10.0: {} @@ -6590,6 +6629,10 @@ snapshots: mimic-response@3.1.0: {} + minimatch@10.0.1: + dependencies: + brace-expansion: 2.0.1 + minimatch@3.1.2: dependencies: brace-expansion: 1.1.11 @@ -6803,6 +6846,11 @@ snapshots: lru-cache: 10.4.3 minipass: 7.1.2 + path-scurry@2.0.0: + dependencies: + lru-cache: 11.0.2 + minipass: 7.1.2 + pathe@1.1.2: {} pg-int8@1.0.1: {} diff --git a/src/cli.ts b/src/cli.ts index e220bcb..95c6239 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -2,6 +2,7 @@ import { ArgumentParser } from "argparse"; import * as esbuild from "esbuild"; +import * as glob from "glob"; const pjson = require('../package.json'); @@ -32,7 +33,7 @@ async function cli() { // Use argparse, which is the port of the python library const parser = new ArgumentParser({ prog: "lmnr", - description: "CLI for Laminar", + description: "CLI for Laminar. Use `lmnr --help` for more information.", }); parser.add_argument("-v", "--version", { action: "version", version: pjson.version }); @@ -42,42 +43,88 @@ async function cli() { dest: "subcommand", }); - const parser_eval = subparsers.add_parser("eval", { + const parserEval = subparsers.add_parser("eval", { description: "Run an evaluation", + help: "Run an evaluation", }); - parser_eval.add_argument("file", { - help: "A file containing the evaluation to run", + parserEval.add_argument("file", { + help: "A file containing the evaluation to run. If no file is provided, " + + "the evaluation will run all `*.eval.ts|js` files in the `evals` directory.", + nargs: "?", }); - parser_eval.set_defaults({ + parserEval.add_argument("--fail-on-error", { + help: "Fail on error. If specified, will fail if encounters a file that cannot be run", + action: "store_true", + }); + + parserEval.set_defaults({ func: async (args: any) => { - // TODO: Add various optimizations, e.g. minify, pure, tree shaking, etc. - const buildOptions = { - entryPoints: [args.file], - outfile: "tmp_out.js", - write: false, // will be loaded in memory as a temp file - platform: "node" as esbuild.Platform, - bundle: true, - external: ["node_modules/*"], - }; - - const result = await esbuild.build(buildOptions); - - if (!result.outputFiles) { - console.error("Error when building: No output files found"); + const files = args.file + ? [args.file] + : glob.sync('evals/**/*.eval.{ts,js}') + + files.sort(); + + if (files.length === 0) { + console.error("No evaluation files found. Please provide a file or " + + "ensure there are files in the `evals` directory."); process.exit(1); } - const outputFileText = result.outputFiles[0].text; + if (!args.file) { + console.log(`Located ${files.length} evaluation files in evals/`); + } - const evaluation = loadModule({ - filename: args.file, - moduleText: outputFileText, - }); + for (const file of files) { + console.log(`Loading ${file}...`); + // TODO: Add various optimizations, e.g. minify, pure, tree shaking, etc. + const buildOptions = { + entryPoints: [file], + outfile: `tmp_out_${file}.js`, + write: false, // will be loaded in memory as a temp file + platform: "node" as esbuild.Platform, + bundle: true, + external: ["node_modules/*"], + }; + + const result = await esbuild.build(buildOptions); + + if (!result.outputFiles) { + console.error("Error when building: No output files found"); + if (args.fail_on_error) { + process.exit(1); + } + continue; + } + + const outputFileText = result.outputFiles[0].text; + + const evaluation = loadModule({ + filename: args.file, + moduleText: outputFileText, + }); + + // @ts-ignore + if (!evaluation?.run) { + console.error(`Evaluation ${file} does not properly call evaluate()`); + if (args.fail_on_error) { + process.exit(1); + } + continue; + } + + // @ts-ignore + await evaluation.run(); + } + } + }); - // @ts-ignore - await evaluation.run(); + parser.set_defaults({ + func: async (args: any) => { + parser.print_help(); + process.exit(0); } }); diff --git a/src/laminar.ts b/src/laminar.ts index a173805..7b5a99e 100644 --- a/src/laminar.ts +++ b/src/laminar.ts @@ -17,7 +17,7 @@ import { trace, TraceFlags } from '@opentelemetry/api'; -import { InitializeOptions, initialize as traceloopInitialize } from './sdk/node-server-sdk' +import { InitializeOptions, initializeTracing } from './sdk/node-server-sdk' import { isStringUUID, otelSpanIdToUUID, otelTraceIdToUUID, uuidToOtelTraceId } from './utils'; import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc'; import { Metadata } from '@grpc/grpc-js'; @@ -113,7 +113,6 @@ export class Laminar { grpcPort, instrumentModules, useExternalTracerProvider, - _spanProcessor, }: LaminarInitializeProps) { let key = projectApiKey ?? process.env.LMNR_PROJECT_API_KEY; @@ -143,20 +142,13 @@ export class Laminar { metadata, }); - if (_spanProcessor) { - console.warn("Laminar using a custom span processor. This feature is " + - "added for tests only. Any use of this feature outside of tests " + - "is not supported and advised against." - ); - } - traceloopInitialize({ + initializeTracing({ exporter, silenceInitializationMessage: true, instrumentModules, disableBatch: false, useExternalTracerProvider, - processor: _spanProcessor, }); } diff --git a/src/sdk/configuration/index.ts b/src/sdk/configuration/index.ts index 0cf2bf9..26db08c 100644 --- a/src/sdk/configuration/index.ts +++ b/src/sdk/configuration/index.ts @@ -12,7 +12,7 @@ export let _configuration: InitializeOptions | undefined; * @param options - The options to initialize the SDK. See the {@link InitializeOptions} for details. * @throws {InitializationError} if the configuration is invalid or if failed to fetch feature data. */ -export const initialize = (options: InitializeOptions) => { +export let initializeTracing = (options: InitializeOptions) => { if (_configuration) { return; } diff --git a/src/sdk/node-server-sdk.ts b/src/sdk/node-server-sdk.ts index 78edfbc..7190385 100644 --- a/src/sdk/node-server-sdk.ts +++ b/src/sdk/node-server-sdk.ts @@ -1,5 +1,5 @@ export * from "./errors"; export { InitializeOptions } from "./interfaces"; -export { initialize } from "./configuration"; +export { initializeTracing } from "./configuration"; export { forceFlush } from "./tracing"; export * from "./tracing/decorators"; diff --git a/test/tracing.test.ts b/test/tracing.test.ts index 11a2669..f9cf726 100644 --- a/test/tracing.test.ts +++ b/test/tracing.test.ts @@ -1,16 +1,31 @@ -import { afterEach, describe, it } from "node:test"; -import { InMemorySpanExporter, SimpleSpanProcessor } from "@opentelemetry/sdk-trace-base"; -import { Laminar, LaminarAttributes, observe, TracingLevel, withLabels, withTracingLevel } from "../src/index"; -import assert from "node:assert"; - -const exporter = new InMemorySpanExporter(); -const processor = new SimpleSpanProcessor(exporter); -Laminar.initialize({ - projectApiKey: "test", - _spanProcessor: processor, -}); +import { afterEach, beforeEach, describe, it, mock } from "node:test"; +import { + InMemorySpanExporter, + SimpleSpanProcessor, +} from "@opentelemetry/sdk-trace-base"; +import { + Laminar, + LaminarAttributes, + observe, + TracingLevel, + withLabels, + withTracingLevel, +} from "../src/index"; +import assert from "node:assert/strict"; +import { initializeTracing } from "../src/sdk/configuration"; describe("tracing", () => { + const exporter = new InMemorySpanExporter(); + const processor = new SimpleSpanProcessor(exporter); + + beforeEach(() => { + // This only uses underlying OpenLLMetry initialization, not Laminar's + // initialization, but this is sufficient for testing. + // Laminar.initialize() is tested in the other suite. + initializeTracing({ processor, exporter }); + }); + + afterEach(() => { exporter.reset(); });