add file runner to the CLI

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lmnr-ai/lmnr",
-  "version": "0.4.26",
+  "version": "0.4.27",
   "description": "TypeScript SDK for Laminar AI",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -77,6 +77,7 @@
     "argparse": "^2.0.1",
     "cli-progress": "^3.12.0",
     "esbuild": "^0.24.0",
+    "glob": "^11.0.0",
     "uuid": "^11.0.3"
   }
 }

src/cli.ts

@@ -2,6 +2,7 @@
 
 import { ArgumentParser } from "argparse";
 import * as esbuild from "esbuild";
+import * as glob from "glob";
 
 const pjson = require('../package.json');
 
@@ -32,7 +33,7 @@ async function cli() {
   // Use argparse, which is the port of the python library
   const parser = new ArgumentParser({
     prog: "lmnr",
-    description: "CLI for Laminar",
+    description: "CLI for Laminar. Use `lmnr <subcommand> --help` for more information.",
   });
 
   parser.add_argument("-v", "--version", { action: "version", version: pjson.version });
@@ -42,42 +43,88 @@ async function cli() {
     dest: "subcommand",
   });
 
-  const parser_eval = subparsers.add_parser("eval", {
+  const parserEval = subparsers.add_parser("eval", {
     description: "Run an evaluation",
+    help: "Run an evaluation",
   });
 
-  parser_eval.add_argument("file", {
-    help: "A file containing the evaluation to run",
+  parserEval.add_argument("file", {
+    help: "A file containing the evaluation to run. If no file is provided, " +
+      "the evaluation will run all `*.eval.ts|js` files in the `evals` directory.",
+    nargs: "?",
   });
 
-  parser_eval.set_defaults({
+  parserEval.add_argument("--fail-on-error", {
+    help: "Fail on error. If specified, will fail if encounters a file that cannot be run",
+    action: "store_true",
+  });
+
+  parserEval.set_defaults({
     func: async (args: any) => {
-      // TODO: Add various optimizations, e.g. minify, pure, tree shaking, etc.
-      const buildOptions = {
-        entryPoints: [args.file],
-        outfile: "tmp_out.js",
-        write: false,  // will be loaded in memory as a temp file
-        platform: "node" as esbuild.Platform,
-        bundle: true,
-        external: ["node_modules/*"],
-      };
-
-      const result = await esbuild.build(buildOptions);
-
-      if (!result.outputFiles) {
-        console.error("Error when building: No output files found");
+      const files = args.file
+        ? [args.file]
+        : glob.sync('evals/**/*.eval.{ts,js}')
+
+      files.sort();
+
+      if (files.length === 0) {
+        console.error("No evaluation files found. Please provide a file or " +
+          "ensure there are files in the `evals` directory.");
         process.exit(1);
       }
 
-      const outputFileText = result.outputFiles[0].text;
+      if (!args.file) {
+        console.log(`Located ${files.length} evaluation files in evals/`);
+      }
 
-      const evaluation = loadModule({
-        filename: args.file,
-        moduleText: outputFileText,
-      });
+      for (const file of files) {
+        console.log(`Loading ${file}...`);
+        // TODO: Add various optimizations, e.g. minify, pure, tree shaking, etc.
+        const buildOptions = {
+          entryPoints: [file],
+          outfile: `tmp_out_${file}.js`,
+          write: false,  // will be loaded in memory as a temp file
+          platform: "node" as esbuild.Platform,
+          bundle: true,
+          external: ["node_modules/*"],
+        };
+
+        const result = await esbuild.build(buildOptions);
+
+        if (!result.outputFiles) {
+          console.error("Error when building: No output files found");
+          if (args.fail_on_error) {
+            process.exit(1);
+          }
+          continue;
+        }
+
+        const outputFileText = result.outputFiles[0].text;
+
+        const evaluation = loadModule({
+          filename: args.file,
+          moduleText: outputFileText,
+        });
+
+        // @ts-ignore
+        if (!evaluation?.run) {
+          console.error(`Evaluation ${file} does not properly call evaluate()`);
+          if (args.fail_on_error) {
+            process.exit(1);
+          }
+          continue;
+        }
+
+        // @ts-ignore
+        await evaluation.run();
+      }
+    }
+  });
 
-      // @ts-ignore
-      await evaluation.run();
+  parser.set_defaults({
+    func: async (args: any) => {
+      parser.print_help();
+      process.exit(0);
     }
   });
 

src/laminar.ts

@@ -17,7 +17,7 @@ import {
   trace,
   TraceFlags
 } from '@opentelemetry/api';
-import { InitializeOptions, initialize as traceloopInitialize } from './sdk/node-server-sdk'
+import { InitializeOptions, initializeTracing } from './sdk/node-server-sdk'
 import { isStringUUID, otelSpanIdToUUID, otelTraceIdToUUID, uuidToOtelTraceId } from './utils';
 import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc';
 import { Metadata } from '@grpc/grpc-js';
@@ -113,7 +113,6 @@ export class Laminar {
     grpcPort,
     instrumentModules,
     useExternalTracerProvider,
-    _spanProcessor,
   }: LaminarInitializeProps) {
 
     let key = projectApiKey ?? process.env.LMNR_PROJECT_API_KEY;
@@ -143,20 +142,13 @@ export class Laminar {
       metadata,
     });
 
-    if (_spanProcessor) {
-      console.warn("Laminar using a custom span processor. This feature is " +
-        "added for tests only. Any use of this feature outside of tests " +
-        "is not supported and advised against."
-      );
-    }
 
-    traceloopInitialize({
+    initializeTracing({
       exporter,
       silenceInitializationMessage: true,
       instrumentModules,
       disableBatch: false,
       useExternalTracerProvider,
-      processor: _spanProcessor,
     });
   }
 

src/sdk/configuration/index.ts

@@ -12,7 +12,7 @@ export let _configuration: InitializeOptions | undefined;
  * @param options - The options to initialize the SDK. See the {@link InitializeOptions} for details.
  * @throws {InitializationError} if the configuration is invalid or if failed to fetch feature data.
  */
-export const initialize = (options: InitializeOptions) => {
+export let initializeTracing = (options: InitializeOptions) => {
   if (_configuration) {
     return;
   }

src/sdk/node-server-sdk.ts

@@ -1,5 +1,5 @@
 export * from "./errors";
 export { InitializeOptions } from "./interfaces";
-export { initialize } from "./configuration";
+export { initializeTracing } from "./configuration";
 export { forceFlush } from "./tracing";
 export * from "./tracing/decorators";

test/tracing.test.ts

@@ -1,16 +1,31 @@
-import { afterEach, describe, it } from "node:test";
-import { InMemorySpanExporter, SimpleSpanProcessor } from "@opentelemetry/sdk-trace-base";
-import { Laminar, LaminarAttributes, observe, TracingLevel, withLabels, withTracingLevel } from "../src/index";
-import assert from "node:assert";
-
-const exporter = new InMemorySpanExporter();
-const processor = new SimpleSpanProcessor(exporter);
-Laminar.initialize({
-  projectApiKey: "test",
-  _spanProcessor: processor,
-});
+import { afterEach, beforeEach, describe, it, mock } from "node:test";
+import {
+  InMemorySpanExporter,
+  SimpleSpanProcessor,
+} from "@opentelemetry/sdk-trace-base";
+import {
+  Laminar,
+  LaminarAttributes,
+  observe,
+  TracingLevel,
+  withLabels,
+  withTracingLevel,
+} from "../src/index";
+import assert from "node:assert/strict";
+import { initializeTracing } from "../src/sdk/configuration";
 
 describe("tracing", () => {
+  const exporter = new InMemorySpanExporter();
+  const processor = new SimpleSpanProcessor(exporter);
+
+  beforeEach(() => {
+    // This only uses underlying OpenLLMetry initialization, not Laminar's
+    // initialization, but this is sufficient for testing.
+    // Laminar.initialize() is tested in the other suite.
+    initializeTracing({ processor, exporter });
+  });
+
+
   afterEach(() => {
     exporter.reset();
   });