Update eval stats (#19)

* Update eval stats

* Add entity path (draft)

* Bump version

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lmnr-ai/lmnr",
-  "version": "0.4.11",
+  "version": "0.4.12",
   "description": "TypeScript SDK for Laminar AI",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

src/evaluations.ts

@@ -1,7 +1,7 @@
 import { Laminar } from "./laminar";
-import { CreateEvaluationResponse, EvaluationDatapoint } from "./types";
+import { EvaluationDatapoint } from "./types";
 import cliProgress from "cli-progress";
-import { isNumber, otelTraceIdToUUID } from "./utils";
+import { isNumber, otelTraceIdToUUID, StringUUID } from "./utils";
 import { observe } from "./decorators";
 import { trace } from "@opentelemetry/api";
 import { SPAN_TYPE } from "./sdk/tracing/attributes";
@@ -176,7 +176,7 @@ class Evaluation<D, T, O> {
         const evaluation = await Laminar.createEvaluation(this.name);
         this.progressReporter.start({name: evaluation.name, projectId: evaluation.projectId, id: evaluation.id, length: this.getLength()});
         try {
-            await this.evaluateInBatches(evaluation);
+            await this.evaluateInBatches(evaluation.id);
         } catch (e) {
             await Laminar.updateEvaluationStatus(evaluation.id, 'Error');
             this.progressReporter.stopWithError(e as Error);
@@ -186,22 +186,23 @@ class Evaluation<D, T, O> {
 
         // If we update with status "Finished", we expect averageScores to be not empty
         const updatedEvaluation = await Laminar.updateEvaluationStatus(evaluation.id, 'Finished');
-        this.progressReporter.stop(updatedEvaluation.averageScores!);
+        const averageScores = updatedEvaluation.stats.averageScores;
+        this.progressReporter.stop(averageScores);
         this.isFinished = true;
 
         await Laminar.shutdown();
     }
 
     // TODO: Calculate duration of the evaluation and add it to the summary
-    public async evaluateInBatches(evaluation: CreateEvaluationResponse): Promise<void> {
+    public async evaluateInBatches(evaluationId: StringUUID): Promise<void> {
         for (let i = 0; i < this.getLength(); i += this.batchSize) {
             const batch = this.data.slice(i, i + this.batchSize);
             try {
                 const results = await this.evaluateBatch(batch);
 
                 // TODO: This must happen on the background, while the next batch is being evaluated
                 // If we do this, then we can calculate the duration of the evaluation and add it to the summary
-                await Laminar.postEvaluationResults(evaluation.id, results);
+                await Laminar.postEvaluationResults(evaluationId, results);
             } catch (e) {
                 console.error(`Error evaluating batch: ${e}`);
             } finally {

src/laminar.ts

@@ -1,7 +1,7 @@
 import { PipelineRunResponse, PipelineRunRequest, EvaluationDatapoint, EvaluationStatus, UpdateEvaluationResponse, CreateEvaluationResponse } from './types';
 import { Attributes, AttributeValue, context, isSpanContextValid, TimeInput, trace } from '@opentelemetry/api';
 import { InitializeOptions, initialize as traceloopInitialize } from './sdk/node-server-sdk'
-import { otelSpanIdToUUID, otelTraceIdToUUID } from './utils';
+import { otelSpanIdToUUID, otelTraceIdToUUID, StringUUID } from './utils';
 import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc';
 import { Metadata } from '@grpc/grpc-js';
 import { ASSOCIATION_PROPERTIES_KEY } from './sdk/tracing/tracing';
@@ -336,7 +336,7 @@ export class Laminar {
     }
 
     public static async postEvaluationResults<D, T, O>(
-        evaluationId: string,
+        evaluationId: StringUUID,
         data: EvaluationDatapoint<D, T, O>[]
     ): Promise<void> {
         const body = JSON.stringify({

src/sdk/tracing/attributes.ts

@@ -1,6 +1,7 @@
 export const SPAN_INPUT = "lmnr.span.input";
 export const SPAN_OUTPUT = "lmnr.span.output";
 export const SPAN_TYPE = "lmnr.span.type";
+export const SPAN_PATH = "lmnr.span.path";
 
 export const ASSOCIATION_PROPERTIES = "lmnr.association.properties";
 export const SESSION_ID = "lmnr.association.properties.session_id";

src/sdk/tracing/decorators.ts

@@ -2,7 +2,9 @@ import { Span, context } from "@opentelemetry/api";
 import { suppressTracing } from "@opentelemetry/core";
 import {
   ASSOCIATION_PROPERTIES_KEY,
+  getSpanPath,
   getTracer,
+  SPAN_PATH_KEY,
 } from "./tracing";
 import { shouldSendTraces } from ".";
 import { Telemetry } from "../telemetry/telemetry";
@@ -40,6 +42,10 @@ export function withEntity<
     );
   }
 
+  const currentSpanPath = getSpanPath(entityContext);
+  const spanPath = currentSpanPath ? `${currentSpanPath}.${name}` : name;
+  entityContext = entityContext.setValue(SPAN_PATH_KEY, spanPath);
+
   if (shouldSuppressTracing) {
     entityContext = suppressTracing(entityContext);
   }

src/sdk/tracing/index.ts

@@ -6,6 +6,7 @@ import { Instrumentation } from "@opentelemetry/instrumentation";
 import { InitializeOptions } from "../interfaces";
 import {
   ASSOCIATION_PROPERTIES_KEY,
+  SPAN_PATH_KEY,
 } from "./tracing";
 import { Telemetry } from "../telemetry/telemetry";
 import { _configuration } from "../configuration";
@@ -25,7 +26,7 @@ import { PineconeInstrumentation } from "@traceloop/instrumentation-pinecone";
 import { LangChainInstrumentation } from "@traceloop/instrumentation-langchain";
 import { ChromaDBInstrumentation } from "@traceloop/instrumentation-chromadb";
 import { QdrantInstrumentation } from "@traceloop/instrumentation-qdrant";
-import { ASSOCIATION_PROPERTIES } from "./attributes";
+import { ASSOCIATION_PROPERTIES, SPAN_PATH } from "./attributes";
 
 let _spanProcessor: SimpleSpanProcessor | BatchSpanProcessor;
 let openAIInstrumentation: OpenAIInstrumentation | undefined;
@@ -254,6 +255,11 @@ export const startTracing = (options: InitializeOptions) => {
     : new BatchSpanProcessor(traceExporter);
 
   _spanProcessor.onStart = (span: Span) => {
+    const spanPath = context.active().getValue(SPAN_PATH_KEY);
+    if (spanPath) {
+      span.setAttribute(SPAN_PATH, spanPath as string);
+    }
+
     // This sets the properties only if the context has them
     const associationProperties = context
       .active()

src/sdk/tracing/tracing.ts

@@ -2,8 +2,7 @@ import { trace, createContextKey } from "@opentelemetry/api";
 import { Context } from "@opentelemetry/api/build/src/context/types";
 
 const TRACER_NAME = "lmnr.tracer";
-export const WORKFLOW_NAME_KEY = createContextKey("workflow_name");
-export const ENTITY_NAME_KEY = createContextKey("entity_name");
+export const SPAN_PATH_KEY = createContextKey("span_path");
 export const ASSOCIATION_PROPERTIES_KEY = createContextKey(
   "association_properties",
 );
@@ -12,8 +11,8 @@ export const getTracer = () => {
   return trace.getTracer(TRACER_NAME);
 };
 
-export const getEntityPath = (entityContext: Context): string | undefined => {
-  const path = entityContext.getValue(ENTITY_NAME_KEY);
+export const getSpanPath = (entityContext: Context): string | undefined => {
+  const path = entityContext.getValue(SPAN_PATH_KEY);
 
   return path ? `${path}` : undefined;
 };

src/types.ts

@@ -44,10 +44,15 @@ export type CreateEvaluationResponse = {
     status: EvaluationStatus
     projectId: StringUUID,
     metadata: Record<string, any> | null,
-    averageScores: Record<string, number> | null,
 }
 
-export type UpdateEvaluationResponse = CreateEvaluationResponse;
+type EvaluationStats = {
+    averageScores: Record<string, number>;
+}
+
+export type UpdateEvaluationResponse = {
+    stats: EvaluationStats;
+};
 
 export type EvaluationDatapoint<D, T, O> = {
     data: Record<string, any> & D;