Add experiment URL, polish

langchain-ai · Jan 13, 2025 · 54d3825 · 54d3825
1 parent e1824bb
commit 54d3825
Show file tree

Hide file tree

Showing 4 changed files with 27 additions and 8 deletions.
diff --git a/js/src/jest/reporter.ts b/js/src/jest/reporter.ts
@@ -5,14 +5,14 @@ import { printReporterTable } from "../utils/jestlike/reporter.js";
 
 class LangSmithEvalReporter extends DefaultReporter {
   async onTestResult(test: any, testResult: any, aggregatedResults: any) {
+    super.onTestResult(test, testResult, aggregatedResults);
     try {
       await printReporterTable(
         testResult.testResults,
         testResult.failureMessage
       );
     } catch (e: any) {
       console.log("Failed to display LangSmith eval results:", e.message);
-      return super.onTestResult(test, testResult, aggregatedResults);
     }
   }
 }

diff --git a/js/src/tests/jestlike/vitest.eval.ts b/js/src/tests/jestlike/vitest.eval.ts
@@ -124,7 +124,7 @@ ls.describe(
       return res;
     });
 
-    test("Should test absolute closeness custom matcher", async () => {
+    test("Absolute closeness custom matcher", async () => {
       await ls.expect("foobar").toBeAbsoluteCloseTo("foobaz", {
         threshold: 3,
       });
@@ -136,7 +136,7 @@ ls.describe(
       });
     });
 
-    test("Should test relative closeness custom matcher", async () => {
+    test("Relative closeness custom matcher", async () => {
       await ls.expect("0123456789").toBeRelativeCloseTo("1123456789", {
         threshold: 0.1,
       });

diff --git a/js/src/utils/jestlike/index.ts b/js/src/utils/jestlike/index.ts
@@ -241,10 +241,18 @@ export function generateWrapperFromJestlikeMethods(
         dataset.id,
         projectConfig
       );
+      const datasetUrl = await testClient.getDatasetUrl({
+        datasetId: dataset.id,
+      });
+      const experimentUrl = `${datasetUrl}/compare?selectedSessions=${project.id}`;
+      console.log(
+        `[LANGSMITH]: Experiment starting! View results at ${experimentUrl}`
+      );
       storageValue = {
         dataset,
         project,
         client: testClient,
+        experimentUrl,
       };
     }
     return storageValue;
@@ -274,7 +282,8 @@ export function generateWrapperFromJestlikeMethods(
         };
 
         beforeAll(async () => {
-          datasetSetupInfo.set(suiteUuid, await runDatasetSetup(context));
+          const storageValue = await runDatasetSetup(context);
+          datasetSetupInfo.set(suiteUuid, storageValue);
         });
 
         afterAll(async () => {
@@ -339,7 +348,7 @@ export function generateWrapperFromJestlikeMethods(
           `${testCounter}.json`
         );
         void method(
-          `${testCounter}: ${name}${totalRuns > 1 ? `, iteration ${i}` : ""}`,
+          `${testCounter}: ${name}${totalRuns > 1 ? `, run ${i}` : ""}`,
           async () => {
             if (context === undefined) {
               throw new Error(
@@ -351,7 +360,7 @@ export function generateWrapperFromJestlikeMethods(
                 "Dataset failed to initialize. Please check your LangSmith environment variables."
               );
             }
-            const { dataset, createdAt, project, client } =
+            const { dataset, createdAt, project, client, experimentUrl } =
               datasetSetupInfo.get(context.suiteUuid);
             const testInput: I = inputs;
             const testOutput: O = expected;
@@ -532,6 +541,7 @@ export function generateWrapperFromJestlikeMethods(
                 expected,
                 outputs: loggedOutput,
                 feedback: testFeedback,
+                experimentUrl,
               })
             );
           },

diff --git a/js/src/utils/jestlike/reporter.ts b/js/src/utils/jestlike/reporter.ts
@@ -66,6 +66,7 @@ export async function printReporterTable(
 ) {
   const rows = [];
   const feedbackKeys = new Set<string>();
+  let experimentUrl;
   for (const result of results) {
     const { title, duration, status } = result;
     const titleComponents = title.split(":");
@@ -123,6 +124,7 @@ export async function printReporterTable(
         },
         {}
       );
+      experimentUrl = experimentUrl ?? fileContent.experimentUrl;
       rows.push([
         {
           Name: formatTestName(testName, duration),
@@ -172,10 +174,10 @@ export async function printReporterTable(
     }
   }
 
-  console.log("");
+  console.log();
   const table = new Table({
     columns: [
-      { name: "Name", alignment: "left" },
+      { name: "Name", alignment: "left", maxLen: 48 },
       { name: "Result", alignment: "left" },
       { name: "Inputs", alignment: "left" },
       { name: "Expected", alignment: "left" },
@@ -192,4 +194,11 @@ export async function printReporterTable(
     console.log(failureMessage);
   }
   table.printTable();
+  if (experimentUrl) {
+    console.log();
+    console.log(
+      ` [LANGSMITH]: View full results in LangSmith at ${experimentUrl}`
+    );
+    console.log();
+  }
 }