Added support for returning files from custom columns

packages/evalite-tests/tests/files.test.ts

@@ -142,3 +142,45 @@ it("Should let users add files to data().input and data().expected", async () =>
     ],
   });
 });
+
+it("Should let users add files to experimental_customColumns", async () => {
+  using fixture = loadFixture("files");
+
+  const captured = captureStdout();
+
+  await runVitest({
+    cwd: fixture.dir,
+    path: undefined,
+    testOutputWritable: captured.writable,
+    mode: "run-once-and-exit",
+  });
+
+  const dir = path.join(fixture.dir, FILES_LOCATION);
+
+  const files = await readdir(dir);
+
+  expect(files).toHaveLength(1);
+
+  const filePath = path.join(dir, files[0]!);
+
+  const file = await readFile(filePath);
+
+  expect(file).toBeTruthy();
+
+  const db = createDatabase(fixture.dbLocation);
+
+  const evals = await getEvalsAsRecord(db);
+
+  expect(evals.FilesWithColumns![0]).toMatchObject({
+    results: [
+      {
+        rendered_columns: [
+          {
+            label: "Column",
+            value: EvaliteFile.fromPath(filePath),
+          },
+        ],
+      },
+    ],
+  });
+});

packages/evalite-tests/tests/fixtures/files/files-4.eval.ts

@@ -0,0 +1,27 @@
+import { evalite } from "evalite";
+import { reportTrace } from "evalite/traces";
+import { readFileSync } from "node:fs";
+import path from "node:path";
+
+evalite("FilesWithColumns", {
+  data: () => {
+    return [
+      {
+        input: "abc",
+        expected: "abcdef",
+      },
+    ];
+  },
+  task: async (input) => {
+    return input + "def";
+  },
+  scorers: [],
+  experimental_customColumns: () => {
+    return [
+      {
+        label: "Column",
+        value: readFileSync(path.join(import.meta.dirname, "test.png")),
+      },
+    ];
+  },
+});

packages/evalite/src/index.ts

@@ -144,13 +144,15 @@ export const evalite = <TInput, TExpected = TInput>(
               experimental_customColumns: opts.experimental_customColumns,
             });
 
-          const [outputWithFiles, tracesWithFiles] = await Promise.all([
-            createEvaliteFileIfNeeded({
-              rootDir,
-              input: output,
-            }),
-            handleFilesInTraces(rootDir, traces),
-          ]);
+          const [outputWithFiles, tracesWithFiles, renderedColumns] =
+            await Promise.all([
+              createEvaliteFileIfNeeded({
+                rootDir,
+                input: output,
+              }),
+              handleFilesInTraces(rootDir, traces),
+              handleFilesInColumns(rootDir, experimental_columns),
+            ]);
 
           task.meta.evalite = {
             result: {
@@ -164,7 +166,7 @@ export const evalite = <TInput, TExpected = TInput>(
               scores,
               traces: tracesWithFiles,
               status: "success",
-              renderedColumns: experimental_columns,
+              renderedColumns,
             },
             duration: Math.round(performance.now() - start),
           };
@@ -230,6 +232,24 @@ export const createScorer = <TInput, TExpected = TInput>(opts: {
 
 export * from "./evalite-file.js";
 
+const handleFilesInColumns = async (
+  rootDir: string,
+  columns: Evalite.RenderedColumn[]
+) => {
+  return await Promise.all(
+    columns.map(async (column) => {
+      const file = await createEvaliteFileIfNeeded({
+        rootDir,
+        input: column.value,
+      });
+      return {
+        ...column,
+        value: file,
+      };
+    })
+  );
+};
+
 const handleFilesInTraces = async (
   rootDir: string,
   traces: Evalite.Trace[]