feat(js/plugins/ollama): Ollama embeddings (#807)

* feat: do not allow defining new actions from within other actions/flows (#725)

* refactor: [JS] introduced a generate utility action to make generate veneer logic reusable (#759)

Co-authored-by: Michael Bleigh <bleigh@google.com>

* refactor: consolidated registry into a class, made registry hierarchical (#639)

* Add DatasetStore interfaces for evals datasets (#781)

* [Fix] Apply #777 to `next` branch (#779)

* Merge main to Next (#792)

* Update index.md (#764)

* test(go): add live tests to go ollama plugin (#720)

* test(go): add ollama live test

* test(go): retrieve port info for test container

* chore(go): refactor ollama plugin live test

* test(go): remove docker from ollama live test

* fix context array (#777)

* Update to pnpm v9.7.1 (#786)

---------

Co-authored-by: Peter Friese <peter@peterfriese.de>
Co-authored-by: Jacob Cable <32874567+cabljac@users.noreply.github.com>
Co-authored-by: ssbushi <66321939+ssbushi@users.noreply.github.com>
Co-authored-by: Anthony Barone <tonybaroneee@gmail.com>

* [Eval] Breaking change -- Modify EvalRunKey (#755)

* Makes inputSchema optional for tools. (#822)

* Add datasetId field to eval run key (#827)

* Fixes history rendering of Dotprompt system role. (#837)

* Local file based DatasetStore implementation and Tools API changes (#797)

* feat(js/plugins/ollama): add initial embedding support

* feat(js/plugins/ollama): integrate ollama embeddings into plugin proper

* Update js/plugins/ollama/src/embeddings.ts

Co-authored-by: Pavel Jbanov <pavelj@google.com>

---------

Co-authored-by: Pavel Jbanov <pavelj@google.com>
Co-authored-by: Michael Bleigh <bleigh@google.com>
Co-authored-by: shrutip90 <shruti.p90@gmail.com>
Co-authored-by: ssbushi <66321939+ssbushi@users.noreply.github.com>
Co-authored-by: Sam Phillips <samphillips@google.com>
Co-authored-by: Peter Friese <peter@peterfriese.de>
Co-authored-by: Anthony Barone <tonybaroneee@gmail.com>
Co-authored-by: Michael Doyle <michaeldoyle@google.com>

docs/errors/no_new_actions_at_runtime.md

@@ -0,0 +1,22 @@
+# No new actions at runtime error
+
+Defining new actions at runtime is not allowed.
+
+✅ DO:
+
+```ts
+const prompt = defineDotprompt({...})
+
+const flow = defineFlow({...}, async (input) => {
+  await prompt.generate(...);
+})
+```
+
+❌ DON'T:
+
+```ts
+const flow = defineFlow({...}, async (input) => {
+  const prompt = defineDotprompt({...})
+  prompt.generate(...);
+})
+```

genkit-tools/cli/src/commands/eval-flow.ts

@@ -180,7 +180,7 @@ export const evalFlow = new Command('eval:flow')
 
         const evalRun = {
           key: {
-            actionId: flowName,
+            actionRef: `/flow/${flowName}`,
             evalRunId,
             createdAt: new Date().toISOString(),
           },

genkit-tools/common/src/eval/index.ts

@@ -14,7 +14,8 @@
  * limitations under the License.
  */
 
-import { EvalStore } from '../types/eval';
+import { DatasetStore, EvalStore } from '../types/eval';
+import { LocalFileDatasetStore } from './localFileDatasetStore';
 import { LocalFileEvalStore } from './localFileEvalStore';
 export { EvalFlowInput, EvalFlowInputSchema } from '../types/eval';
 export * from './exporter';
@@ -24,3 +25,7 @@ export function getEvalStore(): EvalStore {
   // TODO: This should provide EvalStore, based on tools config.
   return LocalFileEvalStore.getEvalStore();
 }
+
+export function getDatasetStore(): DatasetStore {
+  return LocalFileDatasetStore.getDatasetStore();
+}

genkit-tools/common/src/eval/localFileDatasetStore.ts

@@ -0,0 +1,222 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import crypto from 'crypto';
+import fs from 'fs';
+import { readFile, rm, writeFile } from 'fs/promises';
+import path from 'path';
+import { v4 as uuidv4 } from 'uuid';
+import { CreateDatasetRequest, UpdateDatasetRequest } from '../types/apis';
+import {
+  Dataset,
+  DatasetMetadata,
+  DatasetStore,
+  EvalFlowInputSchema,
+} from '../types/eval';
+import { logger } from '../utils/logger';
+
+/**
+ * A local, file-based DatasetStore implementation.
+ */
+export class LocalFileDatasetStore implements DatasetStore {
+  private readonly storeRoot;
+  private readonly indexFile;
+  private readonly INDEX_DELIMITER = '\n';
+  private static cachedDatasetStore: LocalFileDatasetStore | null = null;
+
+  private constructor(storeRoot: string) {
+    this.storeRoot = storeRoot;
+    this.indexFile = this.getIndexFilePath();
+    fs.mkdirSync(this.storeRoot, { recursive: true });
+    if (!fs.existsSync(this.indexFile)) {
+      fs.writeFileSync(path.resolve(this.indexFile), '');
+    }
+    logger.info(
+      `Initialized local file dataset store at root: ${this.storeRoot}`
+    );
+  }
+
+  static getDatasetStore() {
+    if (!this.cachedDatasetStore) {
+      this.cachedDatasetStore = new LocalFileDatasetStore(
+        this.generateRootPath()
+      );
+    }
+    return this.cachedDatasetStore;
+  }
+
+  static reset() {
+    this.cachedDatasetStore = null;
+  }
+
+  async createDataset(req: CreateDatasetRequest): Promise<DatasetMetadata> {
+    return this.createDatasetInternal(req.data, req.displayName);
+  }
+
+  private async createDatasetInternal(
+    data: Dataset,
+    displayName?: string
+  ): Promise<DatasetMetadata> {
+    const datasetId = this.generateDatasetId();
+    const filePath = path.resolve(
+      this.storeRoot,
+      this.generateFileName(datasetId)
+    );
+
+    if (fs.existsSync(filePath)) {
+      logger.error(`Dataset already exists at ` + filePath);
+      throw new Error(
+        `Create dataset failed: file already exists at {$filePath}`
+      );
+    }
+
+    logger.info(`Saving Dataset to ` + filePath);
+    await writeFile(filePath, JSON.stringify(data));
+
+    const now = new Date().toString();
+    const metadata = {
+      datasetId,
+      size: Array.isArray(data) ? data.length : data.samples.length,
+      version: 1,
+      displayName: displayName,
+      createTime: now,
+      updateTime: now,
+    };
+
+    let metadataMap = await this.getMetadataMap();
+    metadataMap[datasetId] = metadata;
+
+    logger.debug(
+      `Saving DatasetMetadata for ID ${datasetId} to ` +
+        path.resolve(this.indexFile)
+    );
+
+    await writeFile(path.resolve(this.indexFile), JSON.stringify(metadataMap));
+    return metadata;
+  }
+
+  async updateDataset(req: UpdateDatasetRequest): Promise<DatasetMetadata> {
+    const datasetId = req.datasetId;
+    const filePath = path.resolve(
+      this.storeRoot,
+      this.generateFileName(datasetId)
+    );
+    if (!fs.existsSync(filePath)) {
+      throw new Error(`Update dataset failed: dataset not found`);
+    }
+
+    let metadataMap = await this.getMetadataMap();
+    const prevMetadata = metadataMap[datasetId];
+    if (!prevMetadata) {
+      throw new Error(`Update dataset failed: dataset metadata not found`);
+    }
+
+    logger.info(`Updating Dataset at ` + filePath);
+    await writeFile(filePath, JSON.stringify(req.patch));
+
+    const now = new Date().toString();
+    const newMetadata = {
+      datasetId: datasetId,
+      size: Array.isArray(req.patch)
+        ? req.patch.length
+        : req.patch.samples.length,
+      version: prevMetadata.version + 1,
+      displayName: req.displayName,
+      createTime: prevMetadata.createTime,
+      updateTime: now,
+    };
+
+    logger.debug(
+      `Updating DatasetMetadata for ID ${datasetId} at ` +
+        path.resolve(this.indexFile)
+    );
+    // Replace the metadata object in the metadata map
+    metadataMap[datasetId] = newMetadata;
+    await writeFile(path.resolve(this.indexFile), JSON.stringify(metadataMap));
+
+    return newMetadata;
+  }
+
+  async getDataset(datasetId: string): Promise<Dataset> {
+    const filePath = path.resolve(
+      this.storeRoot,
+      this.generateFileName(datasetId)
+    );
+    if (!fs.existsSync(filePath)) {
+      throw new Error(`Dataset not found for dataset ID {$id}`);
+    }
+    return await readFile(filePath, 'utf8').then((data) =>
+      EvalFlowInputSchema.parse(JSON.parse(data))
+    );
+  }
+
+  async listDatasets(): Promise<DatasetMetadata[]> {
+    return this.getMetadataMap().then((metadataMap) => {
+      let metadatas = [];
+
+      for (var key in metadataMap) {
+        metadatas.push(metadataMap[key]);
+      }
+      return metadatas;
+    });
+  }
+
+  async deleteDataset(datasetId: string): Promise<void> {
+    const filePath = path.resolve(
+      this.storeRoot,
+      this.generateFileName(datasetId)
+    );
+    await rm(filePath);
+
+    let metadataMap = await this.getMetadataMap();
+    delete metadataMap[datasetId];
+
+    logger.debug(
+      `Deleting DatasetMetadata for ID ${datasetId} in ` +
+        path.resolve(this.indexFile)
+    );
+    await writeFile(path.resolve(this.indexFile), JSON.stringify(metadataMap));
+  }
+
+  private static generateRootPath(): string {
+    const rootHash = crypto
+      .createHash('md5')
+      .update(process.cwd() || 'unknown')
+      .digest('hex');
+    return path.resolve(process.cwd(), `.genkit/${rootHash}/datasets`);
+  }
+
+  private generateDatasetId(): string {
+    return uuidv4();
+  }
+
+  private generateFileName(datasetId: string): string {
+    return `${datasetId}.json`;
+  }
+
+  private getIndexFilePath(): string {
+    return path.resolve(this.storeRoot, 'index.json');
+  }
+
+  private async getMetadataMap(): Promise<any> {
+    if (!fs.existsSync(this.indexFile)) {
+      return Promise.resolve({} as any);
+    }
+    return await readFile(path.resolve(this.indexFile), 'utf8').then((data) =>
+      JSON.parse(data)
+    );
+  }
+}

genkit-tools/common/src/eval/localFileEvalStore.ts

@@ -61,10 +61,7 @@ export class LocalFileEvalStore implements EvalStore {
   }
 
   async save(evalRun: EvalRun): Promise<void> {
-    const fileName = this.generateFileName(
-      evalRun.key.evalRunId,
-      evalRun.key.actionId
-    );
+    const fileName = this.generateFileName(evalRun.key.evalRunId);
 
     logger.info(
       `Saving EvalRun ${evalRun.key.evalRunId} to ` +
@@ -85,13 +82,10 @@ export class LocalFileEvalStore implements EvalStore {
     );
   }
 
-  async load(
-    evalRunId: string,
-    actionId?: string
-  ): Promise<EvalRun | undefined> {
+  async load(evalRunId: string): Promise<EvalRun | undefined> {
     const filePath = path.resolve(
       this.storeRoot,
-      this.generateFileName(evalRunId, actionId)
+      this.generateFileName(evalRunId)
     );
     if (!fs.existsSync(filePath)) {
       return undefined;
@@ -117,8 +111,8 @@ export class LocalFileEvalStore implements EvalStore {
 
     logger.debug(`Found keys: ${JSON.stringify(keys)}`);
 
-    if (query?.filter?.actionId) {
-      keys = keys.filter((key) => key.actionId === query?.filter?.actionId);
+    if (query?.filter?.actionRef) {
+      keys = keys.filter((key) => key.actionRef === query?.filter?.actionRef);
       logger.debug(`Filtered keys: ${JSON.stringify(keys)}`);
     }
 
@@ -127,12 +121,8 @@ export class LocalFileEvalStore implements EvalStore {
     };
   }
 
-  private generateFileName(evalRunId: string, actionId?: string): string {
-    if (!actionId) {
-      return `${evalRunId}.json`;
-    }
-
-    return `${actionId?.replace('/', '_')}-${evalRunId}.json`;
+  private generateFileName(evalRunId: string): string {
+    return `${evalRunId}.json`;
   }
 
   private getIndexFilePath(): string {