diff --git a/packages/core/package.json b/packages/core/package.json index 5731c20..2980505 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -132,6 +132,7 @@ }, "peerDependencies": { "@supabase/supabase-js": "^2.26.0", + "chromadb": "^1.5.3", "langchain": "^0.0.96", "ofetch": "^1.1.1" }, @@ -139,6 +140,9 @@ "@supabase/supabase-js": { "optional": true }, + "chromadb": { + "optional": true + }, "langchain": { "optional": true }, diff --git a/packages/core/src/storage/chroma/index.ts b/packages/core/src/storage/chroma/index.ts new file mode 100644 index 0000000..e6c110a --- /dev/null +++ b/packages/core/src/storage/chroma/index.ts @@ -0,0 +1,64 @@ +import { Chroma } from 'langchain/vectorstores/chroma' +import { Document } from 'langchain/document' +import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter' +import type { WebInfoData } from '../../types' +import { VectorStorage } from '../types' +import type { StorageType, VectorConfig } from '../types' + +export interface ChromaVectorConfig extends VectorConfig { + url: string +} +export class ChromaVectorStorage extends VectorStorage { + constructor(config: ChromaVectorConfig, data?: WebInfoData) { + super(config, data) + } + + async save(data?: WebInfoData) { + if (!data && !this.data) + throw new Error('VectorStorage error: No Storage Data') + + const storageData = (data || this.data)! + const rawDoc = new Document({ pageContent: storageData.content, metadata: this.config.metaData }) + + /* Split text into chunks */ + const textSplitter = new RecursiveCharacterTextSplitter({ + chunkSize: 300, + chunkOverlap: 40, + }) + + const docs = await textSplitter.splitDocuments([rawDoc]) + await Chroma.fromDocuments(docs, this.config.embeddingsInfo.embeddings, { + collectionName: this.config.embeddingsInfo.indexName, + url: this.config.url, + }) + } + + async getRetriever() { + const vectorStore = await Chroma.fromExistingCollection( + this.config.embeddingsInfo.embeddings, + { + collectionName: this.config.embeddingsInfo.indexName, + url: this.config.url, + // TODO: remove this eslint rule after langchainjs add filter options + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-expect-error + filter: { + appName: this.config.metaData.appName, + botId: this.config.metaData.botId, + userId: this.config.metaData.userId, + }, + }) + return vectorStore.asRetriever() + } + + getConfig(): ChromaVectorConfig { + return this.config + } + + getType(): StorageType { + return { + type: 'VectorStorage', + name: 'ChromaVectorConfig', + } + } +} diff --git a/packages/core/src/storage/index.ts b/packages/core/src/storage/index.ts index 9872c8b..afedca5 100644 --- a/packages/core/src/storage/index.ts +++ b/packages/core/src/storage/index.ts @@ -1,5 +1,6 @@ import { NotionDataStorage } from './notion' import { SupabaseImageStorage, SupabaseVectorStorage } from './supabase' +import { ChromaVectorStorage } from './chroma' import type { TStorage } from './types' export * from './types' @@ -20,5 +21,6 @@ export const storageInfo: StorageInfo = { }, VectorStorage: { SupabaseVectorStorage, + ChromaVectorStorage, }, } diff --git a/packages/core/vite.config.ts b/packages/core/vite.config.ts index 42ea925..36189b2 100644 --- a/packages/core/vite.config.ts +++ b/packages/core/vite.config.ts @@ -42,6 +42,7 @@ export default defineConfig({ external: [ '@supabase/supabase-js', /langchain\/.*/, + 'chromadb', 'ofetch', ], output: { @@ -49,6 +50,7 @@ export default defineConfig({ // for externalized deps globals: { '@supabase/supabase-js': '@supabase/supabase-js', + 'chromadb': 'chromadb', 'ofetch': 'ofetch', }, }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c510e5d..9f92136 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -126,9 +126,12 @@ importers: '@supabase/supabase-js': specifier: ^2.26.0 version: 2.26.0 + chromadb: + specifier: ^1.5.3 + version: 1.5.3 langchain: specifier: ^0.0.96 - version: 0.0.96(@supabase/supabase-js@2.26.0) + version: 0.0.96(@supabase/supabase-js@2.26.0)(chromadb@1.5.3) devDependencies: ofetch: specifier: ^1.1.1 @@ -198,9 +201,12 @@ importers: '@vue-flow/minimap': specifier: ^1.1.1 version: 1.1.1(@vue-flow/core@1.20.2) + chromadb: + specifier: ^1.5.3 + version: 1.5.3 langchain: specifier: ^0.0.96 - version: 0.0.96(@supabase/supabase-js@2.26.0) + version: 0.0.96(@supabase/supabase-js@2.26.0)(chromadb@1.5.3) satori: specifier: ^0.10.1 version: 0.10.1 @@ -8045,6 +8051,10 @@ packages: engines: {node: '>=10'} dev: true + /chromadb@1.5.3: + resolution: {integrity: sha512-oQYs3cbSJJiSMYipL/5btX8U3bnW5X1CEMpJnl83XEGWR0uvRp8GtoznKPKPTcuVBMMtKkE1MdeMckWA72M4lw==} + dev: false + /chrome-trace-event@1.0.3: resolution: {integrity: sha512-p3KULyQg4S7NIHixdwbGX+nFHkoBiA4YQmyWtjb8XngSKV124nJmRysgAeujbUVb15vh+RvFUfCPqU7rXk+hZg==} engines: {node: '>=6.0'} @@ -13261,7 +13271,7 @@ packages: resolution: {integrity: sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==} dev: true - /langchain@0.0.96(@supabase/supabase-js@2.26.0): + /langchain@0.0.96(@supabase/supabase-js@2.26.0)(chromadb@1.5.3): resolution: {integrity: sha512-m1d0IWjwZK+D19uBfgmfp5sUTxJXcjJb/YuIyn2cGNmXGqOH2r2cl0TUnhuFKxTXpMTXgTywd7MYF5U5/qkNTQ==} engines: {node: '>=18'} peerDependencies: @@ -13416,6 +13426,7 @@ packages: ansi-styles: 5.2.0 binary-extensions: 2.2.0 camelcase: 6.3.0 + chromadb: 1.5.3 decamelize: 5.0.1 expr-eval: 2.0.2 flat: 5.0.2 diff --git a/server/nuxt3/composables/config.ts b/server/nuxt3/composables/config.ts index 505e9ca..b755ad6 100644 --- a/server/nuxt3/composables/config.ts +++ b/server/nuxt3/composables/config.ts @@ -342,6 +342,17 @@ export const vectorStorageConfig: BasicConfig = { }, output: '', }, + ChromaVectorStorage: { + displayName: 'Chroma Storage', + config: { + url: { + label: 'Chroma URL', + value: '', + require: true, + }, + }, + output: '', + }, }, } export const llmConfig: BasicConfig = { diff --git a/server/nuxt3/package.json b/server/nuxt3/package.json index 9638272..804cdf6 100644 --- a/server/nuxt3/package.json +++ b/server/nuxt3/package.json @@ -29,6 +29,7 @@ "@vue-flow/controls": "^1.1.0", "@vue-flow/core": "^1.20.2", "@vue-flow/minimap": "^1.1.1", + "chromadb": "^1.5.3", "langchain": "^0.0.96", "satori": "^0.10.1", "satori-html": "^0.3.2"