From 198ae17016f8c8f5116330a0415214520c6561cd Mon Sep 17 00:00:00 2001 From: vinodkiran Date: Mon, 13 Nov 2023 21:46:36 +0530 Subject: [PATCH 1/4] MongoDB Atlas Integration: Adding MongoDB Memory --- .../credentials/MongoDBUrlApi.credential.ts | 25 ++++ .../memory/MongoDBMemory/MongoDBMemory.ts | 115 ++++++++++++++++++ .../nodes/memory/MongoDBMemory/mongodb.png | Bin 0 -> 3741 bytes packages/components/package.json | 1 + 4 files changed, 141 insertions(+) create mode 100644 packages/components/credentials/MongoDBUrlApi.credential.ts create mode 100644 packages/components/nodes/memory/MongoDBMemory/MongoDBMemory.ts create mode 100644 packages/components/nodes/memory/MongoDBMemory/mongodb.png diff --git a/packages/components/credentials/MongoDBUrlApi.credential.ts b/packages/components/credentials/MongoDBUrlApi.credential.ts new file mode 100644 index 00000000000..2f2cba3834d --- /dev/null +++ b/packages/components/credentials/MongoDBUrlApi.credential.ts @@ -0,0 +1,25 @@ +import { INodeParams, INodeCredential } from '../src/Interface' + +class MongoDBUrlApi implements INodeCredential { + label: string + name: string + version: number + description: string + inputs: INodeParams[] + + constructor() { + this.label = 'MongoDB ATLAS' + this.name = 'mongoDBUrlApi' + this.version = 1.0 + this.inputs = [ + { + label: 'ATLAS Connection URL', + name: 'mongoDBConnectUrl', + type: 'string', + placeholder: 'mongodb+srv://myDatabaseUser:D1fficultP%40ssw0rd@cluster0.example.mongodb.net/?retryWrites=true&w=majority' + } + ] + } +} + +module.exports = { credClass: MongoDBUrlApi } diff --git a/packages/components/nodes/memory/MongoDBMemory/MongoDBMemory.ts b/packages/components/nodes/memory/MongoDBMemory/MongoDBMemory.ts new file mode 100644 index 00000000000..4c9e8581ec0 --- /dev/null +++ b/packages/components/nodes/memory/MongoDBMemory/MongoDBMemory.ts @@ -0,0 +1,115 @@ +import { getBaseClasses, getCredentialData, getCredentialParam, ICommonObject, INode, INodeData, INodeParams } from '../../../src' +import { MongoDBChatMessageHistory } from 'langchain/stores/message/mongodb' +import { BufferMemory, BufferMemoryInput } from 'langchain/memory' +import { MongoClient } from 'mongodb' + +class MongoDB_Memory implements INode { + label: string + name: string + version: number + description: string + type: string + icon: string + category: string + baseClasses: string[] + credential: INodeParams + inputs: INodeParams[] + + constructor() { + this.label = 'MongoDB Atlas Chat Memory' + this.name = 'MongoDBAtlasChatMemory' + this.version = 1.0 + this.type = 'MongoDBAtlasChatMemory' + this.icon = 'mongodb.png' + this.category = 'Memory' + this.description = 'Stores the conversation in MongoDB Atlas' + this.baseClasses = [this.type, ...getBaseClasses(BufferMemory)] + this.credential = { + label: 'Connect Credential', + name: 'credential', + type: 'credential', + credentialNames: ['mongoDBUrlApi'] + } + this.inputs = [ + { + label: 'Database', + name: 'databaseName', + placeholder: '', + type: 'string' + }, + { + label: 'Collection Name', + name: 'collectionName', + placeholder: '', + type: 'string' + }, + { + label: 'Session ID', + name: 'sessionId', + type: 'string', + default: '5f9cf7c08d5b1a06b80fae61', + description: 'Must be an Hex String of 24 chars. This will be the objectId of the document in MongoDB Atlas' + }, + { + label: 'Memory Key', + name: 'memoryKey', + type: 'string', + default: 'chat_history', + additionalParams: true + } + ] + } + + async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { + return initializeMongoDB(nodeData, options) + } + + async clearSessionMemory(nodeData: INodeData, options: ICommonObject): Promise { + const mongodbMemory = await initializeMongoDB(nodeData, options) + const sessionId = nodeData.inputs?.sessionId as string + options.logger.info(`Clearing MongoDB memory session ${sessionId}`) + await mongodbMemory.clear() + options.logger.info(`Successfully cleared MongoDB memory session ${sessionId}`) + } +} + +const initializeMongoDB = async (nodeData: INodeData, options: ICommonObject): Promise => { + const databaseName = nodeData.inputs?.databaseName as string + const collectionName = nodeData.inputs?.collectionName as string + const sessionId = nodeData.inputs?.sessionId as string + const memoryKey = nodeData.inputs?.memoryKey as string + + const credentialData = await getCredentialData(nodeData.credential ?? '', options) + let mongoDBConnectUrl = getCredentialParam('mongoDBConnectUrl', credentialData, nodeData) + + const client = new MongoClient(mongoDBConnectUrl) + await client.connect() + const collection = client.db(databaseName).collection(collectionName) + + const mongoDBChatMessageHistory = new MongoDBChatMessageHistory({ + collection, + sessionId: sessionId + }) + + return new BufferMemoryExtended({ + memoryKey, + chatHistory: mongoDBChatMessageHistory, + returnMessages: true, + isSessionIdUsingChatMessageId: false + }) +} + +interface BufferMemoryExtendedInput { + isSessionIdUsingChatMessageId: boolean +} + +class BufferMemoryExtended extends BufferMemory { + isSessionIdUsingChatMessageId? = false + + constructor(fields: BufferMemoryInput & Partial) { + super(fields) + this.isSessionIdUsingChatMessageId = fields.isSessionIdUsingChatMessageId + } +} + +module.exports = { nodeClass: MongoDB_Memory } diff --git a/packages/components/nodes/memory/MongoDBMemory/mongodb.png b/packages/components/nodes/memory/MongoDBMemory/mongodb.png new file mode 100644 index 0000000000000000000000000000000000000000..5586fe0ac672f7997014d814389c1d6c436d9d0c GIT binary patch literal 3741 zcmb`Jc{J4R`^U$YrHqk1Gck`mMA>7^WSwCm4MMgVON56fyHH5hDNHC^_9aWUvCB4O z8H(&h8tVuN^<;@+e)IkLp80jo@0{;n-_N&iOnv!JxUh&Ts($ z0B(aTsO$81EC9fCfrEukvneL|008rziIKS;+S};wO8sMVEzz+`Q>L?XktQA7i?7g` z4#!S9-+HcfJoPC3m&8yR0hXysbjD@Fj@91-{(6BtcDfX69~WV$#}dQI@c(CEs2-Ji z3})=awe;V;{9g9oVd={Dx$4JE9~t0QrfZX~!l-ZzjFG-=Mld6Wk$WuB8?nt$!DXp2 zA{jtW`8~__*`?#Ddj$%csd`+85x@YC3Ozd0jZxDs8+NSP=lov_4!^bIn2(@Wb1KmI zeQ5?C!|_NzH{U9ZTY-Lf&CPp74mVO26Y2xPik)wyDx@_=WH-kby-bNL@t`kP+VU*! zMIb%Pv(oTIb53&Q1Isj}>b9axVtid|c~O0OV6{b48__O9y|ts!@tIafM@w^Ct!tLP zWvXIytw$HB-7DAhRVOLAA*8#rGq%*Hr?ZRrvLu9v!=|f6mw2XC#nAn|UEO!9ZRkL^ znzqto_?N%4o5BArxOnsMDg8@0^uLZV1DGYbzZVang)ybf%be<_9mO-NZPHei@kg{w zHt0v%mS!o3<|=L5u&?PPf90M<(P1Y@_@@(bP(<<27xj*E(R!&8=pXi-w!S;5z;lpZ zOXG?cjMy*qqtS95zNKLB@E}&n0XL+l!8$}{>@qm%Pwq{-uR7OZQg+G|2+k{V(G`?S`M#-Ug z9ZxcbwE)+;C(f}!X`0gpz=>NABj@G)SBb$r%=e6Ac;X@o=ikBv)h}?xJ?g?5l>N#r ztN^!JlmDOv;vX3jZ&+MhHlz7XOhnWr!~a5{LRo&e|D8;O6_7&Yz-+h@ZvAF)yg1Ct74yeTzy{M z>#J$YPPlzCwCbI%Q@&HXQaf^c2WiWK;8S{ntp76F*PNP|0l7)MDkSk|o%-N++^JJD zK9oCdQBlAiXO~#j{)9}<42mPtv+MmMueqdPSX6KmucixSJ!P$hCO!aN*ZS*_~8$1^IR(^`S8VcS#h;SXc>?vx2| zzALm#0oLp=U$&?{mjXwH%3gME7dD;G6U(RxE}Mcj9xSwoUQ^n#yQ{7hTiVRj)IPQz zDe?JWyrKqkFb}HUe9-#tbO&rug3sG*;QU$~5*x}br)2l|2Ne{_-B0ROE3zl7X7(f2 zEAd{y(cqK&ySp*l!@MN?<+?kDf$d9u(AiMqolS4>R;y2Xq2R(7lNFGyE5F?Z)>z>j z?alZZ-dxCjg5ddm?Sl=F9$){o&ES3Vq)=jc-}$mmojIuMmbCZCaN=v8=fGI2ro@I9 zPuJ50_#CGQA!J3W=w@C|L;9ogR|@ zbrv%)(Dc5J)CaxK@L^=Ue`YVlPuthW}ka%7Fq9OL)Q3Hzx#Mg(c5IIUj=XlvF9vbEivnf z2*ZDc_YgOl&u2g`asw1byEdjiumD)Jd=Ul8025!rjAIx#z$xt|WcT5LVLatquw?kX zzyyLfxA$Wwf9T`N;oFb;y!1x+DVvPH;_4hU>(m6JU2<>6w{tlo09j<^3SH(c89IbeV+;TYkl;SgbcC&MXNDj|}^==SjFPnS%A~5n(-M-C` z8SWqV)0!bX+`mY5iY{cB1F35{p#G403g;Ry)%|p>nrnlNdsU)1-q;HA76856{aN82 zN920}xVsBKcc}z?7MI7P;Qwa@-x=%!Nc@}#<# z04Od_)JCe}q~C;gONEC&`>h*=<5pJXg*VyOWIf-l;g6z>4%LfBM}3(xY^*0&+&tK^ zUqBI&#M9G;b#mx7_URW~9=_Wb#cg{@>((RUnuIR_ULz1H#gXQeeggfg;FTUfC&98c zv2HqQi_rKN)pK8HPUyyzC+h4F_x_hsb)eKg4OhLhNDtZ$%FUlg5Ee`4}lYFP%Qr2a$Q7Y!07Tda0l{0U(CbV9?G z6c)EIDw?^eF5J{*|!GN*G$G4Q5&@sr}iAcZQae|(A89tpiw5< z@zzek`)>q*NropSjqU4TTwKa=>0pU>HWunUq@hLWx3Z4DMnq5V5APnDu0OxEEKm{m zGgGxWILA2O7nb$BO1Mw+?x>*r0=0LdJ8ot_!A|`4PyD#v3D{*&Sc&iUS?f8u2BCfM z{P+$ks#ZoC@j37At~zt>+2RSfZas5Z>nn`L7TJ>u*XzG3^6}-a&86y%Ki&yP^(L`M z45DTnRC1Qkic!=D!4`Z|**uk;J512mkGj0lPyWH$Re1JvpJ2hSTq}Y|$pie~_wWc= z6xS7K%wfO7l*oX!FbhG^YYp8=kFQ+~|1G}D>HG4k% zU<)p3sh|gP=dE2}m0RS_S5GLI6MlkALrS#%CKQzY*h^Q=4m5H74E7dobHJoSYhy;9 znE6TEDVoZD(ecb1ZtV-i~whn5SBHHG{2nK&~Qi~^#rVj?tBWKPZYVchduFxy5 zy7y5*m^y6#RjhWwzAoUll_Cgby8Yacl7tB4d=e_CtemNn_{;^v78txW%u~<(5-UJ# z#r)SGA!)%WW;bcqjd+MGK-B`tE%S2W~8N{6$9zs%|z zf%q|Jbva4lY4t51YxC_gPXMRr&Qye%&`7K%TS>7U0tHUDV=r-hI_k61zpDLb(UMFd zvc{YjA}y~3Z_(B#oAjxz5`iTB0;Uq2WD5&Do&8y%sE{Z*$)IS0 tLACkG!7qJ7*HW-wMKKxE3+#3P=^7KM_f;;R=|9^513e6?O4lLce*i)hLc9O~ literal 0 HcmV?d00001 diff --git a/packages/components/package.json b/packages/components/package.json index 996419ca08e..ee2adbf9aba 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -55,6 +55,7 @@ "llmonitor": "^0.5.5", "mammoth": "^1.5.1", "moment": "^2.29.3", + "mongodb": "^6.2.0", "mysql2": "^3.5.1", "node-fetch": "^2.6.11", "node-html-markdown": "^1.3.0", From 26c86f1b371c3968293387b957beba3cf89a7b7c Mon Sep 17 00:00:00 2001 From: vinodkiran Date: Mon, 13 Nov 2023 21:56:18 +0530 Subject: [PATCH 2/4] MongoDB Atlas Integration: Adding MongoDB as a Vector Store --- .../vectorstores/MongoDB/MongoDBSearchBase.ts | 145 ++++++++++++++++++ .../vectorstores/MongoDB/MongoDB_Existing.ts | 41 +++++ .../vectorstores/MongoDB/MongoDB_Upsert.ts | 58 +++++++ .../nodes/vectorstores/MongoDB/mongodb.png | Bin 0 -> 3741 bytes 4 files changed, 244 insertions(+) create mode 100644 packages/components/nodes/vectorstores/MongoDB/MongoDBSearchBase.ts create mode 100644 packages/components/nodes/vectorstores/MongoDB/MongoDB_Existing.ts create mode 100644 packages/components/nodes/vectorstores/MongoDB/MongoDB_Upsert.ts create mode 100644 packages/components/nodes/vectorstores/MongoDB/mongodb.png diff --git a/packages/components/nodes/vectorstores/MongoDB/MongoDBSearchBase.ts b/packages/components/nodes/vectorstores/MongoDB/MongoDBSearchBase.ts new file mode 100644 index 00000000000..e9ef8e9a164 --- /dev/null +++ b/packages/components/nodes/vectorstores/MongoDB/MongoDBSearchBase.ts @@ -0,0 +1,145 @@ +import { + getBaseClasses, + getCredentialData, + getCredentialParam, + ICommonObject, + INodeData, + INodeOutputsValue, + INodeParams +} from '../../../src' + +import { Embeddings } from 'langchain/embeddings/base' +import { VectorStore } from 'langchain/vectorstores/base' +import { Document } from 'langchain/document' +import { MongoDBAtlasVectorSearch } from 'langchain/vectorstores/mongodb_atlas' +import { Collection, MongoClient } from 'mongodb' + +export abstract class MongoDBSearchBase { + label: string + name: string + version: number + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + credential: INodeParams + outputs: INodeOutputsValue[] + mongoClient: MongoClient + + protected constructor() { + this.type = 'MongoDB Atlas' + this.icon = 'mongodb.png' + this.category = 'Vector Stores' + this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever'] + this.credential = { + label: 'Connect Credential', + name: 'credential', + type: 'credential', + credentialNames: ['mongoDBUrlApi'] + } + this.inputs = [ + { + label: 'Embeddings', + name: 'embeddings', + type: 'Embeddings' + }, + { + label: 'Database', + name: 'databaseName', + placeholder: '', + type: 'string' + }, + { + label: 'Collection Name', + name: 'collectionName', + placeholder: '', + type: 'string' + }, + { + label: 'Index Name', + name: 'indexName', + placeholder: '', + type: 'string' + }, + { + label: 'Content Field', + name: 'textKey', + description: 'Name of the field (column) that contains the actual content', + type: 'string', + default: 'text', + additionalParams: true, + optional: true + }, + { + label: 'Embedded Field', + name: 'embeddingKey', + description: 'Name of the field (column) that contains the Embedding', + type: 'string', + default: 'embedding', + additionalParams: true, + optional: true + }, + { + label: 'Top K', + name: 'topK', + description: 'Number of top results to fetch. Default to 4', + placeholder: '4', + type: 'number', + additionalParams: true, + optional: true + } + ] + this.outputs = [ + { + label: 'MongoDB Retriever', + name: 'retriever', + baseClasses: this.baseClasses + }, + { + label: 'MongoDB Vector Store', + name: 'vectorStore', + baseClasses: [this.type, ...getBaseClasses(MongoDBAtlasVectorSearch)] + } + ] + } + + abstract constructVectorStore( + embeddings: Embeddings, + collection: Collection, + indexName: string, + textKey: string, + embeddingKey: string, + docs: Document>[] | undefined + ): Promise + + async init(nodeData: INodeData, _: string, options: ICommonObject, docs: Document>[] | undefined): Promise { + const credentialData = await getCredentialData(nodeData.credential ?? '', options) + const databaseName = nodeData.inputs?.databaseName as string + const collectionName = nodeData.inputs?.collectionName as string + const indexName = nodeData.inputs?.indexName as string + let textKey = nodeData.inputs?.textKey as string + let embeddingKey = nodeData.inputs?.embeddingKey as string + const embeddings = nodeData.inputs?.embeddings as Embeddings + const topK = nodeData.inputs?.topK as string + const k = topK ? parseFloat(topK) : 4 + const output = nodeData.outputs?.output as string + + let mongoDBConnectUrl = getCredentialParam('mongoDBConnectUrl', credentialData, nodeData) + + this.mongoClient = new MongoClient(mongoDBConnectUrl) + const collection = this.mongoClient.db(databaseName).collection(collectionName) + if (!textKey || textKey === '') textKey = 'text' + if (!embeddingKey || embeddingKey === '') embeddingKey = 'embedding' + const vectorStore = await this.constructVectorStore(embeddings, collection, indexName, textKey, embeddingKey, docs) + + if (output === 'retriever') { + return vectorStore.asRetriever(k) + } else if (output === 'vectorStore') { + ;(vectorStore as any).k = k + return vectorStore + } + return vectorStore + } +} diff --git a/packages/components/nodes/vectorstores/MongoDB/MongoDB_Existing.ts b/packages/components/nodes/vectorstores/MongoDB/MongoDB_Existing.ts new file mode 100644 index 00000000000..3cbb36b8790 --- /dev/null +++ b/packages/components/nodes/vectorstores/MongoDB/MongoDB_Existing.ts @@ -0,0 +1,41 @@ +import { ICommonObject, INode, INodeData } from '../../../src/Interface' +import { Embeddings } from 'langchain/embeddings/base' +import { VectorStore } from 'langchain/vectorstores/base' +import { Document } from 'langchain/document' + +import { MongoDBSearchBase } from './MongoDBSearchBase' +import { Collection } from 'mongodb' +import { MongoDBAtlasVectorSearch } from 'langchain/vectorstores/mongodb_atlas' + +class MongoDBExisting_VectorStores extends MongoDBSearchBase implements INode { + constructor() { + super() + this.label = 'MongoDB Atlas Load Existing Index' + this.name = 'MongoDBIndex' + this.version = 1.0 + this.description = 'Load existing data from MongoDB Atlas (i.e: Document has been upserted)' + } + + async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { + return super.init(nodeData, _, options, undefined) + } + + constructVectorStore( + embeddings: Embeddings, + collection: Collection, + indexName: string, + textKey: string, + embeddingKey: string, + _: Document>[] | undefined + ): Promise { + const mongoDBAtlasVectorSearch = new MongoDBAtlasVectorSearch(embeddings, { + collection: collection, + indexName: indexName, + textKey: textKey, + embeddingKey: embeddingKey + }) + return Promise.resolve(mongoDBAtlasVectorSearch) + } +} + +module.exports = { nodeClass: MongoDBExisting_VectorStores } diff --git a/packages/components/nodes/vectorstores/MongoDB/MongoDB_Upsert.ts b/packages/components/nodes/vectorstores/MongoDB/MongoDB_Upsert.ts new file mode 100644 index 00000000000..80dfbf19522 --- /dev/null +++ b/packages/components/nodes/vectorstores/MongoDB/MongoDB_Upsert.ts @@ -0,0 +1,58 @@ +import { ICommonObject, INode, INodeData } from '../../../src/Interface' +import { Embeddings } from 'langchain/embeddings/base' +import { Document } from 'langchain/document' + +import { flatten } from 'lodash' +import { VectorStore } from 'langchain/vectorstores/base' +import { MongoDBSearchBase } from './MongoDBSearchBase' +import { Collection } from 'mongodb' +import { MongoDBAtlasVectorSearch } from 'langchain/vectorstores/mongodb_atlas' + +class MongoDBUpsert_VectorStores extends MongoDBSearchBase implements INode { + constructor() { + super() + this.label = 'MongoDB Upsert Document' + this.name = 'MongoDBUpsert' + this.version = 1.0 + this.description = 'Upsert documents to MongoDB Atlas' + this.inputs.unshift({ + label: 'Document', + name: 'document', + type: 'Document', + list: true + }) + } + + constructVectorStore( + embeddings: Embeddings, + collection: Collection, + indexName: string, + textKey: string, + embeddingKey: string, + docs: Document>[] + ): Promise { + return MongoDBAtlasVectorSearch.fromDocuments(docs, embeddings, { + collection: collection, + indexName: indexName, + textKey: textKey, + embeddingKey: embeddingKey + }) + } + + async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { + const docs = nodeData.inputs?.document as Document[] + + const flattenDocs = docs && docs.length ? flatten(docs) : [] + const finalDocs = [] + for (let i = 0; i < flattenDocs.length; i += 1) { + if (flattenDocs[i] && flattenDocs[i].pageContent) { + const document = new Document(flattenDocs[i]) + finalDocs.push(document) + } + } + + return super.init(nodeData, _, options, flattenDocs) + } +} + +module.exports = { nodeClass: MongoDBUpsert_VectorStores } diff --git a/packages/components/nodes/vectorstores/MongoDB/mongodb.png b/packages/components/nodes/vectorstores/MongoDB/mongodb.png new file mode 100644 index 0000000000000000000000000000000000000000..5586fe0ac672f7997014d814389c1d6c436d9d0c GIT binary patch literal 3741 zcmb`Jc{J4R`^U$YrHqk1Gck`mMA>7^WSwCm4MMgVON56fyHH5hDNHC^_9aWUvCB4O z8H(&h8tVuN^<;@+e)IkLp80jo@0{;n-_N&iOnv!JxUh&Ts($ z0B(aTsO$81EC9fCfrEukvneL|008rziIKS;+S};wO8sMVEzz+`Q>L?XktQA7i?7g` z4#!S9-+HcfJoPC3m&8yR0hXysbjD@Fj@91-{(6BtcDfX69~WV$#}dQI@c(CEs2-Ji z3})=awe;V;{9g9oVd={Dx$4JE9~t0QrfZX~!l-ZzjFG-=Mld6Wk$WuB8?nt$!DXp2 zA{jtW`8~__*`?#Ddj$%csd`+85x@YC3Ozd0jZxDs8+NSP=lov_4!^bIn2(@Wb1KmI zeQ5?C!|_NzH{U9ZTY-Lf&CPp74mVO26Y2xPik)wyDx@_=WH-kby-bNL@t`kP+VU*! zMIb%Pv(oTIb53&Q1Isj}>b9axVtid|c~O0OV6{b48__O9y|ts!@tIafM@w^Ct!tLP zWvXIytw$HB-7DAhRVOLAA*8#rGq%*Hr?ZRrvLu9v!=|f6mw2XC#nAn|UEO!9ZRkL^ znzqto_?N%4o5BArxOnsMDg8@0^uLZV1DGYbzZVang)ybf%be<_9mO-NZPHei@kg{w zHt0v%mS!o3<|=L5u&?PPf90M<(P1Y@_@@(bP(<<27xj*E(R!&8=pXi-w!S;5z;lpZ zOXG?cjMy*qqtS95zNKLB@E}&n0XL+l!8$}{>@qm%Pwq{-uR7OZQg+G|2+k{V(G`?S`M#-Ug z9ZxcbwE)+;C(f}!X`0gpz=>NABj@G)SBb$r%=e6Ac;X@o=ikBv)h}?xJ?g?5l>N#r ztN^!JlmDOv;vX3jZ&+MhHlz7XOhnWr!~a5{LRo&e|D8;O6_7&Yz-+h@ZvAF)yg1Ct74yeTzy{M z>#J$YPPlzCwCbI%Q@&HXQaf^c2WiWK;8S{ntp76F*PNP|0l7)MDkSk|o%-N++^JJD zK9oCdQBlAiXO~#j{)9}<42mPtv+MmMueqdPSX6KmucixSJ!P$hCO!aN*ZS*_~8$1^IR(^`S8VcS#h;SXc>?vx2| zzALm#0oLp=U$&?{mjXwH%3gME7dD;G6U(RxE}Mcj9xSwoUQ^n#yQ{7hTiVRj)IPQz zDe?JWyrKqkFb}HUe9-#tbO&rug3sG*;QU$~5*x}br)2l|2Ne{_-B0ROE3zl7X7(f2 zEAd{y(cqK&ySp*l!@MN?<+?kDf$d9u(AiMqolS4>R;y2Xq2R(7lNFGyE5F?Z)>z>j z?alZZ-dxCjg5ddm?Sl=F9$){o&ES3Vq)=jc-}$mmojIuMmbCZCaN=v8=fGI2ro@I9 zPuJ50_#CGQA!J3W=w@C|L;9ogR|@ zbrv%)(Dc5J)CaxK@L^=Ue`YVlPuthW}ka%7Fq9OL)Q3Hzx#Mg(c5IIUj=XlvF9vbEivnf z2*ZDc_YgOl&u2g`asw1byEdjiumD)Jd=Ul8025!rjAIx#z$xt|WcT5LVLatquw?kX zzyyLfxA$Wwf9T`N;oFb;y!1x+DVvPH;_4hU>(m6JU2<>6w{tlo09j<^3SH(c89IbeV+;TYkl;SgbcC&MXNDj|}^==SjFPnS%A~5n(-M-C` z8SWqV)0!bX+`mY5iY{cB1F35{p#G403g;Ry)%|p>nrnlNdsU)1-q;HA76856{aN82 zN920}xVsBKcc}z?7MI7P;Qwa@-x=%!Nc@}#<# z04Od_)JCe}q~C;gONEC&`>h*=<5pJXg*VyOWIf-l;g6z>4%LfBM}3(xY^*0&+&tK^ zUqBI&#M9G;b#mx7_URW~9=_Wb#cg{@>((RUnuIR_ULz1H#gXQeeggfg;FTUfC&98c zv2HqQi_rKN)pK8HPUyyzC+h4F_x_hsb)eKg4OhLhNDtZ$%FUlg5Ee`4}lYFP%Qr2a$Q7Y!07Tda0l{0U(CbV9?G z6c)EIDw?^eF5J{*|!GN*G$G4Q5&@sr}iAcZQae|(A89tpiw5< z@zzek`)>q*NropSjqU4TTwKa=>0pU>HWunUq@hLWx3Z4DMnq5V5APnDu0OxEEKm{m zGgGxWILA2O7nb$BO1Mw+?x>*r0=0LdJ8ot_!A|`4PyD#v3D{*&Sc&iUS?f8u2BCfM z{P+$ks#ZoC@j37At~zt>+2RSfZas5Z>nn`L7TJ>u*XzG3^6}-a&86y%Ki&yP^(L`M z45DTnRC1Qkic!=D!4`Z|**uk;J512mkGj0lPyWH$Re1JvpJ2hSTq}Y|$pie~_wWc= z6xS7K%wfO7l*oX!FbhG^YYp8=kFQ+~|1G}D>HG4k% zU<)p3sh|gP=dE2}m0RS_S5GLI6MlkALrS#%CKQzY*h^Q=4m5H74E7dobHJoSYhy;9 znE6TEDVoZD(ecb1ZtV-i~whn5SBHHG{2nK&~Qi~^#rVj?tBWKPZYVchduFxy5 zy7y5*m^y6#RjhWwzAoUll_Cgby8Yacl7tB4d=e_CtemNn_{;^v78txW%u~<(5-UJ# z#r)SGA!)%WW;bcqjd+MGK-B`tE%S2W~8N{6$9zs%|z zf%q|Jbva4lY4t51YxC_gPXMRr&Qye%&`7K%TS>7U0tHUDV=r-hI_k61zpDLb(UMFd zvc{YjA}y~3Z_(B#oAjxz5`iTB0;Uq2WD5&Do&8y%sE{Z*$)IS0 tLACkG!7qJ7*HW-wMKKxE3+#3P=^7KM_f;;R=|9^513e6?O4lLce*i)hLc9O~ literal 0 HcmV?d00001 From df249ec3f0ee2549ebc106036bc2ddf1986a1971 Mon Sep 17 00:00:00 2001 From: Henry Date: Fri, 17 Nov 2023 13:52:41 +0000 Subject: [PATCH 3/4] fix mongodb vector database where no documents are returned --- .../Elasticsearch/Elasticsearch_Upsert.ts | 2 +- .../vectorstores/MongoDB/MongoDB_Existing.ts | 12 +++++------- .../vectorstores/MongoDB/MongoDB_Upsert.ts | 17 +++++++++-------- .../nodes/vectorstores/Redis/Redis_Upsert.ts | 2 +- 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Upsert.ts b/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Upsert.ts index d396578654e..a8ccd49acfb 100644 --- a/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Upsert.ts +++ b/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Upsert.ts @@ -50,7 +50,7 @@ class ElasicsearchUpsert_VectorStores extends ElasticSearchBase implements INode delete d.metadata.loc }) // end of workaround - return super.init(nodeData, _, options, flattenDocs) + return super.init(nodeData, _, options, finalDocs) } } diff --git a/packages/components/nodes/vectorstores/MongoDB/MongoDB_Existing.ts b/packages/components/nodes/vectorstores/MongoDB/MongoDB_Existing.ts index 3cbb36b8790..7b06814afeb 100644 --- a/packages/components/nodes/vectorstores/MongoDB/MongoDB_Existing.ts +++ b/packages/components/nodes/vectorstores/MongoDB/MongoDB_Existing.ts @@ -1,11 +1,10 @@ -import { ICommonObject, INode, INodeData } from '../../../src/Interface' +import { Collection } from 'mongodb' +import { MongoDBAtlasVectorSearch } from 'langchain/vectorstores/mongodb_atlas' import { Embeddings } from 'langchain/embeddings/base' import { VectorStore } from 'langchain/vectorstores/base' import { Document } from 'langchain/document' - import { MongoDBSearchBase } from './MongoDBSearchBase' -import { Collection } from 'mongodb' -import { MongoDBAtlasVectorSearch } from 'langchain/vectorstores/mongodb_atlas' +import { ICommonObject, INode, INodeData } from '../../../src/Interface' class MongoDBExisting_VectorStores extends MongoDBSearchBase implements INode { constructor() { @@ -20,7 +19,7 @@ class MongoDBExisting_VectorStores extends MongoDBSearchBase implements INode { return super.init(nodeData, _, options, undefined) } - constructVectorStore( + async constructVectorStore( embeddings: Embeddings, collection: Collection, indexName: string, @@ -28,13 +27,12 @@ class MongoDBExisting_VectorStores extends MongoDBSearchBase implements INode { embeddingKey: string, _: Document>[] | undefined ): Promise { - const mongoDBAtlasVectorSearch = new MongoDBAtlasVectorSearch(embeddings, { + return new MongoDBAtlasVectorSearch(embeddings, { collection: collection, indexName: indexName, textKey: textKey, embeddingKey: embeddingKey }) - return Promise.resolve(mongoDBAtlasVectorSearch) } } diff --git a/packages/components/nodes/vectorstores/MongoDB/MongoDB_Upsert.ts b/packages/components/nodes/vectorstores/MongoDB/MongoDB_Upsert.ts index 80dfbf19522..7d22f03526f 100644 --- a/packages/components/nodes/vectorstores/MongoDB/MongoDB_Upsert.ts +++ b/packages/components/nodes/vectorstores/MongoDB/MongoDB_Upsert.ts @@ -1,12 +1,11 @@ -import { ICommonObject, INode, INodeData } from '../../../src/Interface' +import { flatten } from 'lodash' +import { Collection } from 'mongodb' import { Embeddings } from 'langchain/embeddings/base' import { Document } from 'langchain/document' - -import { flatten } from 'lodash' import { VectorStore } from 'langchain/vectorstores/base' -import { MongoDBSearchBase } from './MongoDBSearchBase' -import { Collection } from 'mongodb' import { MongoDBAtlasVectorSearch } from 'langchain/vectorstores/mongodb_atlas' +import { ICommonObject, INode, INodeData } from '../../../src/Interface' +import { MongoDBSearchBase } from './MongoDBSearchBase' class MongoDBUpsert_VectorStores extends MongoDBSearchBase implements INode { constructor() { @@ -23,7 +22,7 @@ class MongoDBUpsert_VectorStores extends MongoDBSearchBase implements INode { }) } - constructVectorStore( + async constructVectorStore( embeddings: Embeddings, collection: Collection, indexName: string, @@ -31,12 +30,14 @@ class MongoDBUpsert_VectorStores extends MongoDBSearchBase implements INode { embeddingKey: string, docs: Document>[] ): Promise { - return MongoDBAtlasVectorSearch.fromDocuments(docs, embeddings, { + const mongoDBAtlasVectorSearch = new MongoDBAtlasVectorSearch(embeddings, { collection: collection, indexName: indexName, textKey: textKey, embeddingKey: embeddingKey }) + await mongoDBAtlasVectorSearch.addDocuments(docs) + return mongoDBAtlasVectorSearch } async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { @@ -51,7 +52,7 @@ class MongoDBUpsert_VectorStores extends MongoDBSearchBase implements INode { } } - return super.init(nodeData, _, options, flattenDocs) + return super.init(nodeData, _, options, finalDocs) } } diff --git a/packages/components/nodes/vectorstores/Redis/Redis_Upsert.ts b/packages/components/nodes/vectorstores/Redis/Redis_Upsert.ts index 9d1a4f4534e..4da58eaffbc 100644 --- a/packages/components/nodes/vectorstores/Redis/Redis_Upsert.ts +++ b/packages/components/nodes/vectorstores/Redis/Redis_Upsert.ts @@ -56,7 +56,7 @@ class RedisUpsert_VectorStores extends RedisSearchBase implements INode { } } - return super.init(nodeData, _, options, flattenDocs) + return super.init(nodeData, _, options, finalDocs) } } From 0c81b194c3dd72f6d9ae78da5d00e6a785afe7dc Mon Sep 17 00:00:00 2001 From: Henry Date: Fri, 17 Nov 2023 14:38:42 +0000 Subject: [PATCH 4/4] changed to enable uuid to be used as sessionId --- .../memory/MongoDBMemory/MongoDBMemory.ts | 45 ++++++++++++++++--- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/packages/components/nodes/memory/MongoDBMemory/MongoDBMemory.ts b/packages/components/nodes/memory/MongoDBMemory/MongoDBMemory.ts index 4c9e8581ec0..7de2ec34788 100644 --- a/packages/components/nodes/memory/MongoDBMemory/MongoDBMemory.ts +++ b/packages/components/nodes/memory/MongoDBMemory/MongoDBMemory.ts @@ -1,6 +1,7 @@ import { getBaseClasses, getCredentialData, getCredentialParam, ICommonObject, INode, INodeData, INodeParams } from '../../../src' import { MongoDBChatMessageHistory } from 'langchain/stores/message/mongodb' import { BufferMemory, BufferMemoryInput } from 'langchain/memory' +import { BaseMessage, mapStoredMessageToChatMessage } from 'langchain/schema' import { MongoClient } from 'mongodb' class MongoDB_Memory implements INode { @@ -44,11 +45,13 @@ class MongoDB_Memory implements INode { type: 'string' }, { - label: 'Session ID', + label: 'Session Id', name: 'sessionId', type: 'string', - default: '5f9cf7c08d5b1a06b80fae61', - description: 'Must be an Hex String of 24 chars. This will be the objectId of the document in MongoDB Atlas' + description: 'If not specified, the first CHAT_MESSAGE_ID will be used as sessionId', + default: '', + additionalParams: true, + optional: true }, { label: 'Memory Key', @@ -67,9 +70,10 @@ class MongoDB_Memory implements INode { async clearSessionMemory(nodeData: INodeData, options: ICommonObject): Promise { const mongodbMemory = await initializeMongoDB(nodeData, options) const sessionId = nodeData.inputs?.sessionId as string - options.logger.info(`Clearing MongoDB memory session ${sessionId}`) + const chatId = options?.chatId as string + options.logger.info(`Clearing MongoDB memory session ${sessionId ? sessionId : chatId}`) await mongodbMemory.clear() - options.logger.info(`Successfully cleared MongoDB memory session ${sessionId}`) + options.logger.info(`Successfully cleared MongoDB memory session ${sessionId ? sessionId : chatId}`) } } @@ -78,6 +82,10 @@ const initializeMongoDB = async (nodeData: INodeData, options: ICommonObject): P const collectionName = nodeData.inputs?.collectionName as string const sessionId = nodeData.inputs?.sessionId as string const memoryKey = nodeData.inputs?.memoryKey as string + const chatId = options?.chatId as string + + let isSessionIdUsingChatMessageId = false + if (!sessionId && chatId) isSessionIdUsingChatMessageId = true const credentialData = await getCredentialData(nodeData.credential ?? '', options) let mongoDBConnectUrl = getCredentialParam('mongoDBConnectUrl', credentialData, nodeData) @@ -88,14 +96,37 @@ const initializeMongoDB = async (nodeData: INodeData, options: ICommonObject): P const mongoDBChatMessageHistory = new MongoDBChatMessageHistory({ collection, - sessionId: sessionId + sessionId: sessionId ? sessionId : chatId }) + mongoDBChatMessageHistory.getMessages = async (): Promise => { + const document = await collection.findOne({ + sessionId: (mongoDBChatMessageHistory as any).sessionId + }) + const messages = document?.messages || [] + return messages.map(mapStoredMessageToChatMessage) + } + + mongoDBChatMessageHistory.addMessage = async (message: BaseMessage): Promise => { + const messages = [message].map((msg) => msg.toDict()) + await collection.updateOne( + { sessionId: (mongoDBChatMessageHistory as any).sessionId }, + { + $push: { messages: { $each: messages } } + }, + { upsert: true } + ) + } + + mongoDBChatMessageHistory.clear = async (): Promise => { + await collection.deleteOne({ sessionId: (mongoDBChatMessageHistory as any).sessionId }) + } + return new BufferMemoryExtended({ memoryKey, chatHistory: mongoDBChatMessageHistory, returnMessages: true, - isSessionIdUsingChatMessageId: false + isSessionIdUsingChatMessageId }) }