Error storing in Weaviate

[lakshya29154]

Checked other resources

• I added a very descriptive title to this question.
• I searched the LangChain documentation with the integrated search.
• I used the GitHub search to find a similar question and didn't find it.

Commit to Help

• I commit to help with one of those options 👆

Example Code

/*eslint-disable*/
import * as fs from "fs";
import { config } from 'dotenv';
import { createHash } from 'crypto';
import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";
import { OpenAIEmbeddings } from "@langchain/openai";
import weaviate, { ApiKey } from "weaviate-ts-client";
import { WeaviateStore } from "@langchain/weaviate";
import { OpenAI } from "@langchain/openai";
import { PromptTemplate } from "@langchain/core/prompts";
config();

const model = new OpenAI({ temperature: 0, apiKey: OPENAI_API_KEY });
export async function swaggerFile(query2)
{
    
const model = new OpenAI({ temperature: 0, apiKey: OPENAI_API_KEY });
const API_URL_PROMPT_TEMPLATE1 = `You are given the below the data for some endpoints
{api_docs}
Using this data , try to find the endpoints that user ask in his query this is the {question} 
Also give the method like get put post delete
Keep in mind the query can ask for multiple endpoints you need to list all the endpoints like step1 step2 
Always answer in steps


Question:{question}
API url:`;
const API_URL_PROMPT_TEMPLATE = new PromptTemplate({
    inputVariables: ["api_docs", "question"],
    template: API_URL_PROMPT_TEMPLATE1,
});

function generateHash(text) {
    return createHash('sha256').update(text).digest('hex');
}

async function splitTextIntoChunks(text) {
    const splitter = new RecursiveCharacterTextSplitter({ chunkSize: 1000, chunkOverlap: 200 });
    const documents = await splitter.createDocuments([text]);
    return documents.map(doc => doc.pageContent);
}

async function generateEmbeddings(text) {
    if (typeof text !== 'string') {
        throw new TypeError('Input text must be a string');
    }
    const embeddings = new OpenAIEmbeddings({ apiKey: process.env.OPENAI_API_KEY });
    return await embeddings.embedQuery(text);
}

async function getNearbyEmbeddings(queryEmbedding) {
    const client = (weaviate).client({
        scheme: process.env.WEAVIATE_SCHEME || "https",
        host: "test-rmmvmv6q.grpc.weaviate.network",
        apiKey: new ApiKey('tNPN1kOkrcmgCpogUYbtYKbYvH4hSTfjx4Fn'),
    });
      console.log("Here is ")
    const store = await WeaviateStore.fromExistingIndex(new OpenAIEmbeddings(), {
        client,
        indexName: "Test",
        textKey: "text",
        metadataKeys: ["hash"],
    });
    // console.log("I am here1")
    // console.log(typeof(queryEmbedding))
    queryEmbedding =  JSON.stringify(queryEmbedding)
    // console.log("queryEmbedding length", queryEmbedding);
    const results = await store.similaritySearchWithScore(queryEmbedding, 5);
    // console.log("This is the result", results);
    return results
    // return results.map((item) => item.metadata.vector);
}

async function main() {
    try {
        const specFilePath = './swagger.json';
        const jsonData = fs.readFileSync(specFilePath, 'utf8');
        console.log("This is json data:", jsonData);

        const hash = generateHash(jsonData);
        const textChunks = await splitTextIntoChunks(jsonData);
        const client = (weaviate).client({
            scheme: process.env.WEAVIATE_SCHEME || "https",
            host: "test-rmmvmv6q.grpc.weaviate.network",
            apiKey: new ApiKey('tNPN1kOkrcmgCpogUYbtYKbYvH4hSTfjx4Fn'),
        });
        await WeaviateStore.fromTexts(
            textChunks,
            textChunks.map(chunk => ({ hash })),
            new OpenAIEmbeddings(),
            {
                client,
                indexName: "Test",
                textKey: "textChunks",
                metadataKeys: ["hash"],
            }
        );

        const query = query2;
        const queryEmbedding = await generateEmbeddings(query);
        console.log("This is query embedding", queryEmbedding);

        const similarEmbeddings = await getNearbyEmbeddings(query);
        console.log("This is similar embedding", similarEmbeddings);

        const apiUrlPrompt = await API_URL_PROMPT_TEMPLATE.format({
            api_docs: similarEmbeddings[0].pageContent,
            question: query,
        });

        const apiUrlResponse = await model.generate([apiUrlPrompt]);
        // console.log("API URL Response:", JSON.stringify(apiUrlResponse, null, 1));
      console.log(apiUrlResponse)
        const apiUrl = apiUrlResponse.generations[0][0]?.text;
        console.log("Generated API URL:", apiUrl);
        return apiUrl;
    } catch (error) {
        console.error('Error:', error);
    }
}
let ans = main()
return ans;
}

Description

I need to store in weAviate earlier I am using older cluster now I just changed my cluster in my weaviate and its is showing error I just change the keys here like host and apiKey of weAviate to make new client but getting error

Error: Error: Error adding vectors: Error: usage error (415):
at WeaviateStore.addVectors (file:///C:/Users/LakshyaTambi

System Info

ok

[dosubot[bot]]

To resolve the "Error adding vectors: Error: usage error (415)" when trying to store data in Weaviate after updating your cluster configuration, ensure that the documents you are trying to insert do not have an id in their metadata. The code explicitly throws an error if a document has an id in its metadata:

if (Object.hasOwn(document.metadata, "id"))
  throw new Error(
    "Document inserted to Weaviate vectorstore should not have `id` in their metadata."
  );

Make sure to remove any id fields from the document metadata before attempting to add them to the Weaviate index. Here is an example of how you might clean your documents before adding them:

documents.forEach(doc => {
  if (doc.metadata && doc.metadata.id) {
    delete doc.metadata.id;
  }
});

Then proceed with adding the documents:

await weaviateStore.addDocuments(documents);

Additionally, ensure that your metadata keys are valid GraphQL names, as invalid keys can also cause issues:

if (args.metadataKeys) {
  this.queryAttrs = [
    ...new Set([
      ...this.queryAttrs,
      ...args.metadataKeys.filter((k) => {
        const keyIsValid = /^[_A-Za-z][_0-9A-Za-z]*$/.test(k);
        if (!keyIsValid) {
          console.warn(
            `Skipping metadata key ${k} as it is not a valid GraphQL Name`
          );
        }
        return keyIsValid;
      }),
    ]),
  ];
}

By ensuring that your metadata keys are valid and removing any id fields from the document metadata, you should be able to resolve the error ^[1][2].

To continue talking to Dosu, mention @dosu.