How to connect with weaviate and stored the all the embedding

[lakshya29154]

Checked other resources

• I added a very descriptive title to this question.
• I searched the LangChain documentation with the integrated search.
• I used the GitHub search to find a similar question and didn't find it.

Commit to Help

• I commit to help with one of those options 👆

Example Code

import * as fs from "fs";
import { config } from 'dotenv';
import { createHash } from 'crypto';
import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";
import { OpenAIEmbeddings } from "@langchain/openai";
import weaviate, { ApiKey } from "weaviate-ts-client";
import { WeaviateStore } from "@langchain/weaviate"
// import { config } from "dotenv";
config();
// import { Weaviate } from 'weaviate-client';
const OPENAI_API_KEY = process.env.OPENAI_API_KEY;
// Function to generate SHA256 hash
function generateHash(text) {
    return createHash('sha256').update(text).digest('hex');
}
async function splitTextIntoChunks(text) {
    const splitter = new RecursiveCharacterTextSplitter({ chunkSize: 1000, chunkOverlap: 200 });
    const documents = await splitter.createDocuments([text]);
    return documents.map(doc => doc.pageContent);
}
// Function to generate embedding
async function generateEmbeddings(textChunks) {
    const embeddings = new OpenAIEmbeddings({ apiKey: process.env.OPENAI_API_KEY });
    const vectors = [];

    for (const chunk of textChunks) {
        const vector = await embeddings.embedQuery(chunk);
        console.log("Generated Vector:", vector); // Log to check the vector
        if (vector && vector.length > 0) {
            vectors.push({ chunk, vector });
        } else {
            console.error("Invalid or empty vector generated for chunk:", chunk);
        }
    }

    return vectors;
}

async function main() {
    try {
        const specFilePath = './swagger.json';
        const jsonData = fs.readFileSync(specFilePath, 'utf8');
        
        // Generate SHA256 hash of JSON data
        const hash = generateHash(jsonData);
        const textchunck = await splitTextIntoChunks(jsonData)
        // Generate embeddings for JSON data
        console.log("Text chunk", textchunck)
        const embeddings = await generateEmbeddings(textchunck);
        console.log("Embeddings", embeddings)
        
        // Initialize Weaviate client
        // const client = (weaviate).client({
        //     scheme: process.env.WEAVIATE_SCHEME || "https",
        //     host: "testing2-06tkquj2.weaviate.network",
        //     apiKey: new ApiKey('suIGpVYwQlUvUauG2TSyJhdcfcjpGdXFesCc'),
        //   });

        // // Store embeddings in Weaviate
        // await weaviate.graphql.insert({
        //     data: {
        //         hash,
        //         embeddings
        //     }
        // });

        // console.log('Embeddings stored successfully.');
    } catch (error) {
        console.error('Error:', error);
    }
}

main();

Description

Here now I want to connect to weaviate and store my all embedding in we aviate

System Info

all ok

[dosubot[bot]]

To connect to Weaviate and store your embeddings, you can use the WeaviateStore class from the @langchain/weaviate package. Below is an example of how to modify your code to achieve this:

1. Initialize the Weaviate Client: Create an instance of the Weaviate client.
2. Create the WeaviateStore Instance: Use the WeaviateStore class to create an instance, passing the necessary arguments including the Weaviate client, index name, and embeddings.
3. Add Documents: Use the fromTexts method to add your documents along with their embeddings to the Weaviate index.

Here is the updated code:

import * as fs from "fs";
import { config } from 'dotenv';
import { createHash } from 'crypto';
import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";
import { OpenAIEmbeddings } from "@langchain/openai";
import weaviate, { ApiKey } from "weaviate-ts-client";
import { WeaviateStore } from "@langchain/weaviate";

config();

const OPENAI_API_KEY = process.env.OPENAI_API_KEY;

function generateHash(text) {
    return createHash('sha256').update(text).digest('hex');
}

async function splitTextIntoChunks(text) {
    const splitter = new RecursiveCharacterTextSplitter({ chunkSize: 1000, chunkOverlap: 200 });
    const documents = await splitter.createDocuments([text]);
    return documents.map(doc => doc.pageContent);
}

async function generateEmbeddings(textChunks) {
    const embeddings = new OpenAIEmbeddings({ apiKey: process.env.OPENAI_API_KEY });
    const vectors = [];

    for (const chunk of textChunks) {
        const vector = await embeddings.embedQuery(chunk);
        console.log("Generated Vector:", vector); // Log to check the vector
        if (vector && vector.length > 0) {
            vectors.push({ chunk, vector });
        } else {
            console.error("Invalid or empty vector generated for chunk:", chunk);
        }
    }

    return vectors;
}

async function main() {
    try {
        const specFilePath = './swagger.json';
        const jsonData = fs.readFileSync(specFilePath, 'utf8');
        
        // Generate SHA256 hash of JSON data
        const hash = generateHash(jsonData);
        const textChunks = await splitTextIntoChunks(jsonData);
        console.log("Text chunk", textChunks);
        
        // Generate embeddings for JSON data
        const embeddings = await generateEmbeddings(textChunks);
        console.log("Embeddings", embeddings);
        
        // Initialize Weaviate client
        const client = weaviate.client({
            scheme: process.env.WEAVIATE_SCHEME || "https",
            host: process.env.WEAVIATE_HOST || "localhost",
            apiKey: new ApiKey(process.env.WEAVIATE_API_KEY || "default"),
        });

        // Create a store and fill it with some texts + metadata
        await WeaviateStore.fromTexts(
            textChunks,
            textChunks.map(chunk => ({ hash })),
            new OpenAIEmbeddings(),
            {
                client,
                indexName: "Test",
                textKey: "text",
                metadataKeys: ["hash"],
            }
        );

        console.log('Embeddings stored successfully.');
    } catch (error) {
        console.error('Error:', error);
    }
}

main();

This code initializes the Weaviate client, creates a WeaviateStore instance, and adds documents to the Weaviate index. Make sure to replace placeholders with your actual data and implementations ^[1][2][3][4].

To continue talking to Dosu, mention @dosu.

[lakshya29154]

import { config } from "dotenv";
config();

import fetch from "node-fetch";
import { JsonSpec } from "langchain/tools";
import { createOpenApiAgent, OpenApiToolkit } from "langchain/agents";
import { OpenAI } from "@langchain/openai";
import { API_URL_PROMPT_TEMPLATE, API_RESPONSE_PROMPT_TEMPLATE } from "./test20.js";

const OPENAI_API_KEY = process.env.OPENAI_API_KEY;
const TOKEN_API_KEY = process.env.TOKEN_API_KEY;
/**

• Function to create and execute an OpenAPI agent based on a Swagger URL and user query.

• @param swaggerUrl The URL of the Swagger specification.

• @param userQuery The user query to find the best suitable API.

• @returns The response from the API.
  */
  async function executeOpenApiAgent(swaggerUrl, userQuery) {
  // Load the OpenAPI specification from the Swagger URL
  const response = await fetch(swaggerUrl);
  if (!response.ok) {
  throw new Error(Failed to fetch Swagger spec: ${response.statusText});
  }
  const contentType = response.headers.get("content-type");
  if (!contentType.includes("application/json")) {
  throw new Error(Invalid content type: ${contentType}. Expected application/json.);
  }
  const data = await response.json();

  // Initialize the language model
  const model = new OpenAI({ temperature: 0, apiKey: OPENAI_API_KEY });

  // Create the OpenAPI toolkit
  const headers = {
  "Content-Type": "application/json",
  Authorization: Bearer ${TOKEN_API_KEY},
  };
  const toolkit = new OpenApiToolkit(new JsonSpec(data), model, headers);

  // Create the OpenAPI agent
  // const executor = createOpenApiAgent(model, toolkit);

  // Generate the API URL using the prompt template
  const apiUrlPrompt = await API_URL_PROMPT_TEMPLATE.format({
  api_docs: JSON.stringify(data),
  question: userQuery,
  });

  // console.log("API URL Prompt:", apiUrlPrompt);

  const apiUrlResponse = await model.generate([apiUrlPrompt]);
  console.log("API URL Response:", JSON.stringify(apiUrlResponse, null, 2));

  // Extract the generated URL from the response
  const apiUrl = apiUrlResponse.generations[0][0]?.text
  console.log("Generated API URL:", apiUrl);
  }
  // if (!apiUrl || !apiUrl.startsWith('http')) {
  // throw new Error(Invalid API URL generated: ${apiUrl});
  // }

  // Execute the API call
  // const apiResponse = await fetch(apiUrl, { headers });
  // if (!apiResponse.ok) {
  // throw new Error(Failed to fetch API endpoint: ${apiResponse.statusText});
  // }
  // const apiResponseData = await apiResponse.json();
  // console.log("API Response Data:", apiResponseData);

// // Summarize the API response using the prompt template
// const apiResponsePrompt = await API_RESPONSE_PROMPT_TEMPLATE.format({
// api_docs: JSON.stringify(data),
// question: userQuery,
// api_url: apiUrl,
// api_response: JSON.stringify(apiResponseData),
// });

// // console.log("API Response Prompt:", apiResponsePrompt);

// const summaryResponse = await model.generate([apiResponsePrompt]);
// // console.log("Summary Response:", JSON.stringify(summaryResponse, null, 2));

// const summary = summaryResponse.generations[0][0]?.text.trim();
// // console.log("Summary:", summary);

// // Return the summarized response
// return summary;
// } catch (error) {
// // console.error('Error executing OpenAPI agent:', error);
// throw error;
// }
// }

// Example usage
const swaggerUrl = 'https://petstore.swagger.io/v2/swagger.json'; // Ensure this points to a JSON spec
const userQuery = 'Can you give me the Api to find the pet by status and then place an order for the pet and then delete the pet?';
executeOpenApiAgent(swaggerUrl, userQuery)
.then(response =>"")
// .catch(error => console.error('Error:', error));

Now there will be a userquery like Can you provide me the endpoint to get the pet by id
We need to make this also in embedding get all the embeddinf from weaviate match from this and give the fetch embedding and question to open ai to generate resposne