Error in storing the vector #5723
Replies: 1 comment 3 replies
-
|
Please help |
Beta Was this translation helpful? Give feedback.
-
|
The error "Error: usage error (415)" typically indicates that the media type is unsupported, which can happen if the index is not properly set up or if there is a mismatch in the expected data format. To resolve this, ensure that the Here is the relevant part of your code: await WeaviateStore.fromTexts(
textChunks,
textChunks.map(async chunk => {
const hash = await generateHash(chunk);
return { hash };
}),
new OpenAIEmbeddings(),
{
client,
indexName: "Test", // Ensure this index exists in your Weaviate instance
textKey: "text",
metadataKeys: ["hash"],
}
);Make sure that the index "Test" is created in your Weaviate instance with the appropriate schema before running this code. You can create the index using the Weaviate console or via the Weaviate API. If the index "Test" does not exist, you can create it using the following example schema: {
"class": "Test",
"properties": [
{
"name": "text",
"dataType": ["text"]
},
{
"name": "hash",
"dataType": ["string"]
}
]
}You can add this schema to your Weaviate instance using the Weaviate client or API. Here is an example of how to add the schema using the Weaviate client in TypeScript: const schemaConfig = {
class: "Test",
properties: [
{
name: "text",
dataType: ["text"]
},
{
name: "hash",
dataType: ["string"]
}
]
};
await client.schema.classCreator().withClass(schemaConfig).do();Ensure that you run this schema creation code before calling Additionally, ensure that the metadata keys you are using are valid GraphQL names, as the if (args.metadataKeys) {
this.queryAttrs = [
...new Set([
...this.queryAttrs,
...args.metadataKeys.filter((k) => {
// https://spec.graphql.org/June2018/#sec-Names
// queryAttrs need to be valid GraphQL Names
const keyIsValid = /^[_A-Za-z][_0-9A-Za-z]*$/.test(k);
if (!keyIsValid) {
console.warn(
`Skipping metadata key ${k} as it is not a valid GraphQL Name`
);
}
return keyIsValid;
}),
]),
];
}This ensures that only valid metadata keys are used in the query attributes, which can help prevent errors related to unsupported media types [1][2]. |
Beta Was this translation helpful? Give feedback.
-
|
Can you give me the full code |
Beta Was this translation helpful? Give feedback.
-
|
To resolve the 415 error when trying to store vectors in Weaviate, you need to ensure that the schema is correctly set up and that the data being sent matches the expected format. Below is the full code, including the schema creation and necessary setup, to store vectors in Weaviate. Schema CreationFirst, create the schema in Weaviate. This example assumes you have a class named import weaviate from 'weaviate-client';
const client = weaviate.client({
scheme: 'https',
host: 'swagger01-bybjaoyp.grpc.weaviate.network',
apiKey: new weaviate.ApiKey('jmIEZWKS5RXJbwpQBCtsTDI5afQMQRBCKIA3'),
});
const schemaConfig = {
class: 'Test',
properties: [
{
name: 'text',
dataType: ['text'],
},
{
name: 'hash',
dataType: ['string'],
},
],
};
client.schema
.classCreator()
.withClass(schemaConfig)
.do()
.then((res) => {
console.log('Schema created:', res);
})
.catch((err) => {
console.error('Error creating schema:', err);
});Vector Store SetupNext, set up the vector store and add vectors and documents. import * as uuid from 'uuid';
import { WeaviateClient, WeaviateObject } from 'weaviate-ts-client';
import { Document } from '@langchain/core/documents';
import { EmbeddingsInterface } from '@langchain/core/embeddings';
export const flattenObjectForWeaviate = (obj: Record<string, any>) => {
const flattenedObject: Record<string, any> = {};
for (const key in obj) {
if (!Object.hasOwn(obj, key)) {
continue;
}
const value = obj[key];
if (typeof obj[key] === 'object' && !Array.isArray(value)) {
const recursiveResult = flattenObjectForWeaviate(value);
for (const deepKey in recursiveResult) {
if (Object.hasOwn(obj, key)) {
flattenedObject[`${key}_${deepKey}`] = recursiveResult[deepKey];
}
}
} else if (Array.isArray(value)) {
if (value.length === 0) {
flattenedObject[key] = value;
} else if (
typeof value[0] !== 'object' &&
value.every((el: any) => typeof el === typeof value[0])
) {
flattenedObject[key] = value;
}
} else {
flattenedObject[key] = value;
}
}
return flattenedObject;
};
export interface WeaviateLibArgs {
client: WeaviateClient;
indexName: string;
textKey?: string;
metadataKeys?: string[];
tenant?: string;
}
export class WeaviateStore {
private client: WeaviateClient;
private indexName: string;
private textKey: string;
private queryAttrs: string[];
private tenant?: string;
constructor(public embeddings: EmbeddingsInterface, args: WeaviateLibArgs) {
this.client = args.client;
this.indexName = args.indexName;
this.textKey = args.textKey || 'text';
this.queryAttrs = [this.textKey];
this.tenant = args.tenant;
if (args.metadataKeys) {
this.queryAttrs = [
...new Set([
...this.queryAttrs,
...args.metadataKeys.filter((k) => {
// https://spec.graphql.org/June2018/#sec-Names
// queryAttrs need to be valid GraphQL Names
const keyIsValid = /^[_A-Za-z][_0-9A-Za-z]*$/.test(k);
if (!keyIsValid) {
console.warn(
`Skipping metadata key ${k} as it is not a valid GraphQL Name`
);
}
return keyIsValid;
}),
]),
];
}
}
async addVectors(
vectors: number[][],
documents: Document[],
options?: { ids?: string[] }
) {
const documentIds = options?.ids ?? documents.map((_) => uuid.v4());
const batch: WeaviateObject[] = documents.map((document, index) => {
if (Object.hasOwn(document.metadata, 'id'))
throw new Error(
'Document inserted to Weaviate vectorstore should not have `id` in their metadata.'
);
const flattenedMetadata = flattenObjectForWeaviate(document.metadata);
return {
...(this.tenant ? { tenant: this.tenant } : {}),
class: this.indexName,
id: documentIds[index],
vector: vectors[index],
properties: {
[this.textKey]: document.pageContent,
...flattenedMetadata,
},
};
});
try {
const responses = await this.client.batch
.objectsBatcher()
.withObjects(...batch)
.do();
// if storing vectors fails, we need to know why
const errorMessages: string[] = [];
responses.forEach((response) => {
if (response?.result?.errors?.error) {
errorMessages.push(
...response.result.errors.error.map(
(err) =>
err.message ??
'!! Unfortunately no error message was presented in the API response !!'
)
);
}
});
if (errorMessages.length > 0) {
throw new Error(errorMessages.join('\n'));
}
} catch (e) {
throw Error(`Error adding vectors: ${e}`);
}
return documentIds;
}
async addDocuments(documents: Document[], options?: { ids?: string[] }) {
return this.addVectors(
await this.embeddings.embedDocuments(documents.map((d) => d.pageContent)),
documents,
options
);
}
async delete(params: {
ids?: string[];
filter?: WeaviateFilter;
}): Promise<void> {
const { ids, filter } = params;
if (ids && ids.length > 0) {
for (const id of ids) {
let deleter = this.client.data
.deleter()
.withClassName(this.indexName)
.withId(id);
if (this.tenant) {
deleter = deleter.withTenant(this.tenant);
}
await deleter.do();
}
} else if (filter) {
let batchDeleter = this.client.batch
.objectsBatchDeleter()
.withClassName(this.indexName)
.withWhere(filter.where);
if (this.tenant) {
batchDeleter = batchDeleter.withTenant(this.tenant);
}
await batchDeleter.do();
} else {
throw new Error(
`This method requires either "ids" or "filter" to be set in the input object`
);
}
}
async similaritySearchVectorWithScore(
query: number[],
k: number,
filter?: WeaviateFilter
): Promise<[Document, number][]> {
const resultsWithEmbedding =
await this.similaritySearchVectorWithScoreAndEmbedding(query, k, filter);
return resultsWithEmbedding.map(([document, score, _embedding]) => [
document,
score,
]);
}
async similaritySearchVectorWithScoreAndEmbedding(
query: number[],
k: number,
filter?: WeaviateFilter
): Promise<[Document, number, number[]][]> {
try {
let builder = this.client.graphql
.get()
.withClassName(this.indexName)
.withFields(
`${this.queryAttrs.join(' ')} _additional { distance vector }`
)
.withNearVector({
vector: query,
distance: filter?.distance,
})
.withLimit(k);
if (this.tenant) {
builder = builder.withTenant(this.tenant);
}
if (filter?.where) {
builder = builder.withWhere(filter.where);
}
const result = await builder.do();
const documents: [Document, number, number[]][] = [];
for (const data of result.data.Get[this.indexName]) {
const { [this.textKey]: text, _additional, ...rest }: ResultRow = data;
documents.push([
new Document({
pageContent: text,
metadata: rest,
}),
_additional.distance,
_additional.vector,
]);
}
return documents;
} catch (e) {
throw Error(`'Error in similaritySearch' ${e}`);
}
}
override async maxMarginalRelevanceSearch(
query: string,
options: MaxMarginalRelevanceSearchOptions<this['FilterType']>,
_callbacks?: undefined
): Promise<Document[]> {
const { k, fetchK = 20, lambda = 0.5, filter } = options;
const queryEmbedding: number[] = await this.embeddings.embedQuery(query);
const allResults: [Document, number, number[]][] =
await this.similaritySearchVectorWithScoreAndEmbedding(
queryEmbedding,
fetchK,
filter
);
const embeddingList = allResults.map(
([_doc, _score, embedding]) => embedding
);
const mmrIndexes = maximalMarginalRelevance(
queryEmbedding,
embeddingList,
lambda,
k
);
return mmrIndexes
.filter((idx) => idx !== -1)
.map((idx) => allResults[idx][0]);
}
static fromTexts(
texts: string[],
metadatas: object | object[],
embeddings: EmbeddingsInterface,
args: WeaviateLibArgs
): Promise<WeaviateStore> {
const docs: Document[] = [];
for (let i = 0; i < texts.length; i += 1) {
const metadata = Array.isArray(metadatas) ? metadatas[i] : metadatas;
const newDoc = new Document({
pageContent: texts[i],
metadata,
});
docs.push(newDoc);
}
return WeaviateStore.fromDocuments(docs, embeddings, args);
}
static async fromDocuments(
docs: Document[],
embeddings: EmbeddingsInterface,
args: WeaviateLibArgs
): Promise<WeaviateStore> {
const instance = new this(embeddings, args);
await instance.addDocuments(docs);
return instance;
}
static async fromExistingIndex(
embeddings: EmbeddingsInterface,
args: WeaviateLibArgs
): Promise<WeaviateStore> {
return new this(embeddings, args);
}
}This code sets up the schema in Weaviate and provides the necessary methods to add vectors and documents to the Weaviate index. Make sure to replace the placeholders with your actual data and configurations [1][2]. |
Beta Was this translation helpful? Give feedback.
-
Checked other resources
Commit to Help
Example Code
Description
I am getting error in storing the vector here Error: Error: Error adding vectors: Error: usage error (415):
at WeaviateStore.addVectors (file:///C:/Users/LakshyaTambi/Desktop/try/node_modules/@langchain/weaviate/dist/vectorstores.js:155:19)
at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
at async WeaviateStore.fromDocuments (file:///C:/Users/LakshyaTambi/Desktop/try/node_modules/@langchain/weaviate/dist/vectorstores.js:322:9)
at async storeEmbedding (file:///C:/Users/LakshyaTambi/Desktop/try/test26.js:63:5)
at async main (file:///C:/Users/LakshyaTambi/Desktop/try/test26.js:127:9)
PS C:\Users\LakshyaTambi\Desktop\try>
System Info
all ok
Beta Was this translation helpful? Give feedback.
All reactions