diff --git a/docs/asciidoc/modules/ROOT/pages/database-integration/vectordb/chroma.adoc b/docs/asciidoc/modules/ROOT/pages/database-integration/vectordb/chroma.adoc index 0da0ea1d3a..e9d8694dbc 100644 --- a/docs/asciidoc/modules/ROOT/pages/database-integration/vectordb/chroma.adoc +++ b/docs/asciidoc/modules/ROOT/pages/database-integration/vectordb/chroma.adoc @@ -7,6 +7,7 @@ note that the list and the signature procedures are consistent with the others, [opts=header, cols="1, 3"] |=== | name | description +| apoc.vectordb.chroma.info(hostOrKey, collection, $config) | Get information about the specified existing collection or throws an error 500 if it does not exist | apoc.vectordb.chroma.createCollection(hostOrKey, collection, similarity, size, $config) | Creates a collection, with the name specified in the 2nd parameter, and with the specified `similarity` and `size`. The default endpoint is `/api/v1/collections`. @@ -38,6 +39,19 @@ With hostOrKey=null, the default is 'http://localhost:8000'. == Examples +.Get collection info (it leverages https://docs.trychroma.com/reference/py-client#get_collection[this API]) +[source,cypher] +---- +CALL apoc.vectordb.chroma.info(hostOrKey, 'test_collection', {}) +---- + +.Example results +[opts="header"] +|=== +| value +| {"name": "test_collection", "metadata": {"size": 4, "hnsw:space": "cosine"}, "database": "default_database", "id": "74ebe008-1ccb-4d3d-8c5d-cdd7cfa526c2", "tenant": "default_tenant"} +|=== + .Create a collection (it leverages https://docs.trychroma.com/usage-guide#creating-inspecting-and-deleting-collections[this API]) [source,cypher] ---- diff --git a/docs/asciidoc/modules/ROOT/pages/database-integration/vectordb/milvus.adoc b/docs/asciidoc/modules/ROOT/pages/database-integration/vectordb/milvus.adoc index 971971856c..948349fde7 100644 --- a/docs/asciidoc/modules/ROOT/pages/database-integration/vectordb/milvus.adoc +++ b/docs/asciidoc/modules/ROOT/pages/database-integration/vectordb/milvus.adoc @@ -6,6 +6,7 @@ Here is a list of all available Milvus procedures: [opts=header, cols="1, 3"] |=== | name | description +| apoc.vectordb.milvus.info(hostOrKey, collection, $config) | Get information about the specified existing collection or returns a response with code 100 if it does not exist | apoc.vectordb.milvus.createCollection(hostOrKey, collection, similarity, size, $config) | Creates a collection, with the name specified in the 2nd parameter, and with the specified `similarity` and `size`. The default endpoint is `/v2/vectordb/collections/create`. @@ -39,6 +40,25 @@ With hostOrKey=null, the default host is 'http://localhost:19530'. Here is a list of example using a local installation using th default port `19531`. +.Get collection info (it leverages https://milvus.io/docs/manage-collections.md#View-Collections[this API]) +[source,cypher] +---- +CALL apoc.vectordb.milvus.info($host, 'test_collection', '', {}) +---- + +.Example results +[opts="header"] +|=== +| value +| {"data": {"shardsNum": 1, "aliases": [], "autoId": false, "description": "", "partitionsNum": 1, "collectionName": "test_collection", + "indexes": [{"metricType": "COSINE", "indexName": "vector", "fieldName": "vector"}], + "load": "LoadStateLoading", "consistencyLevel": "Bounded", + "fields": [{"partitionKey": false, "autoId": false, "name": "id", "description": "", "id": 100, "type": "Int64", "primaryKey": true}, + {"partitionKey": false, "autoId": false, "name": "vector", "description": "", "id": 101, "params": [{"value": 4, "key": "dim"}], "type": "FloatVector", "primaryKey": false} + ], + "collectionID": "451046728334049293", "enableDynamicField": true, "properties": []}, "message": "", "code": 200 +} +|=== .Create a collection (it leverages https://milvus.io/api-reference/restful/v2.4.x/v2/Collection%20(v2)/Create.md[this API]) [source,cypher] diff --git a/docs/asciidoc/modules/ROOT/pages/database-integration/vectordb/pinecone.adoc b/docs/asciidoc/modules/ROOT/pages/database-integration/vectordb/pinecone.adoc index 8972ce404d..f4a4b3c494 100644 --- a/docs/asciidoc/modules/ROOT/pages/database-integration/vectordb/pinecone.adoc +++ b/docs/asciidoc/modules/ROOT/pages/database-integration/vectordb/pinecone.adoc @@ -6,6 +6,7 @@ Here is a list of all available Pinecone procedures: [opts=header, cols="1, 3"] |=== | name | description +| apoc.vectordb.pinecone.info(hostOrKey, collection, $config) | Get information about the specified existing collection or throws a 404 error if it does not exist | apoc.vectordb.pinecone.createCollection(hostOrKey, index, similarity, size, $config) | Creates an index, with the name specified in the 2nd parameter, and with the specified `similarity` and `size`. The default endpoint is `/indexes`. @@ -54,6 +55,26 @@ image::pinecone-index.png[width=800] The following example assume we want to create and manage an index called `test-index`. +.Get collection info (it leverages https://docs.pinecone.io/reference/api/control-plane/describe_collection[this API]) +[source,cypher] +---- +CALL apoc.vectordb.pinecone.info(hostOrKey, 'test-collection', {}) +---- + +.Example results +[opts="header"] +|=== +| value +| { "dimension": 3, + "environment": "us-east1-gcp", + "name": "tiny-collection", + "size": 3126700, + "status": "Ready", + "vector_count": 99 +} +|=== + + .Create an index (it leverages https://docs.pinecone.io/reference/api/control-plane/create_index[this API]) [source,cypher] ---- diff --git a/docs/asciidoc/modules/ROOT/pages/database-integration/vectordb/qdrant.adoc b/docs/asciidoc/modules/ROOT/pages/database-integration/vectordb/qdrant.adoc index a000605e7f..99e5ba5898 100644 --- a/docs/asciidoc/modules/ROOT/pages/database-integration/vectordb/qdrant.adoc +++ b/docs/asciidoc/modules/ROOT/pages/database-integration/vectordb/qdrant.adoc @@ -7,6 +7,7 @@ note that the list and the signature procedures are consistent with the others, [opts=header, cols="1, 3"] |=== | name | description +| apoc.vectordb.qdrant.info(hostOrKey, collection, $config) | Get information about the specified existing collection or throws a FileNotFoundException if it does not exist | apoc.vectordb.qdrant.createCollection(hostOrKey, collection, similarity, size, $config) | Creates a collection, with the name specified in the 2nd parameter, and with the specified `similarity` and `size`. The default endpoint is `/collections/`. @@ -39,6 +40,29 @@ With hostOrKey=null, the default is 'http://localhost:6333'. == Examples +.Get collection info (it leverages https://qdrant.github.io/qdrant/redoc/index.html#tag/collections/operation/get_collection[this API]) +[source,cypher] +---- +CALL apoc.vectordb.qdrant.info(hostOrKey, 'test_collection', {}) +---- + +.Example results +[opts="header"] +|=== +| value +| {"result": {"optimizer_status": "ok", "points_count": 2, "vectors_count": 2, "segments_count": 8, "indexed_vectors_count": 0, + "config": {"params": {"on_disk_payload": true, "vectors": {"size": 4, "distance": "Cosine"}, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1}, + "optimizer_config": {"max_optimization_threads": 1, "indexing_threshold": 20000, "deleted_threshold": 0.2, "flush_interval_sec": 5, "memmap_threshold": null, "default_segment_number": 0, "max_segment_size": null, "vacuum_min_vector_number": 1000}, "quantization_config": null, + "hnsw_config": {"max_indexing_threads": 0, "full_scan_threshold": 10000, "ef_construct": 100, "m": 16, "on_disk": false}, + "wal_config": {"wal_segments_ahead": 0, "wal_capacity_mb": 32} + }, + "status": green, + "payload_schema": {} + }, + "time": 1.2725E-4, "status": ok +} +|=== + .Create a collection (it leverages https://qdrant.github.io/qdrant/redoc/index.html#tag/collections/operation/create_collection[this API]) [source,cypher] ---- diff --git a/docs/asciidoc/modules/ROOT/pages/database-integration/vectordb/weaviate.adoc b/docs/asciidoc/modules/ROOT/pages/database-integration/vectordb/weaviate.adoc index 8235fe9124..53e2209401 100644 --- a/docs/asciidoc/modules/ROOT/pages/database-integration/vectordb/weaviate.adoc +++ b/docs/asciidoc/modules/ROOT/pages/database-integration/vectordb/weaviate.adoc @@ -7,6 +7,7 @@ note that the list and the signature procedures are consistent with the others, [opts=header, cols="1, 3"] |=== | name | description +| apoc.vectordb.weaviate.info($host, $collectionName, $config) | Get information about the specified existing collection or throws a FileNotFoundException if it does not exist | apoc.vectordb.weaviate.createCollection(hostOrKey, collection, similarity, size, $config) | Creates a collection, with the name specified in the 2nd parameter, and with the specified `similarity` and `size`. The default endpoint is `/schema`. @@ -40,6 +41,33 @@ With hostOrKey=null, the default is 'http://localhost:8080/v1'. == Examples +.Get collection info (it leverages https://weaviate.io/developers/weaviate/api/rest#tag/schema/get/schema/{className}[this API]) +[source, cypher] +---- +CALL apoc.vectordb.weaviate.info($host, 'test_collection', {}) +---- + +.Example results +[opts="header"] +|=== +| value +| {"vectorizer": "none", + "invertedIndexConfig": {"bm25": {"b": 0.75, "k1": 1.2}, "stopwords": {"additions": null, "removals": null, "preset": en}, "cleanupIntervalSeconds": 60}, + "vectorIndexConfig": {"ef": -1, "dynamicEfMin": 100, "pq": {"centroids": 256, "trainingLimit": 100000, "encoder": {"type": "kmeans", "distribution": "log-normal"}, + "enabled": false, "bitCompression": false, "segments": 0 + }, + "distance": cosine, "skip": false, "dynamicEfFactor": 8, "bq": {"enabled": false}, + "vectorCacheMaxObjects": 1000000000000, "cleanupIntervalSeconds": 300, "dynamicEfMax": 500, "efConstruction": 128, "flatSearchCutoff": 40000, "maxConnections": 64}, + "multiTenancyConfig": {"enabled": false}, + "vectorIndexType": "hnsw", "replicationConfig": {"factor": 1}, + "shardingConfig": {"desiredVirtualCount": 128, "desiredCount": 1, "actualCount": 1, "function": "murmur3", "virtualPerPhysical": 128, "strategy": "hash", "actualVirtualCount": 128, "key": "_id"}, + "class": "TestCollection", + "properties": [{"name": "city", "description": "This property was generated by Weaviate's auto-schema feature on Wed Jul 10 12:50:18 2024", "indexFilterable": true, "tokenization": "word", "indexSearchable": true, "dataType": ["text"]}, + {"name": "foo", "description": "This property was generated by Weaviate's auto-schema feature on Wed Jul 10 12:50:18 2024", "indexFilterable": true, "tokenization": word, "indexSearchable": true, "dataType": ["text"]} + ] +} +|=== + .Create a collection (it leverages https://weaviate.io/developers/weaviate/api/rest#tag/schema/post/schema[this API]) [source,cypher] ---- diff --git a/extended-it/src/test/java/apoc/vectordb/ChromaDbTest.java b/extended-it/src/test/java/apoc/vectordb/ChromaDbTest.java index 291caf8424..2ceede5c3f 100644 --- a/extended-it/src/test/java/apoc/vectordb/ChromaDbTest.java +++ b/extended-it/src/test/java/apoc/vectordb/ChromaDbTest.java @@ -19,6 +19,7 @@ import static apoc.ml.Prompt.API_KEY_CONF; import static apoc.ml.RestAPIConfig.HEADERS_KEY; +import static apoc.util.ExtendedTestUtil.assertFails; import static apoc.util.MapUtil.map; import static apoc.util.TestUtil.testCall; import static apoc.util.TestUtil.testResult; @@ -29,7 +30,6 @@ import static apoc.vectordb.VectorDbTestUtil.assertBerlinResult; import static apoc.vectordb.VectorDbTestUtil.assertLondonResult; import static apoc.vectordb.VectorDbTestUtil.assertNodesCreated; -import static apoc.vectordb.VectorDbTestUtil.assertRagWithVectors; import static apoc.vectordb.VectorDbTestUtil.assertReadOnlyProcWithMappingResults; import static apoc.vectordb.VectorDbTestUtil.assertRelsCreated; import static apoc.vectordb.VectorDbTestUtil.dropAndDeleteAll; @@ -41,7 +41,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; import static org.neo4j.configuration.GraphDatabaseSettings.DEFAULT_DATABASE_NAME; import static org.neo4j.configuration.GraphDatabaseSettings.SYSTEM_DATABASE_NAME; @@ -50,6 +49,7 @@ public class ChromaDbTest { private static final ChromaDBContainer CHROMA_CONTAINER = new ChromaDBContainer("chromadb/chroma:0.4.25.dev137"); private static final String READONLY_KEY = "my_readonly_api_key"; private static final Map READONLY_AUTHORIZATION = getAuthHeader(READONLY_KEY); + private static final String COLLECTION_NAME = "test_collection"; private static String HOST; @@ -109,6 +109,24 @@ public static void tearDown() throws Exception { public void before() { dropAndDeleteAll(db); } + + @Test + public void getInfo() { + testResult(db, "CALL apoc.vectordb.chroma.info($host, $collection, $conf) ", + map("host", HOST, "collection", COLLECTION_NAME, "conf", map(ALL_RESULTS_KEY, true)), + r -> { + Map row = (Map) r.next().get("value"); + assertEquals(COLLECTION_NAME, row.get("name")); + }); + } + + @Test + public void getInfoNotExistentCollection() { + assertFails(db, "CALL apoc.vectordb.chroma.info($host, 'wrong_collection', $conf) ", + map("host", HOST, "collection", COLLECTION_NAME, "conf", map(ALL_RESULTS_KEY, true)), + "Server returned HTTP response code: 500" + ); + } @Test public void getVectors() { diff --git a/extended-it/src/test/java/apoc/vectordb/MilvusTest.java b/extended-it/src/test/java/apoc/vectordb/MilvusTest.java index 2597c6cd7f..d120da66ef 100644 --- a/extended-it/src/test/java/apoc/vectordb/MilvusTest.java +++ b/extended-it/src/test/java/apoc/vectordb/MilvusTest.java @@ -117,6 +117,28 @@ public void before() { dropAndDeleteAll(db); } + @Test + public void getInfo() { + testResult(db, "CALL apoc.vectordb.milvus.info($host, 'test_collection', '', $conf) ", + map("host", HOST, "conf", map(FIELDS_KEY, FIELDS)), + r -> { + Map row = r.next(); + Map value = (Map) row.get("value"); + assertEquals(200L, value.get("code")); + }); + } + + @Test + public void getInfoNotExistentCollection() { + testResult(db, "CALL apoc.vectordb.milvus.info($host, 'wrong_collection', '', $conf) ", + map("host", HOST, "conf", map(FIELDS_KEY, FIELDS)), + r -> { + Map row = r.next(); + Map value = (Map) row.get("value"); + assertEquals(100L, value.get("code")); + }); + } + @Test public void getVectorsWithoutVectorResult() { testResult(db, "CALL apoc.vectordb.milvus.get($host, 'test_collection', [1], $conf) ", diff --git a/extended-it/src/test/java/apoc/vectordb/QdrantTest.java b/extended-it/src/test/java/apoc/vectordb/QdrantTest.java index d11432c6f9..16b547b3a1 100644 --- a/extended-it/src/test/java/apoc/vectordb/QdrantTest.java +++ b/extended-it/src/test/java/apoc/vectordb/QdrantTest.java @@ -1,5 +1,6 @@ package apoc.vectordb; +import apoc.ml.Prompt; import apoc.util.TestUtil; import apoc.util.Util; import org.junit.AfterClass; @@ -17,15 +18,15 @@ import java.util.List; import java.util.Map; -import apoc.ml.Prompt; -import static apoc.ml.RestAPIConfig.HEADERS_KEY; import static apoc.ml.Prompt.API_KEY_CONF; +import static apoc.ml.RestAPIConfig.HEADERS_KEY; +import static apoc.util.ExtendedTestUtil.assertFails; import static apoc.util.MapUtil.map; import static apoc.util.TestUtil.testCall; import static apoc.util.TestUtil.testResult; import static apoc.vectordb.VectorDbHandler.Type.QDRANT; -import static apoc.vectordb.VectorDbTestUtil.EntityType.NODE; import static apoc.vectordb.VectorDbTestUtil.EntityType.FALSE; +import static apoc.vectordb.VectorDbTestUtil.EntityType.NODE; import static apoc.vectordb.VectorDbTestUtil.EntityType.REL; import static apoc.vectordb.VectorDbTestUtil.assertBerlinResult; import static apoc.vectordb.VectorDbTestUtil.assertLondonResult; @@ -43,7 +44,8 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import static org.neo4j.configuration.GraphDatabaseSettings.*; +import static org.neo4j.configuration.GraphDatabaseSettings.DEFAULT_DATABASE_NAME; +import static org.neo4j.configuration.GraphDatabaseSettings.SYSTEM_DATABASE_NAME; public class QdrantTest { private static final String ADMIN_KEY = "my_admin_api_key"; @@ -117,6 +119,29 @@ public static void tearDown() throws Exception { public void before() { dropAndDeleteAll(db); } + + @Test + public void getInfo() { + testResult( + db, + "CALL apoc.vectordb.qdrant.info($host, 'test_collection', $conf)", + map("host", HOST, "conf", ADMIN_HEADER_CONF), + r -> { + Map res = r.next(); + Map value = (Map) res.get("value"); + assertEquals("ok", value.get("status")); + }); + } + + @Test + public void getInfoNotExistentCollection() { + assertFails( + db, + "CALL apoc.vectordb.qdrant.info($host, 'wrong_collection', $conf)", + map("host", HOST, "conf", ADMIN_HEADER_CONF), + "java.io.FileNotFoundException" + ); + } @Test public void getVectorsWithReadOnlyApiKey() { diff --git a/extended-it/src/test/java/apoc/vectordb/WeaviateTest.java b/extended-it/src/test/java/apoc/vectordb/WeaviateTest.java index b76869d003..d9b725d5ca 100644 --- a/extended-it/src/test/java/apoc/vectordb/WeaviateTest.java +++ b/extended-it/src/test/java/apoc/vectordb/WeaviateTest.java @@ -22,6 +22,7 @@ import static apoc.ml.Prompt.API_KEY_CONF; import static apoc.ml.RestAPIConfig.HEADERS_KEY; +import static apoc.util.ExtendedTestUtil.assertFails; import static apoc.util.TestUtil.testCall; import static apoc.util.TestUtil.testCallEmpty; import static apoc.util.TestUtil.testResult; @@ -56,6 +57,7 @@ public class WeaviateTest { private static final List FIELDS = List.of("city", "foo"); private static final String ADMIN_KEY = "jane-secret-key"; private static final String READONLY_KEY = "ian-secret-key"; + private static final String COLLECTION_NAME = "TestCollection"; private static final WeaviateContainer WEAVIATE_CONTAINER = new WeaviateContainer("semitechnologies/weaviate:1.24.5") .withEnv("AUTHENTICATION_APIKEY_ENABLED", "true") @@ -114,10 +116,10 @@ public static void setUp() throws Exception { MapUtil.map("host", HOST, "id1", ID_1, "id2", ID_2, "conf", ADMIN_HEADER_CONF), r -> { ResourceIterator values = r.columnAs("value"); - assertEquals("TestCollection", values.next().get("class")); - assertEquals("TestCollection", values.next().get("class")); - assertEquals("TestCollection", values.next().get("class")); - assertEquals("TestCollection", values.next().get("class")); + assertEquals(COLLECTION_NAME, values.next().get("class")); + assertEquals(COLLECTION_NAME, values.next().get("class")); + assertEquals(COLLECTION_NAME, values.next().get("class")); + assertEquals(COLLECTION_NAME, values.next().get("class")); assertFalse(values.hasNext()); }); @@ -134,8 +136,8 @@ public static void setUp() throws Exception { @AfterClass public static void tearDown() throws Exception { - testCallEmpty(db, "CALL apoc.vectordb.weaviate.deleteCollection($host, 'TestCollection', $conf)", - MapUtil.map("host", HOST, "conf", ADMIN_HEADER_CONF) + testCallEmpty(db, "CALL apoc.vectordb.weaviate.deleteCollection($host, $collectionName, $conf)", + MapUtil.map("host", HOST, "collectionName", COLLECTION_NAME, "conf", ADMIN_HEADER_CONF) ); WEAVIATE_CONTAINER.stop(); @@ -147,6 +149,27 @@ public void before() { dropAndDeleteAll(db); } + @Test + public void getInfo() { + testResult(db, "CALL apoc.vectordb.weaviate.info($host, $collectionName, $conf)", + map("host", HOST, "collectionName", COLLECTION_NAME, "conf", map(ALL_RESULTS_KEY, true, HEADERS_KEY, READONLY_AUTHORIZATION)), + r -> { + Map row = r.next(); + Map value = (Map) row.get("value"); + assertEquals(COLLECTION_NAME, value.get("class")); + }); + } + + @Test + public void getInfoNotExistentCollection() { + assertFails( + db, + "CALL apoc.vectordb.weaviate.info($host, 'wrong_collection', $conf)", + map("host", HOST, "collectionName", COLLECTION_NAME, "conf", map(ALL_RESULTS_KEY, true, HEADERS_KEY, READONLY_AUTHORIZATION)), + "java.io.FileNotFoundException" + ); + } + @Test public void getVectorsWithReadOnlyApiKey() { testResult(db, "CALL apoc.vectordb.weaviate.get($host, 'TestCollection', [$id1], $conf)", diff --git a/extended/src/main/java/apoc/vectordb/ChromaDb.java b/extended/src/main/java/apoc/vectordb/ChromaDb.java index 23d8d1e996..beee123d66 100644 --- a/extended/src/main/java/apoc/vectordb/ChromaDb.java +++ b/extended/src/main/java/apoc/vectordb/ChromaDb.java @@ -45,6 +45,22 @@ public class ChromaDb { @Context public URLAccessChecker urlAccessChecker; + @Procedure("apoc.vectordb.chroma.info") + @Description("apoc.vectordb.chroma.info(hostOrKey, collection, $configuration) - Get information about the specified existing collection or throws an error if it does not exist") + public Stream info(@Name("hostOrKey") String hostOrKey, @Name("collection") String collection, @Name(value = "configuration", defaultValue = "{}") Map configuration) throws Exception { + String url = "%s/api/v1/collections/%s"; + + Map config = getVectorDbInfo(hostOrKey, collection, configuration, url); + + methodAndPayloadNull(config); + + RestAPIConfig restAPIConfig = new RestAPIConfig( config, Map.of(), Map.of() ); + + return executeRequest(restAPIConfig, urlAccessChecker) + .map(v -> (Map) v) + .map(MapResult::new); + } + @Procedure("apoc.vectordb.chroma.createCollection") @Description("apoc.vectordb.chroma.createCollection(hostOrKey, collection, similarity, size, $configuration) - Creates a collection, with the name specified in the 2nd parameter, and with the specified `similarity` and `size`") public Stream createCollection(@Name("hostOrKey") String hostOrKey, diff --git a/extended/src/main/java/apoc/vectordb/Milvus.java b/extended/src/main/java/apoc/vectordb/Milvus.java index 05ce11468e..8922836a34 100644 --- a/extended/src/main/java/apoc/vectordb/Milvus.java +++ b/extended/src/main/java/apoc/vectordb/Milvus.java @@ -18,6 +18,7 @@ import java.util.Map; import java.util.stream.Stream; +import static apoc.ml.RestAPIConfig.BODY_KEY; import static apoc.ml.RestAPIConfig.METHOD_KEY; import static apoc.vectordb.VectorDb.executeRequest; import static apoc.vectordb.VectorDb.getEmbeddingResultStream; @@ -40,6 +41,21 @@ public class Milvus { @Context public URLAccessChecker urlAccessChecker; + @Procedure("apoc.vectordb.milvus.info") + @Description("apoc.vectordb.milvus.info(hostOrKey, collection, $configuration) - Get information about the specified existing collection or returns a response with code 100 if it does not exist") + public Stream info(@Name("hostOrKey") String hostOrKey, @Name("collection") String collection, @Name(value = "dbName", defaultValue = "default") String dbName, @Name(value = "configuration", defaultValue = "{}") Map configuration) throws Exception { + String url = "%s/collections/describe"; + Map config = getVectorDbInfo(hostOrKey, collection, configuration, url); + + config.put(BODY_KEY, Map.of("dbName", dbName, "collectionName", collection)); + + RestAPIConfig restAPIConfig = new RestAPIConfig( config, Map.of(), Map.of() ); + + return executeRequest(restAPIConfig, urlAccessChecker) + .map(v -> (Map) v) + .map(MapResult::new); + } + @Procedure("apoc.vectordb.milvus.createCollection") @Description("apoc.vectordb.milvus.createCollection(hostOrKey, collection, similarity, size, $configuration) - Creates a collection, with the name specified in the 2nd parameter, and with the specified `similarity` and `size`") public Stream createCollection(@Name("hostOrKey") String hostOrKey, diff --git a/extended/src/main/java/apoc/vectordb/Pinecone.java b/extended/src/main/java/apoc/vectordb/Pinecone.java index 896213b11b..4801452d07 100644 --- a/extended/src/main/java/apoc/vectordb/Pinecone.java +++ b/extended/src/main/java/apoc/vectordb/Pinecone.java @@ -23,6 +23,7 @@ import static apoc.vectordb.VectorDb.getEmbeddingResultStream; import static apoc.vectordb.VectorDbHandler.Type.PINECONE; import static apoc.vectordb.VectorDbUtil.getCommonVectorDbInfo; +import static apoc.vectordb.VectorDbUtil.methodAndPayloadNull; import static apoc.vectordb.VectorDbUtil.setReadOnlyMappingMode; @Extended @@ -41,6 +42,22 @@ public class Pinecone { @Context public URLAccessChecker urlAccessChecker; + @Procedure("apoc.vectordb.pinecone.info") + @Description("apoc.vectordb.pinecone.info(hostOrKey, collection, $configuration) - Get information about the specified existing collection or throws an error if it does not exist") + public Stream getInfo(@Name("hostOrKey") String hostOrKey, + @Name("collection") String collection, + @Name(value = "configuration", defaultValue = "{}") Map configuration) throws Exception { + String url = "%s/collections/%s"; + Map config = getVectorDbInfo(hostOrKey, collection, configuration, url); + + methodAndPayloadNull(config); + + RestAPIConfig restAPIConfig = new RestAPIConfig(config, Map.of(), Map.of()); + return executeRequest(restAPIConfig, urlAccessChecker) + .map(v -> (Map) v) + .map(MapResult::new); + } + @Procedure("apoc.vectordb.pinecone.createCollection") @Description("apoc.vectordb.pinecone.createCollection(hostOrKey, collection, similarity, size, $configuration) - Creates a collection, with the name specified in the 2nd parameter, and with the specified `similarity` and `size`") public Stream createCollection(@Name("hostOrKey") String hostOrKey, diff --git a/extended/src/main/java/apoc/vectordb/Qdrant.java b/extended/src/main/java/apoc/vectordb/Qdrant.java index f381e2b158..05df6e4465 100644 --- a/extended/src/main/java/apoc/vectordb/Qdrant.java +++ b/extended/src/main/java/apoc/vectordb/Qdrant.java @@ -39,6 +39,21 @@ public class Qdrant { @Context public URLAccessChecker urlAccessChecker; + + @Procedure("apoc.vectordb.qdrant.info") + @Description("apoc.vectordb.qdrant.info(hostOrKey, collection, $configuration) - Get information about the specified existing collection or throws an error if it does not exist") + public Stream info(@Name("hostOrKey") String hostOrKey, @Name("collection") String collection, @Name(value = "configuration", defaultValue = "{}") Map configuration) throws Exception { + String url = "%s/collections/%s"; + Map config = getVectorDbInfo(hostOrKey, collection, configuration, url); + + methodAndPayloadNull(config); + + RestAPIConfig restAPIConfig = new RestAPIConfig( config, Map.of(), Map.of() ); + + return executeRequest(restAPIConfig, urlAccessChecker) + .map(v -> (Map) v) + .map(MapResult::new); + } @Procedure("apoc.vectordb.qdrant.createCollection") @Description("apoc.vectordb.qdrant.createCollection(hostOrKey, collection, similarity, size, $configuration) - Creates a collection, with the name specified in the 2nd parameter, and with the specified `similarity` and `size`") diff --git a/extended/src/main/java/apoc/vectordb/VectorDbUtil.java b/extended/src/main/java/apoc/vectordb/VectorDbUtil.java index 9a16cd1d12..ea0c4f1d3b 100644 --- a/extended/src/main/java/apoc/vectordb/VectorDbUtil.java +++ b/extended/src/main/java/apoc/vectordb/VectorDbUtil.java @@ -9,12 +9,16 @@ import org.neo4j.graphdb.Node; import org.neo4j.graphdb.Relationship; +import java.net.HttpURLConnection; +import java.net.URL; import java.util.HashMap; import java.util.List; import java.util.Map; import static apoc.ml.RestAPIConfig.BASE_URL_KEY; +import static apoc.ml.RestAPIConfig.BODY_KEY; import static apoc.ml.RestAPIConfig.ENDPOINT_KEY; +import static apoc.ml.RestAPIConfig.METHOD_KEY; import static apoc.util.SystemDbUtil.withSystemDb; import static apoc.vectordb.VectorEmbeddingConfig.MAPPING_KEY; import static apoc.vectordb.VectorMappingConfig.MODE_KEY; @@ -40,20 +44,45 @@ public record EmbeddingResult( Object id, Double score, List vector, Map metadata, String text, Node node, Relationship rel) {} - + + /** + * Get vector configuration from config. parameter and system db database + */ public static Map getCommonVectorDbInfo( String hostOrKey, String collection, Map configuration, String templateUrl, VectorDbHandler handler) { Map config = new HashMap<>(configuration); + Map systemDbProps = getSystemDbProps(hostOrKey, handler); + + String baseUrl = getBaseUrl(hostOrKey, handler, config, systemDbProps); + + getMapping(config, systemDbProps); + + config = getCredentialsFromSystemDb(handler, config, systemDbProps); + + // endpoint creation + String endpoint = templateUrl.formatted(baseUrl, collection); + getEndpoint(config, endpoint); + + return config; + } + + /** + * Retrieve, if exists, the properties stored via `apoc.vectordb.configure` procedure + */ + private static Map getSystemDbProps(String hostOrKey, VectorDbHandler handler) { Map props = withSystemDb(transaction -> { Label label = Label.label(handler.getLabel()); Node node = transaction.findNode(label, SystemPropertyKeys.name.name(), hostOrKey); return node == null ? Map.of() : node.getAllProperties(); }); + return props; + } - String url = getUrl(hostOrKey, handler, props); - config.put(BASE_URL_KEY, url); - + /** + * Retrieve, if exists, the mapping stored via `apoc.vectordb.configure` procedure or via configuration parameter with key `mapping` + */ + private static void getMapping(Map config, Map props) { Map mappingConfVal = (Map) config.get(MAPPING_KEY); if ( MapUtils.isEmpty(mappingConfVal) ) { String mappingStoreVal = (String) props.get(MAPPING_KEY); @@ -61,20 +90,24 @@ public static Map getCommonVectorDbInfo( config.put( MAPPING_KEY, Util.fromJson(mappingStoreVal, Map.class) ); } } + } + private static Map getCredentialsFromSystemDb(VectorDbHandler handler, Map config, Map props) { String credentials = (String) props.get(ExtendedSystemPropertyKeys.credentials.name()); if (credentials != null) { Object credentialsObj = Util.fromJson(credentials, Object.class); config = handler.getCredentials(credentialsObj, config); } - - String endpoint = templateUrl.formatted(url, collection); - getEndpoint(config, endpoint); - return config; } + private static String getBaseUrl(String hostOrKey, VectorDbHandler handler, Map config, Map props) { + String url = getUrl(hostOrKey, handler, props); + config.put(BASE_URL_KEY, url); + return url; + } + private static String getUrl(String hostOrKey, VectorDbHandler handler, Map props) { if (props.isEmpty()) { return handler.getUrl(hostOrKey); @@ -86,4 +119,15 @@ public static void setReadOnlyMappingMode(Map configuration) { Map mappingConf = (Map) configuration.getOrDefault(MAPPING_KEY, new HashMap<>()); mappingConf.put(MODE_KEY, READ_ONLY.toString()); } + + /** + * The "method" should be "GET", but is null as a workaround. + * Since with `method: POST` the {@link apoc.util.Util#openUrlConnection(URL, Map)} has a `setChunkedStreamingMode` + * that makes the request to respond `405: Method Not Allowed` even if {@link HttpURLConnection#getRequestMethod()} is "GET". + * In any case, by putting `body: null`, the request is still in GET by default + */ + public static void methodAndPayloadNull(Map config) { + config.put(METHOD_KEY, null); + config.put(BODY_KEY, null); + } } diff --git a/extended/src/main/java/apoc/vectordb/Weaviate.java b/extended/src/main/java/apoc/vectordb/Weaviate.java index 7653c32e46..cc56f61966 100644 --- a/extended/src/main/java/apoc/vectordb/Weaviate.java +++ b/extended/src/main/java/apoc/vectordb/Weaviate.java @@ -44,6 +44,24 @@ public class Weaviate { @Context public URLAccessChecker urlAccessChecker; + @Procedure("apoc.vectordb.weaviate.info") + @Description("apoc.vectordb.weaviate.info(hostOrKey, collection, $configuration) - Get information about the specified existing collection or throws an error if it does not exist") + public Stream createCollection(@Name("hostOrKey") String hostOrKey, + @Name("collection") String collection, + @Name(value = "configuration", defaultValue = "{}") Map configuration) throws Exception { + var config = getVectorDbInfo(hostOrKey, collection, configuration, "%s/schema/%s"); + + methodAndPayloadNull(config); + + Map additionalBodies = Map.of("class", collection); + + RestAPIConfig restAPIConfig = new RestAPIConfig(config, Map.of(), additionalBodies); + + return executeRequest(restAPIConfig, urlAccessChecker) + .map(v -> (Map) v) + .map(MapResult::new); + } + @Procedure("apoc.vectordb.weaviate.createCollection") @Description("apoc.vectordb.weaviate.createCollection(hostOrKey, collection, similarity, size, $configuration) - Creates a collection, with the name specified in the 2nd parameter, and with the specified `similarity` and `size`") public Stream createCollection(@Name("hostOrKey") String hostOrKey, diff --git a/extended/src/main/resources/extended.txt b/extended/src/main/resources/extended.txt index 91d53cdf62..6be04562fb 100644 --- a/extended/src/main/resources/extended.txt +++ b/extended/src/main/resources/extended.txt @@ -229,6 +229,7 @@ apoc.vectordb.chroma.get apoc.vectordb.chroma.getAndUpdate apoc.vectordb.chroma.query apoc.vectordb.chroma.queryAndUpdate +apoc.vectordb.chroma.info apoc.vectordb.qdrant.createCollection apoc.vectordb.qdrant.deleteCollection apoc.vectordb.qdrant.upsert @@ -237,6 +238,7 @@ apoc.vectordb.qdrant.get apoc.vectordb.qdrant.getAndUpdate apoc.vectordb.qdrant.query apoc.vectordb.qdrant.queryAndUpdate +apoc.vectordb.qdrant.info apoc.vectordb.weaviate.createCollection apoc.vectordb.weaviate.deleteCollection apoc.vectordb.weaviate.upsert @@ -245,6 +247,7 @@ apoc.vectordb.weaviate.get apoc.vectordb.weaviate.getAndUpdate apoc.vectordb.weaviate.query apoc.vectordb.weaviate.queryAndUpdate +apoc.vectordb.weaviate.info apoc.vectordb.pinecone.createCollection apoc.vectordb.pinecone.deleteCollection apoc.vectordb.pinecone.upsert @@ -253,6 +256,7 @@ apoc.vectordb.pinecone.get apoc.vectordb.pinecone.getAndUpdate apoc.vectordb.pinecone.query apoc.vectordb.pinecone.queryAndUpdate +apoc.vectordb.pinecone.info apoc.vectordb.milvus.createCollection apoc.vectordb.milvus.deleteCollection apoc.vectordb.milvus.upsert @@ -261,6 +265,7 @@ apoc.vectordb.milvus.get apoc.vectordb.milvus.getAndUpdate apoc.vectordb.milvus.query apoc.vectordb.milvus.queryAndUpdate +apoc.vectordb.milvus.info apoc.vectordb.custom.get apoc.vectordb.custom apoc.vectordb.configure \ No newline at end of file diff --git a/extended/src/test/java/apoc/vectordb/PineconeTest.java b/extended/src/test/java/apoc/vectordb/PineconeTest.java index ff9d9e2abb..ea0d49fa00 100644 --- a/extended/src/test/java/apoc/vectordb/PineconeTest.java +++ b/extended/src/test/java/apoc/vectordb/PineconeTest.java @@ -19,6 +19,7 @@ import static apoc.ml.Prompt.API_KEY_CONF; import static apoc.ml.RestAPIConfig.HEADERS_KEY; +import static apoc.util.ExtendedTestUtil.assertFails; import static apoc.util.MapUtil.map; import static apoc.util.TestUtil.testCall; import static apoc.util.TestUtil.testCallEmpty; @@ -126,6 +127,29 @@ public void before() { dropAndDeleteAll(db); } + @Test + public void getInfo() { + testResult(db, "CALL apoc.vectordb.pinecone.info($host, $coll, $conf) ", + map("host", HOST, "coll", collName, + "conf", map(ALL_RESULTS_KEY, true, HEADERS_KEY, ADMIN_AUTHORIZATION) + ), + r -> { + Map row = r.next(); + Map value = (Map) row.get("value"); + assertEquals(collName, value.get("name")); + }); + } + + @Test + public void getInfoNotExistentCollection() { + assertFails(db, "CALL apoc.vectordb.pinecone.info($host, 'wrong_collection', $conf) ", + map("host", HOST, "coll", collName, + "conf", map(ALL_RESULTS_KEY, true, HEADERS_KEY, ADMIN_AUTHORIZATION) + ), + "Server returned HTTP response code: 500" + ); + } + @Test public void getVectors() { testResult(db, "CALL apoc.vectordb.pinecone.get($host, $coll, ['1', '2'], $conf) " +