Add iterator support to pyknowhere interface. (zilliztech#189)

* Add iterator support to pyknowhere interface. Signed-off-by: Buqian Zheng <zhengbuqian@gmail.com> * Rename AnnIterator to AnnIteratorWrap Signed-off-by: Buqian Zheng <zhengbuqian@gmail.com> * throw exception when nullptr iterator is provided Signed-off-by: Buqian Zheng <zhengbuqian@gmail.com> --------- Signed-off-by: Buqian Zheng <zhengbuqian@gmail.com>
alexanderguzhva · Nov 22, 2023 · 2191316 · 2191316
1 parent 9489598
commit 2191316
Show file tree

Hide file tree

Showing 3 changed files with 45 additions and 2 deletions.
diff --git a/python/knowhere/knowhere.i b/python/knowhere/knowhere.i
@@ -51,11 +51,13 @@ import_array();
 %include <std_pair.i>
 %include <std_map.i>
 %include <std_shared_ptr.i>
+%include <std_vector.i>
 %include <exception.i>
 %shared_ptr(knowhere::DataSet)
 %shared_ptr(knowhere::BinarySet)
 %template(DataSetPtr) std::shared_ptr<knowhere::DataSet>;
 %template(BinarySetPtr) std::shared_ptr<knowhere::BinarySet>;
+%template(int64_float_pair) std::pair<long long int, float>;
 %include <knowhere/expected.h>
 %include <knowhere/dataset.h>
 %include <knowhere/binaryset.h>
@@ -104,7 +106,7 @@ del Enum
 %inline %{
 
 class GILReleaser {
-public:
+ public:
     GILReleaser() : save(PyEval_SaveThread()) {
     }
     ~GILReleaser() {
@@ -113,6 +115,28 @@ public:
     PyThreadState* save;
 };
 
+class AnnIteratorWrap {
+ public:
+    AnnIteratorWrap(std::shared_ptr<IndexNode::iterator> it = nullptr) : it_(it) {
+        if (it_ == nullptr) {
+            throw std::runtime_error("ann iterator must not be nullptr.");
+        }
+    }
+    ~AnnIteratorWrap() {
+    }
+
+    bool HasNext() {
+        return it_->HasNext();
+    }
+
+    std::pair<int64_t, float> Next() {
+        return it_->Next();
+    }
+
+ private:
+    std::shared_ptr<IndexNode::iterator> it_;
+};
+
 class IndexWrap {
  public:
     IndexWrap(const std::string& name, const int32_t& version) {
@@ -157,6 +181,22 @@ class IndexWrap {
         }
     }
 
+    std::vector<AnnIteratorWrap>
+    GetAnnIterator(knowhere::DataSetPtr dataset, const std::string& json, const knowhere::BitsetView& bitset, knowhere::Status& status) {
+        GILReleaser rel;
+        auto res = idx.AnnIterator(*dataset, knowhere::Json::parse(json), bitset);
+        std::vector<AnnIteratorWrap> result;
+        if (!res.has_value()) {
+            status = res.error();
+            return result;
+        }
+        status = knowhere::Status::success;
+        for (auto it : res.value()) {
+            result.emplace_back(it);
+        }
+        return result;
+    }
+
     knowhere::DataSetPtr
     RangeSearch(knowhere::DataSetPtr dataset, const std::string& json, const knowhere::BitsetView& bitset, knowhere::Status& status){
         GILReleaser rel;
@@ -468,3 +508,5 @@ SetSimdType(const std::string type) {
 }
 
 %}
+
+%template(AnnIteratorWrapVector) std::vector<AnnIteratorWrap>;
diff --git a/python/setup.py b/python/setup.py
@@ -90,7 +90,7 @@ def get_readme():
     description=(
         "A library for efficient similarity search and clustering of vectors."
     ),
-    url="https://github.com/milvus-io/knowhere",
+    url="https://github.com/zilliztech/knowhere",
     author="Milvus Team",
     author_email="milvus-team@zilliz.com",
     license='Apache License 2.0',

diff --git a/src/common/config.cc b/src/common/config.cc
@@ -25,6 +25,7 @@ static const std::unordered_set<std::string> ext_legal_json_keys = {"metric_type
                                                                     "M",               // HNSW param
                                                                     "efConstruction",  // HNSW param
                                                                     "ef",              // HNSW param
+                                                                    "seed_ef",         // HNSW iterator param
                                                                     "level",
                                                                     "index_type",
                                                                     "index_mode",