Skip to content

Commit

Permalink
Bump Faiss commit from 33c0ba5 to 4eecd91 (opensearch-project#2194)
Browse files Browse the repository at this point in the history 
* Bump Faiss commit from 33c0ba5 to 4eecd91

Signed-off-by: Naveen Tatikonda <navtat@amazon.com>

* Update Faiss patches after commit bump

Signed-off-by: Naveen Tatikonda <navtat@amazon.com>

---------

Signed-off-by: Naveen Tatikonda <navtat@amazon.com>
  • Loading branch information
@naveentatikonda
naveentatikonda authored Oct 11, 2024
1 parent 5a56829 commit d9c7ba5
Show file tree
Hide file tree
Showing 3 changed files with 34 additions and 36 deletions.
2 changes: 1 addition & 1 deletion jni/external/faiss
Submodule faiss updated 186 files
46 changes: 22 additions & 24 deletions jni/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
From 9e5affabe2caacf38f5585a0b906620dd35deef5 Mon Sep 17 00:00:00 2001
From e775a8e65da96232822d5aed77f538592fccffda Mon Sep 17 00:00:00 2001
From: Heemin Kim <heemin@amazon.com>
Date: Tue, 30 Jan 2024 14:43:56 -0800
Subject: [PATCH] Add IDGrouper for HNSW

Signed-off-by: Heemin Kim <heemin@amazon.com>
---
faiss/CMakeLists.txt | 3 +
faiss/Index.h | 8 +-
faiss/Index.h | 6 +-
faiss/IndexHNSW.cpp | 13 +-
faiss/IndexIDMap.cpp | 29 +++++
faiss/IndexIDMap.h | 22 ++++
Expand All @@ -18,34 +18,34 @@ Signed-off-by: Heemin Kim <heemin@amazon.com>
tests/CMakeLists.txt | 2 +
tests/test_group_heap.cpp | 98 +++++++++++++++
tests/test_id_grouper.cpp | 241 +++++++++++++++++++++++++++++++++++++
13 files changed, 890 insertions(+), 5 deletions(-)
13 files changed, 889 insertions(+), 4 deletions(-)
create mode 100644 faiss/impl/IDGrouper.cpp
create mode 100644 faiss/impl/IDGrouper.h
create mode 100644 faiss/utils/GroupHeap.h
create mode 100644 tests/test_group_heap.cpp
create mode 100644 tests/test_id_grouper.cpp

diff --git a/faiss/CMakeLists.txt b/faiss/CMakeLists.txt
index 1b0860f3..f3d72df3 100644
index 2871d974..d0bcec6a 100644
--- a/faiss/CMakeLists.txt
+++ b/faiss/CMakeLists.txt
@@ -54,6 +54,7 @@ set(FAISS_SRC
@@ -55,6 +55,7 @@ set(FAISS_SRC
impl/AuxIndexStructures.cpp
impl/CodePacker.cpp
impl/IDSelector.cpp
+ impl/IDGrouper.cpp
impl/FaissException.cpp
impl/HNSW.cpp
impl/NSG.cpp
@@ -149,6 +150,7 @@ set(FAISS_HEADERS
@@ -151,6 +152,7 @@ set(FAISS_HEADERS
impl/AuxIndexStructures.h
impl/CodePacker.h
impl/IDSelector.h
+ impl/IDGrouper.h
impl/DistanceComputer.h
impl/FaissAssert.h
impl/FaissException.h
@@ -184,6 +186,7 @@ set(FAISS_HEADERS
@@ -186,6 +188,7 @@ set(FAISS_HEADERS
invlists/InvertedListsIOHook.h
utils/AlignedTable.h
utils/bf16.h
Expand All @@ -54,23 +54,21 @@ index 1b0860f3..f3d72df3 100644
utils/WorkerThread.h
utils/distances.h
diff --git a/faiss/Index.h b/faiss/Index.h
index 3d1bdb99..a8622858 100644
index f57140ec..8f511e5d 100644
--- a/faiss/Index.h
+++ b/faiss/Index.h
@@ -38,9 +38,10 @@

@@ -51,8 +51,9 @@
namespace faiss {

-/// Forward declarations see impl/AuxIndexStructures.h, impl/IDSelector.h and
-/// impl/DistanceComputer.h
+/// Forward declarations see impl/AuxIndexStructures.h, impl/IDSelector.h
/// Forward declarations see impl/AuxIndexStructures.h, impl/IDSelector.h
-/// and impl/DistanceComputer.h
+/// ,impl/IDGrouper.h and impl/DistanceComputer.h
struct IDSelector;
+struct IDGrouper;
struct RangeSearchResult;
struct DistanceComputer;

@@ -52,6 +53,9 @@ struct DistanceComputer;
@@ -64,6 +65,9 @@ struct DistanceComputer;
struct SearchParameters {
/// if non-null, only these IDs will be considered during search.
IDSelector* sel = nullptr;
Expand All @@ -81,10 +79,10 @@ index 3d1bdb99..a8622858 100644
virtual ~SearchParameters() {}
};
diff --git a/faiss/IndexHNSW.cpp b/faiss/IndexHNSW.cpp
index 8e5c654f..d473b6ad 100644
index 6a1186ca..9c8a8255 100644
--- a/faiss/IndexHNSW.cpp
+++ b/faiss/IndexHNSW.cpp
@@ -320,10 +320,17 @@ void IndexHNSW::search(
@@ -301,10 +301,17 @@ void IndexHNSW::search(
const SearchParameters* params_in) const {
FAISS_THROW_IF_NOT(k > 0);

Expand Down Expand Up @@ -198,10 +196,10 @@ index 2d164123..a68887bd 100644
+
} // namespace faiss
diff --git a/faiss/impl/HNSW.cpp b/faiss/impl/HNSW.cpp
index 3ba5f72f..c574ce39 100644
index c3693fd9..7ae28062 100644
--- a/faiss/impl/HNSW.cpp
+++ b/faiss/impl/HNSW.cpp
@@ -831,6 +831,12 @@ int extract_k_from_ResultHandler(ResultHandler<C>& res) {
@@ -906,6 +906,12 @@ int extract_k_from_ResultHandler(ResultHandler<C>& res) {
if (auto hres = dynamic_cast<RH::SingleResultHandler*>(&res)) {
return hres->k;
}
Expand Down Expand Up @@ -329,19 +327,19 @@ index 00000000..d56113d9
+
+} // namespace faiss
diff --git a/faiss/impl/ResultHandler.h b/faiss/impl/ResultHandler.h
index 713fe8e4..d307fd70 100644
index 3116eb24..126ed015 100644
--- a/faiss/impl/ResultHandler.h
+++ b/faiss/impl/ResultHandler.h
@@ -13,6 +13,8 @@

@@ -14,6 +14,8 @@
#include <faiss/impl/AuxIndexStructures.h>
#include <faiss/impl/FaissException.h>
#include <faiss/impl/IDSelector.h>
+#include <faiss/impl/IDGrouper.h>
+#include <faiss/utils/GroupHeap.h>
#include <faiss/utils/Heap.h>
#include <faiss/utils/partitioning.h>
#include <iostream>
@@ -267,6 +269,193 @@ struct HeapBlockResultHandler : BlockResultHandler<C> {

@@ -286,6 +288,193 @@ struct HeapBlockResultHandler : BlockResultHandler<C, use_sel> {
}
};

Expand Down Expand Up @@ -725,7 +723,7 @@ index 00000000..3b7078da
+} // namespace faiss
\ No newline at end of file
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 3980d7dd..c888a5a6 100644
index c41edf0c..87ab2020 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -27,6 +27,8 @@ set(FAISS_TEST_SRC
Expand Down
22 changes: 11 additions & 11 deletions jni/patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
From a33e6ef35385009f24200586294f96235cb95d61 Mon Sep 17 00:00:00 2001
From 9b33874562c9e62abf4a863657c54f0d349b0f67 Mon Sep 17 00:00:00 2001
From: John Mazanec <jmazane@amazon.com>
Date: Wed, 21 Feb 2024 15:34:15 -0800
Subject: [PATCH] Enable precomp table to be shared ivfpq
Expand All @@ -22,7 +22,7 @@ Signed-off-by: John Mazanec <jmazane@amazon.com>
create mode 100644 tests/test_ivfpq_share_table.cpp

diff --git a/faiss/IndexIVFPQ.cpp b/faiss/IndexIVFPQ.cpp
index 0b7f4d05..07bc7e83 100644
index 100f499c..09508890 100644
--- a/faiss/IndexIVFPQ.cpp
+++ b/faiss/IndexIVFPQ.cpp
@@ -59,6 +59,29 @@ IndexIVFPQ::IndexIVFPQ(
Expand Down Expand Up @@ -55,7 +55,7 @@ index 0b7f4d05..07bc7e83 100644
}

/****************************************************************
@@ -466,11 +489,23 @@ void IndexIVFPQ::precompute_table() {
@@ -464,11 +487,23 @@ void IndexIVFPQ::precompute_table() {
use_precomputed_table,
quantizer,
pq,
Expand All @@ -80,7 +80,7 @@ index 0b7f4d05..07bc7e83 100644
namespace {

#define TIC t0 = get_cycles()
@@ -650,7 +685,7 @@ struct QueryTables {
@@ -648,7 +683,7 @@ struct QueryTables {

fvec_madd(
pq.M * pq.ksub,
Expand All @@ -89,7 +89,7 @@ index 0b7f4d05..07bc7e83 100644
-2.0,
sim_table_2,
sim_table);
@@ -679,7 +714,7 @@ struct QueryTables {
@@ -677,7 +712,7 @@ struct QueryTables {
k >>= cpq.nbits;

// get corresponding table
Expand All @@ -98,7 +98,7 @@ index 0b7f4d05..07bc7e83 100644
(ki * pq.M + cm * Mf) * pq.ksub;

if (polysemous_ht == 0) {
@@ -709,7 +744,7 @@ struct QueryTables {
@@ -707,7 +742,7 @@ struct QueryTables {
dis0 = coarse_dis;

const float* s =
Expand All @@ -107,7 +107,7 @@ index 0b7f4d05..07bc7e83 100644
for (int m = 0; m < pq.M; m++) {
sim_table_ptrs[m] = s;
s += pq.ksub;
@@ -729,7 +764,7 @@ struct QueryTables {
@@ -727,7 +762,7 @@ struct QueryTables {
int ki = k & ((uint64_t(1) << cpq.nbits) - 1);
k >>= cpq.nbits;

Expand All @@ -116,7 +116,7 @@ index 0b7f4d05..07bc7e83 100644
(ki * pq.M + cm * Mf) * pq.ksub;

for (int m = m0; m < m0 + Mf; m++) {
@@ -1346,6 +1381,8 @@ IndexIVFPQ::IndexIVFPQ() {
@@ -1344,6 +1379,8 @@ IndexIVFPQ::IndexIVFPQ() {
do_polysemous_training = false;
polysemous_ht = 0;
polysemous_training = nullptr;
Expand Down Expand Up @@ -302,13 +302,13 @@ index 00dd2f11..91f35a6e 100644
/// same as the regular IVFPQ encoder. The codes are not reorganized by
/// blocks a that point
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index c888a5a6..83ecedfd 100644
index 87ab2020..a859516c 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -37,6 +37,7 @@ set(FAISS_TEST_SRC
test_disable_pq_sdc_tables.cpp
@@ -38,6 +38,7 @@ set(FAISS_TEST_SRC
test_common_ivf_empty_index.cpp
test_callback.cpp
test_utils.cpp
+ test_ivfpq_share_table.cpp
)

Expand Down

0 comments on commit d9c7ba5

Please sign in to comment.