From 4aa3904bf69c0f968ba1e99c4661fa3aafe0d4be Mon Sep 17 00:00:00 2001 From: Aditi Ahuja Date: Thu, 18 Apr 2024 12:59:18 +0530 Subject: [PATCH] update formula --- section_faiss_vector_index.go | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/section_faiss_vector_index.go b/section_faiss_vector_index.go index 5df7e432..d510a0cd 100644 --- a/section_faiss_vector_index.go +++ b/section_faiss_vector_index.go @@ -348,16 +348,17 @@ func (v *vectorIndexOpaque) mergeAndWriteVectorIndexes(sbs []*SegmentBase, nvecs := len(finalVecIDs) - // index type to be created after merge based on the number of vectors in - // indexData added into the index. + indexOptimizedFor := indexes[0].indexOptimizedFor + + // index type to be created after merge based on the number of vectors + // in indexData added into the index and chosen optimization. nlist := determineCentroids(nvecs) - indexDescription, indexClass := determineIndexToUse(nvecs, nlist) + indexDescription, indexClass := determineIndexToUse(nvecs, nlist, indexOptimizedFor) // safe to assume that all the indexes are of the same config values, given // that they are extracted from the field mapping info. dims := vecIndexes[0].D() metric := vecIndexes[0].MetricType() - indexOptimizedFor := indexes[0].indexOptimizedFor // freeing the reconstructed indexes immediately - waiting till the end // to do the same is not needed because the following operations don't need @@ -437,10 +438,7 @@ func determineCentroids(nvecs int) int { var nlist int switch { - // At 1M vectors, nlist = 4k gave a reasonably high recall with the right nprobe, - // whereas 1M/100 = 10000 centroids would increase training time without - // corresponding increase in recall - case nvecs >= 1000000: + case nvecs >= 200000: nlist = int(4 * math.Sqrt(float64(nvecs))) case nvecs >= 1000: // 100 points per cluster is a reasonable default, considering the default @@ -459,15 +457,17 @@ const ( // Returns a description string for the index and quantizer type // and an index type. -func determineIndexToUse(nvecs, nlist int) (string, int) { - switch { - case nvecs >= 10000: - return fmt.Sprintf("IVF%d,SQ8", nlist), IndexTypeIVF - case nvecs >= 1000: - return fmt.Sprintf("IVF%d,Flat", nlist), IndexTypeIVF - default: +func determineIndexToUse(nvecs, nlist int, indexOptimisedFor string) (string, int) { + if nvecs < 1000 { return "IDMap2,Flat", IndexTypeFlat } + + // If optimised for latency, use Flat. + if indexOptimisedFor == index.IndexOptimizedForLatency { + return fmt.Sprintf("IVF%d,Flat", nlist), IndexTypeIVF + } + + return fmt.Sprintf("IVF%d,SQ8", nlist), IndexTypeIVF } func (vo *vectorIndexOpaque) writeVectorIndexes(w *CountHashWriter) (offset uint64, err error) { @@ -492,7 +492,8 @@ func (vo *vectorIndexOpaque) writeVectorIndexes(w *CountHashWriter) (offset uint nvecs := len(ids) nlist := determineCentroids(nvecs) - indexDescription, indexClass := determineIndexToUse(nvecs, nlist) + indexDescription, indexClass := determineIndexToUse(nvecs, nlist, + content.indexOptimizedFor) faissIndex, err := faiss.IndexFactory(int(content.dim), indexDescription, metric) if err != nil { return 0, err