diff --git a/section_faiss_vector_index.go b/section_faiss_vector_index.go index 35852e5e..5da6073d 100644 --- a/section_faiss_vector_index.go +++ b/section_faiss_vector_index.go @@ -348,17 +348,17 @@ func (v *vectorIndexOpaque) mergeAndWriteVectorIndexes(sbs []*SegmentBase, nvecs := len(finalVecIDs) - // index type to be created after merge based on the number of vectors - // in indexData added into the index. - nlist := determineCentroids(nvecs) - indexDescription, indexClass := determineIndexToUse(nvecs, nlist) - // safe to assume that all the indexes are of the same config values, given // that they are extracted from the field mapping info. dims := vecIndexes[0].D() metric := vecIndexes[0].MetricType() indexOptimizedFor := indexes[0].indexOptimizedFor + // index type to be created after merge based on the number of vectors + // in indexData added into the index. + nlist := determineCentroids(nvecs) + indexDescription, indexClass := determineIndexToUse(nvecs, nlist, indexOptimizedFor) + // freeing the reconstructed indexes immediately - waiting till the end // to do the same is not needed because the following operations don't need // the reconstructed ones anymore and doing so will hold up memory which can @@ -456,9 +456,12 @@ const ( // Returns a description string for the index and quantizer type // and an index type. -func determineIndexToUse(nvecs, nlist int) (string, int) { +func determineIndexToUse(nvecs, nlist int, indexOptimizedFor string) (string, int) { switch { case nvecs >= 10000: + if indexOptimizedFor == index.IndexOptimizedForMemory { + return fmt.Sprintf("IVF%d,SQ4", nlist), IndexTypeIVF + } return fmt.Sprintf("IVF%d,SQ8", nlist), IndexTypeIVF case nvecs >= 1000: return fmt.Sprintf("IVF%d,Flat", nlist), IndexTypeIVF @@ -489,7 +492,8 @@ func (vo *vectorIndexOpaque) writeVectorIndexes(w *CountHashWriter) (offset uint nvecs := len(ids) nlist := determineCentroids(nvecs) - indexDescription, indexClass := determineIndexToUse(nvecs, nlist) + indexDescription, indexClass := determineIndexToUse(nvecs, nlist, + content.indexOptimizedFor) faissIndex, err := faiss.IndexFactory(int(content.dim), indexDescription, metric) if err != nil { return 0, err