knowhere light for milvus local (zilliztech#502)

Signed-off-by: yusheng.ma <yusheng.ma@zilliz.com>

CMakeLists.txt

@@ -23,11 +23,11 @@ include(cmake/utils/compile_flags.cmake)
 include(cmake/utils/platform_check.cmake)
 
 knowhere_option(WITH_RAFT "Build with RAFT indexes" OFF)
-if (WITH_RAFT)
-    set(CMAKE_CUDA_ARCHITECTURES RAPIDS)
-    include(cmake/libs/librapids.cmake)
-    project(knowhere CXX C CUDA)
-    include(cmake/libs/libraft.cmake)
+if(WITH_RAFT)
+  set(CMAKE_CUDA_ARCHITECTURES RAPIDS)
+  include(cmake/libs/librapids.cmake)
+  project(knowhere CXX C CUDA)
+  include(cmake/libs/libraft.cmake)
 endif()
 
 knowhere_option(WITH_UT "Build with UT test" OFF)
@@ -38,22 +38,22 @@ knowhere_option(WITH_COVERAGE "Build with coverage" OFF)
 knowhere_option(WITH_CCACHE "Build with ccache" ON)
 knowhere_option(WITH_PROFILER "Build with profiler" OFF)
 knowhere_option(WITH_FAISS_TESTS "Build with Faiss unit tests" OFF)
+knowhere_option(WITH_LIGHT "Build with light weight version" OFF)
 
-# cardinal is an enterprise vector search engine and can only be enabled for cloud environment
+# cardinal is an enterprise vector search engine and can only be enabled for
+# cloud environment
 knowhere_option(WITH_CARDINAL "Build with cardinal" OFF)
 
-# this is needed for clang on ubuntu:20.04, otherwise
-#   the linked fails with 'undefined reference' error.
-# fmt v9 was used by the time the error was encountered.
-# clang on ubuntu:22.04 seems to be unaffected.
-# gcc seems to be unaffected.
+# this is needed for clang on ubuntu:20.04, otherwise the linked fails with
+# 'undefined reference' error. fmt v9 was used by the time the error was
+# encountered. clang on ubuntu:22.04 seems to be unaffected. gcc seems to be
+# unaffected.
 add_definitions(-DFMT_HEADER_ONLY)
 
-# this is needed for clang on ubuntu:20.04, otherwise
-#   the linked fails with 'undefined reference' error.
-# fmt v9 was used by the time the error was encountered.
-# clang on ubuntu:22.04 seems to be unaffected.
-# gcc seems to be unaffected.
+# this is needed for clang on ubuntu:20.04, otherwise the linked fails with
+# 'undefined reference' error. fmt v9 was used by the time the error was
+# encountered. clang on ubuntu:22.04 seems to be unaffected. gcc seems to be
+# unaffected.
 add_definitions(-DFMT_HEADER_ONLY)
 
 if(KNOWHERE_VERSION)
@@ -73,10 +73,14 @@ if(WITH_CCACHE)
   endif()
 endif()
 
-list( APPEND CMAKE_MODULE_PATH ${CMAKE_BINARY_DIR}/)
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_BINARY_DIR}/)
 
 add_definitions(-DNOT_COMPILE_FOR_SWIG)
 
+if(WITH_LIGHT)
+  add_definitions(-DKNOWHERE_WITH_LIGHT)
+endif()
+
 include(cmake/utils/platform_check.cmake)
 include(cmake/utils/compile_flags.cmake)
 include(cmake/libs/libfaiss.cmake)
@@ -96,7 +100,9 @@ find_package(fmt REQUIRED)
 find_package(opentelemetry-cpp REQUIRED)
 
 set(CMAKE_CXX_STANDARD 17)
-set(CMAKE_OSX_DEPLOYMENT_TARGET "10.13" CACHE STRING "Minimum OS X deployment version" FORCE)
+set(CMAKE_OSX_DEPLOYMENT_TARGET
+    "10.13"
+    CACHE STRING "Minimum OS X deployment version" FORCE)
 
 if(OPENMP_FOUND)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
@@ -108,11 +114,27 @@ if(WITH_COVERAGE)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage")
 endif()
 
-knowhere_file_glob(GLOB_RECURSE KNOWHERE_SRCS src/common/*.cc src/index/*.cc
-                   src/io/*.cc src/common/*.cu src/index/*.cu src/io/*.cu)
+if(NOT WITH_LIGHT)
+  knowhere_file_glob(
+    GLOB_RECURSE
+    KNOWHERE_SRCS
+    src/common/*.cc
+    src/index/*.cc
+    src/io/*.cc
+    src/common/*.cu
+    src/index/*.cu
+    src/io/*.cu)
+endif()
 
-set(KNOWHERE_LINKER_LIBS "")
+if(WITH_LIGHT)
+  knowhere_file_glob(GLOB_RECURSE KNOWHERE_SRCS src/common/*.cc
+                     src/index/hnsw/hnsw.cc src/io/*.cc)
+  knowhere_file_glob(GLOB_RECURSE KNOWHERE_TRACER_SRCS src/common/tracer.cc
+                     src/common/prometheus_client.cc)
+  list(REMOVE_ITEM KNOWHERE_SRCS ${KNOWHERE_TRACER_SRCS})
+endif()
 
+set(KNOWHERE_LINKER_LIBS "")
 
 if(WITH_CARDINAL)
   add_definitions(-DKNOWHERE_WITH_CARDINAL)
@@ -128,13 +150,12 @@ else()
 endif()
 
 knowhere_file_glob(GLOB_RECURSE KNOWHERE_GPU_SRCS src/index/gpu/flat_gpu/*.cc
-                     src/index/gpu/ivf_gpu/*.cc)
+                   src/index/gpu/ivf_gpu/*.cc)
 list(REMOVE_ITEM KNOWHERE_SRCS ${KNOWHERE_GPU_SRCS})
 
 if(NOT WITH_RAFT)
-  knowhere_file_glob(GLOB_RECURSE KNOWHERE_RAFT_SRCS
-                     src/common/raft/*.cu src/common/raft/*.cc
-                     src/index/gpu_raft/*.cc)
+  knowhere_file_glob(GLOB_RECURSE KNOWHERE_RAFT_SRCS src/common/raft/*.cu
+                     src/common/raft/*.cc src/index/gpu_raft/*.cc)
   list(REMOVE_ITEM KNOWHERE_SRCS ${KNOWHERE_RAFT_SRCS})
 endif()
 
@@ -144,23 +165,37 @@ include_directories(include)
 list(APPEND KNOWHERE_LINKER_LIBS faiss)
 list(APPEND KNOWHERE_LINKER_LIBS glog::glog)
 list(APPEND KNOWHERE_LINKER_LIBS nlohmann_json::nlohmann_json)
-list(APPEND KNOWHERE_LINKER_LIBS prometheus-cpp::core prometheus-cpp::push)
+if(NOT WITH_LIGHT)
+  list(APPEND KNOWHERE_LINKER_LIBS prometheus-cpp::core prometheus-cpp::push)
+endif()
 list(APPEND KNOWHERE_LINKER_LIBS fmt::fmt-header-only)
 list(APPEND KNOWHERE_LINKER_LIBS Folly::folly)
-list(APPEND KNOWHERE_LINKER_LIBS opentelemetry-cpp::opentelemetry_trace)
-list(APPEND KNOWHERE_LINKER_LIBS opentelemetry-cpp::opentelemetry_exporter_ostream_span)
-list(APPEND KNOWHERE_LINKER_LIBS opentelemetry-cpp::opentelemetry_exporter_jaeger_trace)
-list(APPEND KNOWHERE_LINKER_LIBS opentelemetry-cpp::opentelemetry_exporter_otlp_grpc)
-list(APPEND KNOWHERE_LINKER_LIBS opentelemetry-cpp::opentelemetry_exporter_otlp_http)
+if(NOT WITH_LIGHT)
+  list(APPEND KNOWHERE_LINKER_LIBS opentelemetry-cpp::opentelemetry_trace)
+  list(APPEND KNOWHERE_LINKER_LIBS
+       opentelemetry-cpp::opentelemetry_exporter_ostream_span)
+  list(APPEND KNOWHERE_LINKER_LIBS
+       opentelemetry-cpp::opentelemetry_exporter_jaeger_trace)
+  list(APPEND KNOWHERE_LINKER_LIBS
+       opentelemetry-cpp::opentelemetry_exporter_otlp_grpc)
+  list(APPEND KNOWHERE_LINKER_LIBS
+       opentelemetry-cpp::opentelemetry_exporter_otlp_http)
+endif()
 
 add_library(knowhere SHARED ${KNOWHERE_SRCS})
 add_dependencies(knowhere ${KNOWHERE_LINKER_LIBS})
 if(WITH_RAFT)
-  list(APPEND KNOWHERE_LINKER_LIBS raft::raft raft::compiled_static CUDA::cublas CUDA::cusparse CUDA::cusolver)
+  list(
+    APPEND
+    KNOWHERE_LINKER_LIBS
+    raft::raft
+    raft::compiled_static
+    CUDA::cublas
+    CUDA::cusparse
+    CUDA::cusolver)
 endif()
 target_link_libraries(knowhere PUBLIC ${KNOWHERE_LINKER_LIBS})
-target_include_directories(
-  knowhere PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
+target_include_directories(knowhere PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
 
 if(WITH_UT)
   add_subdirectory(tests/ut)

cmake/libs/libcardinal.cmake

@@ -5,26 +5,27 @@ set(CARDINAL_REPO_DIR "${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/cardinal")
 
 message(STATUS "Build Cardinal-${CARDINAL_VERSION}")
 
-if (NOT EXISTS "${CARDINAL_REPO_DIR}/.git")
-    execute_process(COMMAND git clone ${CARDINAL_REPO_URL} ${CARDINAL_REPO_DIR}
-            RESULT_VARIABLE CARDINAL_CLONE_RESULT
-            OUTPUT_VARIABLE CARDINAL_CLONE_OUTPUT
-            ERROR_VARIABLE CARDINAL_CLONE_ERROR
-            OUTPUT_STRIP_TRAILING_WHITESPACE
-            ERROR_STRIP_TRAILING_WHITESPACE)
-    if (NOT CARDINAL_CLONE_RESULT EQUAL "0")
-        message(FATAL_ERROR "Failed to clone cardinal: ${CARDINAL_CLONE_ERROR}")
-    else()
-        execute_process(COMMAND git -C ${CARDINAL_REPO_DIR} checkout ${CARDINAL_VERSION}
-            RESULT_VARIABLE CARDINAL_CHECKOUT_RESULT
-            OUTPUT_VARIABLE CARDINAL_CHECKOUT_OUTPUT
-            ERROR_VARIABLE CARDINAL_CHECKOUT_ERROR
-            OUTPUT_STRIP_TRAILING_WHITESPACE
-            ERROR_STRIP_TRAILING_WHITESPACE)
-        if (NOT CARDINAL_CHECKOUT_RESULT EQUAL "0")
-            message(FATAL_ERROR "Failed to checkout cardinal: ${CARDINAL_CHECKOUT_ERROR}")
-        endif()
+if(NOT EXISTS "${CARDINAL_REPO_DIR}/.git")
+  execute_process(
+    COMMAND git clone ${CARDINAL_REPO_URL} ${CARDINAL_REPO_DIR}
+    RESULT_VARIABLE CARDINAL_CLONE_RESULT
+    OUTPUT_VARIABLE CARDINAL_CLONE_OUTPUT
+    ERROR_VARIABLE CARDINAL_CLONE_ERROR
+    OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_STRIP_TRAILING_WHITESPACE)
+  if(NOT CARDINAL_CLONE_RESULT EQUAL "0")
+    message(FATAL_ERROR "Failed to clone cardinal: ${CARDINAL_CLONE_ERROR}")
+  else()
+    execute_process(
+      COMMAND git -C ${CARDINAL_REPO_DIR} checkout ${CARDINAL_VERSION}
+      RESULT_VARIABLE CARDINAL_CHECKOUT_RESULT
+      OUTPUT_VARIABLE CARDINAL_CHECKOUT_OUTPUT
+      ERROR_VARIABLE CARDINAL_CHECKOUT_ERROR
+      OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_STRIP_TRAILING_WHITESPACE)
+    if(NOT CARDINAL_CHECKOUT_RESULT EQUAL "0")
+      message(
+        FATAL_ERROR "Failed to checkout cardinal: ${CARDINAL_CHECKOUT_ERROR}")
     endif()
+  endif()
 endif()
 
 include(${CARDINAL_REPO_DIR}/know/libcardinal.cmake)

cmake/libs/libdiskann.cmake

@@ -24,14 +24,16 @@ set(DISKANN_SOURCES
 find_package(folly REQUIRED)
 
 add_library(diskann STATIC ${DISKANN_SOURCES})
-target_link_libraries(diskann PUBLIC ${AIO_LIBRARIES}
-                                     ${DISKANN_BOOST_PROGRAM_OPTIONS_LIB}
-                                     nlohmann_json::nlohmann_json
-                                     Folly::folly
-                                     fmt::fmt-header-only
-                                     prometheus-cpp::core
-                                     prometheus-cpp::push
-                                     glog::glog)
+target_link_libraries(
+  diskann
+  PUBLIC ${AIO_LIBRARIES}
+         ${DISKANN_BOOST_PROGRAM_OPTIONS_LIB}
+         nlohmann_json::nlohmann_json
+         Folly::folly
+         fmt::fmt-header-only
+         prometheus-cpp::core
+         prometheus-cpp::push
+         glog::glog)
 if(__X86_64)
   target_compile_options(
     diskann PRIVATE -fno-builtin-malloc -fno-builtin-calloc

cmake/libs/libfaiss.cmake

@@ -6,13 +6,15 @@ knowhere_file_glob(
 knowhere_file_glob(GLOB FAISS_AVX512_SRCS
                    thirdparty/faiss/faiss/impl/*avx512.cpp)
 
-knowhere_file_glob(GLOB FAISS_AVX2_SRCS
-                   thirdparty/faiss/faiss/impl/*avx.cpp
-                   thirdparty/faiss/faiss/impl/pq4_fast_scan_search_1.cpp
-                   thirdparty/faiss/faiss/impl/pq4_fast_scan_search_qbs.cpp
-                   thirdparty/faiss/faiss/utils/partitioning_avx2.cpp
-                   thirdparty/faiss/faiss/IndexPQFastScan.cpp
-                   thirdparty/faiss/faiss/IndexIVFPQFastScan.cpp)
+knowhere_file_glob(
+  GLOB
+  FAISS_AVX2_SRCS
+  thirdparty/faiss/faiss/impl/*avx.cpp
+  thirdparty/faiss/faiss/impl/pq4_fast_scan_search_1.cpp
+  thirdparty/faiss/faiss/impl/pq4_fast_scan_search_qbs.cpp
+  thirdparty/faiss/faiss/utils/partitioning_avx2.cpp
+  thirdparty/faiss/faiss/IndexPQFastScan.cpp
+  thirdparty/faiss/faiss/IndexIVFPQFastScan.cpp)
 
 list(REMOVE_ITEM FAISS_SRCS ${FAISS_AVX512_SRCS})
 
@@ -43,7 +45,8 @@ if(__X86_64)
 endif()
 
 if(__AARCH64)
-  set(UTILS_SRC src/simd/hook.cc src/simd/distances_ref.cc src/simd/distances_neon.cc)
+  set(UTILS_SRC src/simd/hook.cc src/simd/distances_ref.cc
+                src/simd/distances_neon.cc)
   add_library(knowhere_utils STATIC ${UTILS_SRC})
   target_link_libraries(knowhere_utils PUBLIC glog::glog)
 endif()
@@ -74,14 +77,8 @@ if(__X86_64)
   list(REMOVE_ITEM FAISS_SRCS ${FAISS_AVX2_SRCS})
 
   add_library(faiss_avx2 OBJECT ${FAISS_AVX2_SRCS})
-  target_compile_options(
-    faiss_avx2
-    PRIVATE $<$<COMPILE_LANGUAGE:CXX>:
-            -msse4.2
-            -mavx2
-            -mfma
-            -mf16c
-            -mpopcnt>)
+  target_compile_options(faiss_avx2 PRIVATE $<$<COMPILE_LANGUAGE:CXX>: -msse4.2
+                                            -mavx2 -mfma -mf16c -mpopcnt>)
   add_library(faiss_avx512 OBJECT ${FAISS_AVX512_SRCS})
   target_compile_options(
     faiss_avx512

cmake/libs/libraft.cmake

@@ -19,11 +19,8 @@ set(RAFT_VERSION "${RAPIDS_VERSION}")
 set(RAFT_FORK "wphicks")
 set(RAFT_PINNED_TAG "knowhere-2.4")
 
-
-rapids_find_package(CUDAToolkit REQUIRED
-  BUILD_EXPORT_SET knowhere-exports
-  INSTALL_EXPORT_SET knowhere-exports
-)
+rapids_find_package(CUDAToolkit REQUIRED BUILD_EXPORT_SET knowhere-exports
+                    INSTALL_EXPORT_SET knowhere-exports)
 
 function(find_and_configure_raft)
   set(oneValueArgs VERSION FORK PINNED_TAG)
@@ -53,14 +50,21 @@ function(find_and_configure_raft)
     "BUILD_BENCH OFF"
     "RAFT_USE_FAISS_STATIC OFF") # Turn this on to build FAISS into your binary
 
-    if(raft_ADDED)
-        message(VERBOSE "KNOWHERE: Using RAFT located in ${raft_SOURCE_DIR}")
-    else()
-        message(VERBOSE "KNOWHERE: Using RAFT located in ${raft_DIR}")
-    endif()
+  if(raft_ADDED)
+    message(VERBOSE "KNOWHERE: Using RAFT located in ${raft_SOURCE_DIR}")
+  else()
+    message(VERBOSE "KNOWHERE: Using RAFT located in ${raft_DIR}")
+  endif()
 endfunction()
 
 # Change pinned tag here to test a commit in CI To use a different RAFT locally,
 # set the CMake variable CPM_raft_SOURCE=/path/to/local/raft
-find_and_configure_raft(VERSION ${RAFT_VERSION}.00 FORK ${RAFT_FORK} PINNED_TAG
-                        ${RAFT_PINNED_TAG} COMPILE_LIBRARY OFF)
+find_and_configure_raft(
+  VERSION
+  ${RAFT_VERSION}.00
+  FORK
+  ${RAFT_FORK}
+  PINNED_TAG
+  ${RAFT_PINNED_TAG}
+  COMPILE_LIBRARY
+  OFF)

cmake/libs/librapids.cmake

@@ -23,7 +23,7 @@ if(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/RAPIDS.cmake)
 endif()
 include(${CMAKE_CURRENT_BINARY_DIR}/RAPIDS.cmake)
 
-include(rapids-cpm)  # Dependency tracking
+include(rapids-cpm) # Dependency tracking
 include(rapids-find) # Wrappers for finding packages
 include(rapids-cuda) # Common CMake CUDA logic
 

cmake/utils/compile_flags.cmake

@@ -18,11 +18,11 @@ endif()
 set(CMAKE_CXX_FLAGS "-Wall -fPIC ${CMAKE_CXX_FLAGS}")
 
 if(__X86_64)
-    set(CMAKE_CXX_FLAGS "-msse4.2 ${CMAKE_CXX_FLAGS}")
+  set(CMAKE_CXX_FLAGS "-msse4.2 ${CMAKE_CXX_FLAGS}")
 endif()
 
-set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g")
-set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG")
+set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g  -Wl,--no-as-needed")
+set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG  -Wl,--no-as-needed")
 
 if(WITH_RAFT)
   set(CMAKE_CUDA_FLAGS_DEBUG "-O0 -g -Xcompiler=-w ")

cmake/utils/platform_check.cmake

@@ -5,7 +5,9 @@ macro(detect_target_arch)
   check_symbol_exists(__x86_64__ "" __X86_64)
   check_symbol_exists(__powerpc64__ "" __PPC64)
 
-  if(NOT __AARCH64 AND NOT __X86_64 AND NOT __PPC64)
+  if(NOT __AARCH64
+     AND NOT __X86_64
+     AND NOT __PPC64)
     message(FATAL "knowhere only support amd64, ppc64 and arm64 architecture.")
   endif()
 endmacro()