diff --git a/CMakeLists.txt b/CMakeLists.txt index 90bcda213..8d709a05b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,11 +23,11 @@ include(cmake/utils/compile_flags.cmake) include(cmake/utils/platform_check.cmake) knowhere_option(WITH_RAFT "Build with RAFT indexes" OFF) -if (WITH_RAFT) - set(CMAKE_CUDA_ARCHITECTURES RAPIDS) - include(cmake/libs/librapids.cmake) - project(knowhere CXX C CUDA) - include(cmake/libs/libraft.cmake) +if(WITH_RAFT) + set(CMAKE_CUDA_ARCHITECTURES RAPIDS) + include(cmake/libs/librapids.cmake) + project(knowhere CXX C CUDA) + include(cmake/libs/libraft.cmake) endif() knowhere_option(WITH_UT "Build with UT test" OFF) @@ -38,22 +38,22 @@ knowhere_option(WITH_COVERAGE "Build with coverage" OFF) knowhere_option(WITH_CCACHE "Build with ccache" ON) knowhere_option(WITH_PROFILER "Build with profiler" OFF) knowhere_option(WITH_FAISS_TESTS "Build with Faiss unit tests" OFF) +knowhere_option(WITH_LIGHT "Build with light weight version" OFF) -# cardinal is an enterprise vector search engine and can only be enabled for cloud environment +# cardinal is an enterprise vector search engine and can only be enabled for +# cloud environment knowhere_option(WITH_CARDINAL "Build with cardinal" OFF) -# this is needed for clang on ubuntu:20.04, otherwise -# the linked fails with 'undefined reference' error. -# fmt v9 was used by the time the error was encountered. -# clang on ubuntu:22.04 seems to be unaffected. -# gcc seems to be unaffected. +# this is needed for clang on ubuntu:20.04, otherwise the linked fails with +# 'undefined reference' error. fmt v9 was used by the time the error was +# encountered. clang on ubuntu:22.04 seems to be unaffected. gcc seems to be +# unaffected. add_definitions(-DFMT_HEADER_ONLY) -# this is needed for clang on ubuntu:20.04, otherwise -# the linked fails with 'undefined reference' error. -# fmt v9 was used by the time the error was encountered. -# clang on ubuntu:22.04 seems to be unaffected. -# gcc seems to be unaffected. +# this is needed for clang on ubuntu:20.04, otherwise the linked fails with +# 'undefined reference' error. fmt v9 was used by the time the error was +# encountered. clang on ubuntu:22.04 seems to be unaffected. gcc seems to be +# unaffected. add_definitions(-DFMT_HEADER_ONLY) if(KNOWHERE_VERSION) @@ -73,10 +73,14 @@ if(WITH_CCACHE) endif() endif() -list( APPEND CMAKE_MODULE_PATH ${CMAKE_BINARY_DIR}/) +list(APPEND CMAKE_MODULE_PATH ${CMAKE_BINARY_DIR}/) add_definitions(-DNOT_COMPILE_FOR_SWIG) +if(WITH_LIGHT) + add_definitions(-DKNOWHERE_WITH_LIGHT) +endif() + include(cmake/utils/platform_check.cmake) include(cmake/utils/compile_flags.cmake) include(cmake/libs/libfaiss.cmake) @@ -96,7 +100,9 @@ find_package(fmt REQUIRED) find_package(opentelemetry-cpp REQUIRED) set(CMAKE_CXX_STANDARD 17) -set(CMAKE_OSX_DEPLOYMENT_TARGET "10.13" CACHE STRING "Minimum OS X deployment version" FORCE) +set(CMAKE_OSX_DEPLOYMENT_TARGET + "10.13" + CACHE STRING "Minimum OS X deployment version" FORCE) if(OPENMP_FOUND) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") @@ -108,11 +114,27 @@ if(WITH_COVERAGE) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage") endif() -knowhere_file_glob(GLOB_RECURSE KNOWHERE_SRCS src/common/*.cc src/index/*.cc - src/io/*.cc src/common/*.cu src/index/*.cu src/io/*.cu) +if(NOT WITH_LIGHT) + knowhere_file_glob( + GLOB_RECURSE + KNOWHERE_SRCS + src/common/*.cc + src/index/*.cc + src/io/*.cc + src/common/*.cu + src/index/*.cu + src/io/*.cu) +endif() -set(KNOWHERE_LINKER_LIBS "") +if(WITH_LIGHT) + knowhere_file_glob(GLOB_RECURSE KNOWHERE_SRCS src/common/*.cc + src/index/hnsw/hnsw.cc src/io/*.cc) + knowhere_file_glob(GLOB_RECURSE KNOWHERE_TRACER_SRCS src/common/tracer.cc + src/common/prometheus_client.cc) + list(REMOVE_ITEM KNOWHERE_SRCS ${KNOWHERE_TRACER_SRCS}) +endif() +set(KNOWHERE_LINKER_LIBS "") if(WITH_CARDINAL) add_definitions(-DKNOWHERE_WITH_CARDINAL) @@ -128,13 +150,12 @@ else() endif() knowhere_file_glob(GLOB_RECURSE KNOWHERE_GPU_SRCS src/index/gpu/flat_gpu/*.cc - src/index/gpu/ivf_gpu/*.cc) + src/index/gpu/ivf_gpu/*.cc) list(REMOVE_ITEM KNOWHERE_SRCS ${KNOWHERE_GPU_SRCS}) if(NOT WITH_RAFT) - knowhere_file_glob(GLOB_RECURSE KNOWHERE_RAFT_SRCS - src/common/raft/*.cu src/common/raft/*.cc - src/index/gpu_raft/*.cc) + knowhere_file_glob(GLOB_RECURSE KNOWHERE_RAFT_SRCS src/common/raft/*.cu + src/common/raft/*.cc src/index/gpu_raft/*.cc) list(REMOVE_ITEM KNOWHERE_SRCS ${KNOWHERE_RAFT_SRCS}) endif() @@ -144,23 +165,37 @@ include_directories(include) list(APPEND KNOWHERE_LINKER_LIBS faiss) list(APPEND KNOWHERE_LINKER_LIBS glog::glog) list(APPEND KNOWHERE_LINKER_LIBS nlohmann_json::nlohmann_json) -list(APPEND KNOWHERE_LINKER_LIBS prometheus-cpp::core prometheus-cpp::push) +if(NOT WITH_LIGHT) + list(APPEND KNOWHERE_LINKER_LIBS prometheus-cpp::core prometheus-cpp::push) +endif() list(APPEND KNOWHERE_LINKER_LIBS fmt::fmt-header-only) list(APPEND KNOWHERE_LINKER_LIBS Folly::folly) -list(APPEND KNOWHERE_LINKER_LIBS opentelemetry-cpp::opentelemetry_trace) -list(APPEND KNOWHERE_LINKER_LIBS opentelemetry-cpp::opentelemetry_exporter_ostream_span) -list(APPEND KNOWHERE_LINKER_LIBS opentelemetry-cpp::opentelemetry_exporter_jaeger_trace) -list(APPEND KNOWHERE_LINKER_LIBS opentelemetry-cpp::opentelemetry_exporter_otlp_grpc) -list(APPEND KNOWHERE_LINKER_LIBS opentelemetry-cpp::opentelemetry_exporter_otlp_http) +if(NOT WITH_LIGHT) + list(APPEND KNOWHERE_LINKER_LIBS opentelemetry-cpp::opentelemetry_trace) + list(APPEND KNOWHERE_LINKER_LIBS + opentelemetry-cpp::opentelemetry_exporter_ostream_span) + list(APPEND KNOWHERE_LINKER_LIBS + opentelemetry-cpp::opentelemetry_exporter_jaeger_trace) + list(APPEND KNOWHERE_LINKER_LIBS + opentelemetry-cpp::opentelemetry_exporter_otlp_grpc) + list(APPEND KNOWHERE_LINKER_LIBS + opentelemetry-cpp::opentelemetry_exporter_otlp_http) +endif() add_library(knowhere SHARED ${KNOWHERE_SRCS}) add_dependencies(knowhere ${KNOWHERE_LINKER_LIBS}) if(WITH_RAFT) - list(APPEND KNOWHERE_LINKER_LIBS raft::raft raft::compiled_static CUDA::cublas CUDA::cusparse CUDA::cusolver) + list( + APPEND + KNOWHERE_LINKER_LIBS + raft::raft + raft::compiled_static + CUDA::cublas + CUDA::cusparse + CUDA::cusolver) endif() target_link_libraries(knowhere PUBLIC ${KNOWHERE_LINKER_LIBS}) -target_include_directories( - knowhere PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include) +target_include_directories(knowhere PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include) if(WITH_UT) add_subdirectory(tests/ut) diff --git a/cmake/libs/libcardinal.cmake b/cmake/libs/libcardinal.cmake index 171fb279d..63f900090 100644 --- a/cmake/libs/libcardinal.cmake +++ b/cmake/libs/libcardinal.cmake @@ -5,26 +5,27 @@ set(CARDINAL_REPO_DIR "${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/cardinal") message(STATUS "Build Cardinal-${CARDINAL_VERSION}") -if (NOT EXISTS "${CARDINAL_REPO_DIR}/.git") - execute_process(COMMAND git clone ${CARDINAL_REPO_URL} ${CARDINAL_REPO_DIR} - RESULT_VARIABLE CARDINAL_CLONE_RESULT - OUTPUT_VARIABLE CARDINAL_CLONE_OUTPUT - ERROR_VARIABLE CARDINAL_CLONE_ERROR - OUTPUT_STRIP_TRAILING_WHITESPACE - ERROR_STRIP_TRAILING_WHITESPACE) - if (NOT CARDINAL_CLONE_RESULT EQUAL "0") - message(FATAL_ERROR "Failed to clone cardinal: ${CARDINAL_CLONE_ERROR}") - else() - execute_process(COMMAND git -C ${CARDINAL_REPO_DIR} checkout ${CARDINAL_VERSION} - RESULT_VARIABLE CARDINAL_CHECKOUT_RESULT - OUTPUT_VARIABLE CARDINAL_CHECKOUT_OUTPUT - ERROR_VARIABLE CARDINAL_CHECKOUT_ERROR - OUTPUT_STRIP_TRAILING_WHITESPACE - ERROR_STRIP_TRAILING_WHITESPACE) - if (NOT CARDINAL_CHECKOUT_RESULT EQUAL "0") - message(FATAL_ERROR "Failed to checkout cardinal: ${CARDINAL_CHECKOUT_ERROR}") - endif() +if(NOT EXISTS "${CARDINAL_REPO_DIR}/.git") + execute_process( + COMMAND git clone ${CARDINAL_REPO_URL} ${CARDINAL_REPO_DIR} + RESULT_VARIABLE CARDINAL_CLONE_RESULT + OUTPUT_VARIABLE CARDINAL_CLONE_OUTPUT + ERROR_VARIABLE CARDINAL_CLONE_ERROR + OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_STRIP_TRAILING_WHITESPACE) + if(NOT CARDINAL_CLONE_RESULT EQUAL "0") + message(FATAL_ERROR "Failed to clone cardinal: ${CARDINAL_CLONE_ERROR}") + else() + execute_process( + COMMAND git -C ${CARDINAL_REPO_DIR} checkout ${CARDINAL_VERSION} + RESULT_VARIABLE CARDINAL_CHECKOUT_RESULT + OUTPUT_VARIABLE CARDINAL_CHECKOUT_OUTPUT + ERROR_VARIABLE CARDINAL_CHECKOUT_ERROR + OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_STRIP_TRAILING_WHITESPACE) + if(NOT CARDINAL_CHECKOUT_RESULT EQUAL "0") + message( + FATAL_ERROR "Failed to checkout cardinal: ${CARDINAL_CHECKOUT_ERROR}") endif() + endif() endif() include(${CARDINAL_REPO_DIR}/know/libcardinal.cmake) diff --git a/cmake/libs/libdiskann.cmake b/cmake/libs/libdiskann.cmake index d9fbf8066..c93843a9c 100644 --- a/cmake/libs/libdiskann.cmake +++ b/cmake/libs/libdiskann.cmake @@ -24,14 +24,16 @@ set(DISKANN_SOURCES find_package(folly REQUIRED) add_library(diskann STATIC ${DISKANN_SOURCES}) -target_link_libraries(diskann PUBLIC ${AIO_LIBRARIES} - ${DISKANN_BOOST_PROGRAM_OPTIONS_LIB} - nlohmann_json::nlohmann_json - Folly::folly - fmt::fmt-header-only - prometheus-cpp::core - prometheus-cpp::push - glog::glog) +target_link_libraries( + diskann + PUBLIC ${AIO_LIBRARIES} + ${DISKANN_BOOST_PROGRAM_OPTIONS_LIB} + nlohmann_json::nlohmann_json + Folly::folly + fmt::fmt-header-only + prometheus-cpp::core + prometheus-cpp::push + glog::glog) if(__X86_64) target_compile_options( diskann PRIVATE -fno-builtin-malloc -fno-builtin-calloc diff --git a/cmake/libs/libfaiss.cmake b/cmake/libs/libfaiss.cmake index 931077ee0..402558759 100644 --- a/cmake/libs/libfaiss.cmake +++ b/cmake/libs/libfaiss.cmake @@ -6,13 +6,15 @@ knowhere_file_glob( knowhere_file_glob(GLOB FAISS_AVX512_SRCS thirdparty/faiss/faiss/impl/*avx512.cpp) -knowhere_file_glob(GLOB FAISS_AVX2_SRCS - thirdparty/faiss/faiss/impl/*avx.cpp - thirdparty/faiss/faiss/impl/pq4_fast_scan_search_1.cpp - thirdparty/faiss/faiss/impl/pq4_fast_scan_search_qbs.cpp - thirdparty/faiss/faiss/utils/partitioning_avx2.cpp - thirdparty/faiss/faiss/IndexPQFastScan.cpp - thirdparty/faiss/faiss/IndexIVFPQFastScan.cpp) +knowhere_file_glob( + GLOB + FAISS_AVX2_SRCS + thirdparty/faiss/faiss/impl/*avx.cpp + thirdparty/faiss/faiss/impl/pq4_fast_scan_search_1.cpp + thirdparty/faiss/faiss/impl/pq4_fast_scan_search_qbs.cpp + thirdparty/faiss/faiss/utils/partitioning_avx2.cpp + thirdparty/faiss/faiss/IndexPQFastScan.cpp + thirdparty/faiss/faiss/IndexIVFPQFastScan.cpp) list(REMOVE_ITEM FAISS_SRCS ${FAISS_AVX512_SRCS}) @@ -43,7 +45,8 @@ if(__X86_64) endif() if(__AARCH64) - set(UTILS_SRC src/simd/hook.cc src/simd/distances_ref.cc src/simd/distances_neon.cc) + set(UTILS_SRC src/simd/hook.cc src/simd/distances_ref.cc + src/simd/distances_neon.cc) add_library(knowhere_utils STATIC ${UTILS_SRC}) target_link_libraries(knowhere_utils PUBLIC glog::glog) endif() @@ -74,14 +77,8 @@ if(__X86_64) list(REMOVE_ITEM FAISS_SRCS ${FAISS_AVX2_SRCS}) add_library(faiss_avx2 OBJECT ${FAISS_AVX2_SRCS}) - target_compile_options( - faiss_avx2 - PRIVATE $<$: - -msse4.2 - -mavx2 - -mfma - -mf16c - -mpopcnt>) + target_compile_options(faiss_avx2 PRIVATE $<$: -msse4.2 + -mavx2 -mfma -mf16c -mpopcnt>) add_library(faiss_avx512 OBJECT ${FAISS_AVX512_SRCS}) target_compile_options( faiss_avx512 diff --git a/cmake/libs/libraft.cmake b/cmake/libs/libraft.cmake index 0b804d997..157ef661b 100644 --- a/cmake/libs/libraft.cmake +++ b/cmake/libs/libraft.cmake @@ -19,11 +19,8 @@ set(RAFT_VERSION "${RAPIDS_VERSION}") set(RAFT_FORK "wphicks") set(RAFT_PINNED_TAG "knowhere-2.4") - -rapids_find_package(CUDAToolkit REQUIRED - BUILD_EXPORT_SET knowhere-exports - INSTALL_EXPORT_SET knowhere-exports -) +rapids_find_package(CUDAToolkit REQUIRED BUILD_EXPORT_SET knowhere-exports + INSTALL_EXPORT_SET knowhere-exports) function(find_and_configure_raft) set(oneValueArgs VERSION FORK PINNED_TAG) @@ -53,14 +50,21 @@ function(find_and_configure_raft) "BUILD_BENCH OFF" "RAFT_USE_FAISS_STATIC OFF") # Turn this on to build FAISS into your binary - if(raft_ADDED) - message(VERBOSE "KNOWHERE: Using RAFT located in ${raft_SOURCE_DIR}") - else() - message(VERBOSE "KNOWHERE: Using RAFT located in ${raft_DIR}") - endif() + if(raft_ADDED) + message(VERBOSE "KNOWHERE: Using RAFT located in ${raft_SOURCE_DIR}") + else() + message(VERBOSE "KNOWHERE: Using RAFT located in ${raft_DIR}") + endif() endfunction() # Change pinned tag here to test a commit in CI To use a different RAFT locally, # set the CMake variable CPM_raft_SOURCE=/path/to/local/raft -find_and_configure_raft(VERSION ${RAFT_VERSION}.00 FORK ${RAFT_FORK} PINNED_TAG - ${RAFT_PINNED_TAG} COMPILE_LIBRARY OFF) +find_and_configure_raft( + VERSION + ${RAFT_VERSION}.00 + FORK + ${RAFT_FORK} + PINNED_TAG + ${RAFT_PINNED_TAG} + COMPILE_LIBRARY + OFF) diff --git a/cmake/libs/librapids.cmake b/cmake/libs/librapids.cmake index 315c4c3d2..7b0c4e7c8 100644 --- a/cmake/libs/librapids.cmake +++ b/cmake/libs/librapids.cmake @@ -23,7 +23,7 @@ if(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/RAPIDS.cmake) endif() include(${CMAKE_CURRENT_BINARY_DIR}/RAPIDS.cmake) -include(rapids-cpm) # Dependency tracking +include(rapids-cpm) # Dependency tracking include(rapids-find) # Wrappers for finding packages include(rapids-cuda) # Common CMake CUDA logic diff --git a/cmake/utils/compile_flags.cmake b/cmake/utils/compile_flags.cmake index 1cf21303b..123dac15a 100644 --- a/cmake/utils/compile_flags.cmake +++ b/cmake/utils/compile_flags.cmake @@ -18,11 +18,11 @@ endif() set(CMAKE_CXX_FLAGS "-Wall -fPIC ${CMAKE_CXX_FLAGS}") if(__X86_64) - set(CMAKE_CXX_FLAGS "-msse4.2 ${CMAKE_CXX_FLAGS}") + set(CMAKE_CXX_FLAGS "-msse4.2 ${CMAKE_CXX_FLAGS}") endif() -set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g") -set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG") +set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g -Wl,--no-as-needed") +set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG -Wl,--no-as-needed") if(WITH_RAFT) set(CMAKE_CUDA_FLAGS_DEBUG "-O0 -g -Xcompiler=-w ") diff --git a/cmake/utils/platform_check.cmake b/cmake/utils/platform_check.cmake index 0345b92fc..afc41d07a 100644 --- a/cmake/utils/platform_check.cmake +++ b/cmake/utils/platform_check.cmake @@ -5,7 +5,9 @@ macro(detect_target_arch) check_symbol_exists(__x86_64__ "" __X86_64) check_symbol_exists(__powerpc64__ "" __PPC64) - if(NOT __AARCH64 AND NOT __X86_64 AND NOT __PPC64) + if(NOT __AARCH64 + AND NOT __X86_64 + AND NOT __PPC64) message(FATAL "knowhere only support amd64, ppc64 and arm64 architecture.") endif() endmacro() diff --git a/src/common/comp/brute_force.cc b/src/common/comp/brute_force.cc index 8cd117b6a..2f676f63a 100644 --- a/src/common/comp/brute_force.cc +++ b/src/common/comp/brute_force.cc @@ -27,7 +27,7 @@ #include "knowhere/sparse_utils.h" #include "knowhere/utils.h" -#ifdef NOT_COMPILE_FOR_SWIG +#if defined(NOT_COMPILE_FOR_SWIG) && !defined(KNOWHERE_WITH_LIGHT) #include "knowhere/tracer.h" #endif @@ -61,7 +61,7 @@ BruteForce::Search(const DataSetPtr base_dataset, const DataSetPtr query_dataset return expected::Err(status, msg); } -#ifdef NOT_COMPILE_FOR_SWIG +#if defined(NOT_COMPILE_FOR_SWIG) && !defined(KNOWHERE_WITH_LIGHT) std::shared_ptr span = nullptr; if (cfg.trace_id.has_value()) { auto ctx = tracer::GetTraceCtxFromCfg(&cfg); @@ -155,7 +155,7 @@ BruteForce::Search(const DataSetPtr base_dataset, const DataSetPtr query_dataset } auto res = GenResultDataSet(nq, cfg.k.value(), labels.release(), distances.release()); -#ifdef NOT_COMPILE_FOR_SWIG +#if defined(NOT_COMPILE_FOR_SWIG) && !defined(KNOWHERE_WITH_LIGHT) if (cfg.trace_id.has_value()) { span->End(); } @@ -184,7 +184,7 @@ BruteForce::SearchWithBuf(const DataSetPtr base_dataset, const DataSetPtr query_ BruteForceConfig cfg; RETURN_IF_ERROR(Config::Load(cfg, config, knowhere::SEARCH)); -#ifdef NOT_COMPILE_FOR_SWIG +#if defined(NOT_COMPILE_FOR_SWIG) && !defined(KNOWHERE_WITH_LIGHT) std::shared_ptr span = nullptr; if (cfg.trace_id.has_value()) { auto ctx = tracer::GetTraceCtxFromCfg(&cfg); @@ -274,7 +274,7 @@ BruteForce::SearchWithBuf(const DataSetPtr base_dataset, const DataSetPtr query_ } RETURN_IF_ERROR(WaitAllSuccess(futs)); -#ifdef NOT_COMPILE_FOR_SWIG +#if defined(NOT_COMPILE_FOR_SWIG) && !defined(KNOWHERE_WITH_LIGHT) if (cfg.trace_id.has_value()) { span->End(); } @@ -309,7 +309,7 @@ BruteForce::RangeSearch(const DataSetPtr base_dataset, const DataSetPtr query_da return expected::Err(status, std::move(msg)); } -#ifdef NOT_COMPILE_FOR_SWIG +#if defined(NOT_COMPILE_FOR_SWIG) && !defined(KNOWHERE_WITH_LIGHT) std::shared_ptr span = nullptr; if (cfg.trace_id.has_value()) { auto ctx = tracer::GetTraceCtxFromCfg(&cfg); @@ -417,7 +417,7 @@ BruteForce::RangeSearch(const DataSetPtr base_dataset, const DataSetPtr query_da GetRangeSearchResult(result_dist_array, result_id_array, is_ip, nq, radius, range_filter, distances, ids, lims); auto res = GenResultDataSet(nq, ids, distances, lims); -#ifdef NOT_COMPILE_FOR_SWIG +#if defined(NOT_COMPILE_FOR_SWIG) && !defined(KNOWHERE_WITH_LIGHT) if (cfg.trace_id.has_value()) { span->End(); } @@ -444,7 +444,7 @@ BruteForce::SearchSparseWithBuf(const DataSetPtr base_dataset, const DataSetPtr return status; } -#ifdef NOT_COMPILE_FOR_SWIG +#if defined(NOT_COMPILE_FOR_SWIG) && !defined(KNOWHERE_WITH_LIGHT) std::shared_ptr span = nullptr; if (cfg.trace_id.has_value()) { auto ctx = tracer::GetTraceCtxFromCfg(&cfg); @@ -504,7 +504,7 @@ BruteForce::SearchSparseWithBuf(const DataSetPtr base_dataset, const DataSetPtr } WaitAllSuccess(futs); -#ifdef NOT_COMPILE_FOR_SWIG +#if defined(NOT_COMPILE_FOR_SWIG) && !defined(KNOWHERE_WITH_LIGHT) if (cfg.trace_id.has_value()) { span->End(); } @@ -560,7 +560,7 @@ BruteForce::AnnIterator(const DataSetPtr base_dataset, const DataSetPtr query_da return expected>>::Err(result.error(), result.what()); } -#ifdef NOT_COMPILE_FOR_SWIG +#if defined(NOT_COMPILE_FOR_SWIG) && !defined(KNOWHERE_WITH_LIGHT) std::shared_ptr span = nullptr; if (cfg.trace_id.has_value()) { auto ctx = tracer::GetTraceCtxFromCfg(&cfg); @@ -623,7 +623,7 @@ BruteForce::AnnIterator(const DataSetPtr base_dataset, const DataSetPtr query_da ret, "failed to brute force search for iterator"); } -#ifdef NOT_COMPILE_FOR_SWIG +#if defined(NOT_COMPILE_FOR_SWIG) && !defined(KNOWHERE_WITH_LIGHT) if (cfg.trace_id.has_value()) { span->End(); } @@ -654,7 +654,7 @@ BruteForce::AnnIterator>(const DataSetPtr bas std::string metric_str = cfg.metric_type.value(); -#ifdef NOT_COMPILE_FOR_SWIG +#if defined(NOT_COMPILE_FOR_SWIG) && !defined(KNOWHERE_WITH_LIGHT) std::shared_ptr span = nullptr; if (cfg.trace_id.has_value()) { auto ctx = tracer::GetTraceCtxFromCfg(&cfg); @@ -703,7 +703,7 @@ BruteForce::AnnIterator>(const DataSetPtr bas } WaitAllSuccess(futs); -#ifdef NOT_COMPILE_FOR_SWIG +#if defined(NOT_COMPILE_FOR_SWIG) && !defined(KNOWHERE_WITH_LIGHT) if (cfg.trace_id.has_value()) { span->End(); } @@ -745,7 +745,6 @@ template knowhere::Status knowhere::BruteForce::SearchWithBuf(const knowhere::DataSetPtr base_dataset, const knowhere::DataSetPtr query_dataset, int64_t* ids, float* dis, const knowhere::Json& config, const knowhere::BitsetView& bitset); - template knowhere::expected knowhere::BruteForce::RangeSearch(const knowhere::DataSetPtr base_dataset, const knowhere::DataSetPtr query_dataset, @@ -762,7 +761,6 @@ template knowhere::expected knowhere::BruteForce::RangeSearch(const knowhere::DataSetPtr base_dataset, const knowhere::DataSetPtr query_dataset, const knowhere::Json& config, const knowhere::BitsetView& bitset); - template knowhere::expected>> knowhere::BruteForce::AnnIterator(const knowhere::DataSetPtr base_dataset, const knowhere::DataSetPtr query_dataset, diff --git a/src/common/index.cc b/src/common/index.cc index 9d115cdd3..e8e80fe03 100644 --- a/src/common/index.cc +++ b/src/common/index.cc @@ -16,7 +16,7 @@ #include "knowhere/expected.h" #include "knowhere/log.h" -#ifdef NOT_COMPILE_FOR_SWIG +#if defined(NOT_COMPILE_FOR_SWIG) && !defined(KNOWHERE_WITH_LIGHT) #include "knowhere/prometheus_client.h" #include "knowhere/tracer.h" #endif @@ -39,7 +39,7 @@ Index::Build(const DataSet& dataset, const Json& json) { auto cfg = this->node->CreateConfig(); RETURN_IF_ERROR(LoadConfig(cfg.get(), json, knowhere::TRAIN, "Build")); -#ifdef NOT_COMPILE_FOR_SWIG +#if defined(NOT_COMPILE_FOR_SWIG) && !defined(KNOWHERE_WITH_LIGHT) TimeRecorder rc("Build index", 2); auto res = this->node->Build(dataset, *cfg); auto time = rc.ElapseFromBegin("done"); @@ -78,7 +78,7 @@ Index::Search(const DataSet& dataset, const Json& json, const BitsetView& bit } const auto bitset = BitsetView(bitset_.data(), bitset_.size(), bitset_.get_filtered_out_num_()); -#ifdef NOT_COMPILE_FOR_SWIG +#if defined(NOT_COMPILE_FOR_SWIG) && !defined(KNOWHERE_WITH_LIGHT) const BaseConfig& b_cfg = static_cast(*cfg); std::shared_ptr span = nullptr; if (b_cfg.trace_id.has_value()) { @@ -118,7 +118,7 @@ Index::AnnIterator(const DataSet& dataset, const Json& json, const BitsetView } const auto bitset = BitsetView(bitset_.data(), bitset_.size(), bitset_.get_filtered_out_num_()); -#ifdef NOT_COMPILE_FOR_SWIG +#if defined(NOT_COMPILE_FOR_SWIG) && !defined(KNOWHERE_WITH_LIGHT) // note that this time includes only the initial search phase of iterator. TimeRecorder rc("AnnIterator"); auto res = this->node->AnnIterator(dataset, *cfg, bitset); @@ -142,7 +142,7 @@ Index::RangeSearch(const DataSet& dataset, const Json& json, const BitsetView } const auto bitset = BitsetView(bitset_.data(), bitset_.size(), bitset_.get_filtered_out_num_()); -#ifdef NOT_COMPILE_FOR_SWIG +#if defined(NOT_COMPILE_FOR_SWIG) && !defined(KNOWHERE_WITH_LIGHT) const BaseConfig& b_cfg = static_cast(*cfg); std::shared_ptr span = nullptr; if (b_cfg.trace_id.has_value()) { @@ -226,7 +226,7 @@ Index::Deserialize(const BinarySet& binset, const Json& json) { return res; } -#ifdef NOT_COMPILE_FOR_SWIG +#if defined(NOT_COMPILE_FOR_SWIG) && !defined(KNOWHERE_WITH_LIGHT) TimeRecorder rc("Load index", 2); res = this->node->Deserialize(binset, *cfg); auto time = rc.ElapseFromBegin("done"); @@ -255,7 +255,7 @@ Index::DeserializeFromFile(const std::string& filename, const Json& json) { return res; } -#ifdef NOT_COMPILE_FOR_SWIG +#if defined(NOT_COMPILE_FOR_SWIG) && !defined(KNOWHERE_WITH_LIGHT) TimeRecorder rc("Load index from file", 2); res = this->node->DeserializeFromFile(filename, *cfg); auto time = rc.ElapseFromBegin("done"); diff --git a/thirdparty/hnswlib/hnswlib/hnswalg.h b/thirdparty/hnswlib/hnswlib/hnswalg.h index 631fa4cea..656c4a865 100644 --- a/thirdparty/hnswlib/hnswlib/hnswalg.h +++ b/thirdparty/hnswlib/hnswlib/hnswalg.h @@ -28,7 +28,9 @@ #include "io/memory_io.h" #include "knowhere/config.h" #include "knowhere/heap.h" +#if defined(NOT_COMPILE_FOR_SWIG) && !defined(KNOWHERE_WITH_LIGHT) #include "knowhere/prometheus_client.h" +#endif #include "knowhere/utils.h" #include "neighbor.h" #include "visited_list_pool.h" @@ -491,7 +493,7 @@ class HierarchicalNSW : public AlgorithmInterface { data_point, retset.pop(), visited, accumulative_alpha, bitset, add_search_candidate, feder_result); hops++; } -#ifdef NOT_COMPILE_FOR_SWIG +#if defined(NOT_COMPILE_FOR_SWIG) && !defined(KNOWHERE_WITH_LIGHT) knowhere::knowhere_hnsw_search_hops.Observe(hops); #endif return retset; @@ -1375,7 +1377,7 @@ class HierarchicalNSW : public AlgorithmInterface { // do bruteforce search when delete rate high if (!bitset.empty()) { const size_t filtered_out_num = bitset.count(); -#ifdef NOT_COMPILE_FOR_SWIG +#if defined(NOT_COMPILE_FOR_SWIG) && !defined(KNOWHERE_WITH_LIGHT) double ratio = ((double)filtered_out_num) / bitset.size(); knowhere::knowhere_hnsw_bitset_ratio.Observe(ratio); #endif @@ -1536,7 +1538,7 @@ class HierarchicalNSW : public AlgorithmInterface { // do bruteforce range search when delete rate high if (!bitset.empty()) { const size_t filtered_out_num = bitset.count(); -#ifdef NOT_COMPILE_FOR_SWIG +#if defined(NOT_COMPILE_FOR_SWIG) && !defined(KNOWHERE_WITH_LIGHT) double ratio = ((double)filtered_out_num) / bitset.size(); knowhere::knowhere_hnsw_bitset_ratio.Observe(ratio); #endif