refactor(storage): improve inverted index match phrase query (#16547)

* refactor(storage): improve inverted index match phrase query

* fix typos

* fix machete

* fix machete

Cargo.toml

@@ -408,7 +408,7 @@ openai_api_rust = { git = "https://github.com/datafuse-extras/openai-api", rev =
 orc-rust = { git = "https://github.com/datafuse-extras/datafusion-orc", rev = "03372b97" }
 recursive = { git = "https://github.com/datafuse-extras/recursive.git", rev = "6af35a1" }
 sled = { git = "https://github.com/datafuse-extras/sled", tag = "v0.34.7-datafuse.1" }
-tantivy = { git = "https://github.com/datafuse-extras/tantivy", rev = "37aeac0" }
-tantivy-common = { git = "https://github.com/datafuse-extras/tantivy", rev = "37aeac0", package = "tantivy-common" }
-tantivy-jieba = { git = "https://github.com/datafuse-extras/tantivy-jieba", rev = "124a8fc" }
+tantivy = { git = "https://github.com/datafuse-extras/tantivy", rev = "7502370" }
+tantivy-common = { git = "https://github.com/datafuse-extras/tantivy", rev = "7502370", package = "tantivy-common" }
+tantivy-jieba = { git = "https://github.com/datafuse-extras/tantivy-jieba", rev = "0e300e9" }
 xorfilter-rs = { git = "https://github.com/datafuse-extras/xorfilter", tag = "databend-alpha.4" }

src/common/arrow/src/arrow/array/list/mutable.rs

@@ -182,7 +182,9 @@ impl<O: Offset, M: MutableArray> MutableListArray<O, M> {
     pub fn try_push_valid(&mut self) -> Result<()> {
         let total_length = self.values.len();
         let offset = self.offsets.last().to_usize();
-        let length = total_length.checked_sub(offset).ok_or(Error::Overflow)?;
+        let length = total_length
+            .checked_sub(offset)
+            .ok_or_else(|| Error::Overflow)?;
 
         self.offsets.try_push_usize(length)?;
         if let Some(validity) = &mut self.validity {

src/common/arrow/src/arrow/offset.rs

@@ -127,7 +127,9 @@ impl<O: Offset> Offsets<O> {
     pub fn try_push(&mut self, length: O) -> Result<(), Error> {
         let old_length = self.last();
         assert!(length >= O::zero());
-        let new_length = old_length.checked_add(&length).ok_or(Error::Overflow)?;
+        let new_length = old_length
+            .checked_add(&length)
+            .ok_or_else(|| Error::Overflow)?;
         self.0.push(new_length);
         Ok(())
     }
@@ -140,10 +142,12 @@ impl<O: Offset> Offsets<O> {
     /// * checks that this length does not overflow
     #[inline]
     pub fn try_push_usize(&mut self, length: usize) -> Result<(), Error> {
-        let length = O::from_usize(length).ok_or(Error::Overflow)?;
+        let length = O::from_usize(length).ok_or_else(|| Error::Overflow)?;
 
         let old_length = self.last();
-        let new_length = old_length.checked_add(&length).ok_or(Error::Overflow)?;
+        let new_length = old_length
+            .checked_add(&length)
+            .ok_or_else(|| Error::Overflow)?;
         self.0.push(new_length);
         Ok(())
     }
@@ -267,8 +271,8 @@ impl<O: Offset> Offsets<O> {
 
         let last_offset = original_offset
             .checked_add(total_length)
-            .ok_or(Error::Overflow)?;
-        O::from_usize(last_offset).ok_or(Error::Overflow)?;
+            .ok_or_else(|| Error::Overflow)?;
+        O::from_usize(last_offset).ok_or_else(|| Error::Overflow)?;
         Ok(())
     }
 
@@ -279,7 +283,9 @@ impl<O: Offset> Offsets<O> {
         let mut length = *self.last();
         let other_length = *other.last();
         // check if the operation would overflow
-        length.checked_add(&other_length).ok_or(Error::Overflow)?;
+        length
+            .checked_add(&other_length)
+            .ok_or_else(|| Error::Overflow)?;
 
         let lengths = other.as_slice().windows(2).map(|w| w[1] - w[0]);
         let offsets = lengths.map(|new_length| {
@@ -306,7 +312,9 @@ impl<O: Offset> Offsets<O> {
         let other_length = other.last().expect("Length to be non-zero");
         let mut length = *self.last();
         // check if the operation would overflow
-        length.checked_add(other_length).ok_or(Error::Overflow)?;
+        length
+            .checked_add(other_length)
+            .ok_or_else(|| Error::Overflow)?;
 
         let lengths = other.windows(2).map(|w| w[1] - w[0]);
         let offsets = lengths.map(|new_length| {

src/query/ee/tests/it/inverted_index/index_refresh.rs

@@ -143,7 +143,6 @@ async fn test_fuse_do_refresh_inverted_index() -> Result<()> {
         &index_version,
     );
 
-    let field_nums = query_fields.len();
     let has_score = true;
     let need_position = false;
     let mut field_ids = HashSet::new();
@@ -177,15 +176,14 @@ async fn test_fuse_do_refresh_inverted_index() -> Result<()> {
         let matched_rows = index_reader
             .clone()
             .do_filter(
-                field_nums,
                 need_position,
                 has_score,
                 query.box_clone(),
                 &field_ids,
                 &index_record,
                 &fuzziness,
                 tokenizer_manager,
-                block_meta.row_count as u32,
+                block_meta.row_count,
                 &index_loc,
             )
             .await?;

src/query/service/src/pipelines/processors/transforms/transform_add_internal_columns.rs

@@ -40,8 +40,8 @@ impl Transform for TransformAddInternalColumns {
 
     fn transform(&mut self, mut block: DataBlock) -> Result<DataBlock> {
         if let Some(meta) = block.take_meta() {
-            let internal_column_meta =
-                InternalColumnMeta::downcast_from(meta).ok_or(ErrorCode::Internal("It's a bug"))?;
+            let internal_column_meta = InternalColumnMeta::downcast_from(meta)
+                .ok_or_else(|| ErrorCode::Internal("It's a bug"))?;
             let num_rows = block.num_rows();
             for internal_column in self.internal_columns.values() {
                 let column =

src/query/service/src/servers/flight_sql/flight_sql_service/session.rs

@@ -64,7 +64,7 @@ impl FlightSqlServiceImpl {
     pub(super) fn get_user_password(metadata: &MetadataMap) -> Result<(String, String), String> {
         let basic = "Basic ";
         let authorization = Self::get_header_value(metadata, "authorization")
-            .ok_or("authorization not parsable".to_string())?;
+            .ok_or_else(|| "authorization not parsable".to_string())?;
 
         if !authorization.starts_with(basic) {
             return Err(format!("Auth type not implemented: {authorization}"));

src/query/service/src/servers/http/v1/session/refresh_handler.rs

@@ -48,11 +48,11 @@ pub async fn refresh_handler(
     let mgr = ClientSessionManager::instance();
     match &ctx.credential {
         Credential::Jwt { .. } => {
-            let session_id =
-                req.session_id
-                    .ok_or(HttpErrorCode::bad_request(ErrorCode::BadArguments(
-                        "JWT session should provide session_id when refresh session",
-                    )))?;
+            let session_id = req.session_id.ok_or_else(|| {
+                HttpErrorCode::bad_request(ErrorCode::BadArguments(
+                    "JWT session should provide session_id when refresh session",
+                ))
+            })?;
             mgr.refresh_in_memory_states(&session_id);
 
             let tenant = ctx.session.get_current_tenant();

src/query/sql/src/planner/planner_cache.rs

@@ -165,7 +165,7 @@ impl TableRefVisitor {
 
         let func_name = func.name.name.to_lowercase();
         // If the function is not suitable for caching, we should not cache the plan
-        if !is_cacheable_function(&func_name) || func_name == "score" {
+        if !is_cacheable_function(&func_name) {
             self.cache_miss = true;
         }
     }

src/query/storages/common/index/Cargo.toml

@@ -16,7 +16,6 @@ ignored = ["xorfilter-rs", "match-template"]
 [dependencies]
 anyerror = { workspace = true }
 cbordata = { version = "0.6.0" }
-crc32fast = "1.3.2"
 databend-common-arrow = { workspace = true }
 databend-common-ast = { workspace = true }
 databend-common-exception = { workspace = true }
@@ -29,8 +28,8 @@ levenshtein_automata = "0.2.1"
 log = { workspace = true }
 match-template = { workspace = true }
 parquet = { workspace = true }
+roaring = "0.10.1"
 serde = { workspace = true }
-serde_json = { workspace = true }
 tantivy = { workspace = true }
 tantivy-common = { workspace = true }
 tantivy-fst = "0.5"