Updated Tantivy version for slop queries

CHANGELOG.md

@@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
  - Support for boolean field
+ - Support for slop in phrase queries
 
 ### Fixed
 

docs/reference/query-language.md

@@ -35,6 +35,36 @@ Quickwit supports parenthesis to group multiple clauses:
 (color:red OR color:green) AND size:large
 ```
 
+### Slop Operator
+
+Quickwit also supports phrase queries with a slop parameter using the slop operator `~` followed by the value of the slop. For instance, the query `body:"small bike"~2` will match documents containing the word `small`, followed by one or two words immediately followed by the word `bike`.
+
+:::caution
+Slop queries can only be used on field indexed with the [record option](./../configuration/index-config.md#text-type) set to `position` value.
+:::
+
+#### Examples:
+
+With the following corpus:
+```json 
+[
+    {"id": 1, "body": "a red bike"},
+    {"id": 2, "body": "a small blue bike"},
+    {"id": 3, "body": "a small, rusty, and yellow bike"},
+    {"id": 4, "body": "fred's small bike"},
+    {"id": 5, "body": "a tiny shelter"}
+]
+```
+The following queries will output: 
+
+- `body:"small bird"~2`: no match []
+- `body:"red bike"~2`: matches [1]
+- `body:"small blue bike"~3`: matches [2]
+- `body:"small bike"`: matches [4]
+- `body:"small bike"~1`: matches [2, 4]
+- `body:"small bike"~2`: matches [2, 4] 
+- `body:"small bike"~3`: matches [2, 3, 4]
+
 ### Escaping Special Characters
 
 Special reserved characters are: `+` , `^`, `` ` ``, `:`, `{`, `}`, `"`, `[`, `]`, `(`, `)`, `~`, `!`, `\\`, `*`, `SPACE`. Such characters can still appear in query terms, but they need to be escaped by an antislash `\` .

quickwit-search/src/tests.rs

@@ -72,6 +72,76 @@ async fn test_single_node_simple() -> anyhow::Result<()> {
     Ok(())
 }
 
+async fn slop_search_and_check(
+    test_sandbox: &TestSandbox,
+    index_id: &str,
+    query: &str,
+    expected_num_match: u64,
+) -> anyhow::Result<()> {
+    let search_request = SearchRequest {
+        index_id: index_id.to_string(),
+        query: query.to_string(),
+        search_fields: vec!["body".to_string()],
+        start_timestamp: None,
+        end_timestamp: None,
+        max_hits: 5,
+        start_offset: 0,
+        ..Default::default()
+    };
+    let single_node_result = single_node_search(
+        &search_request,
+        &*test_sandbox.metastore(),
+        test_sandbox.storage_uri_resolver(),
+    )
+    .await?;
+    assert_eq!(
+        single_node_result.num_hits, expected_num_match,
+        "query: {}",
+        query
+    );
+    assert_eq!(
+        single_node_result.hits.len(),
+        expected_num_match as usize,
+        "query: {}",
+        query
+    );
+    Ok(())
+}
+
+#[tokio::test]
+async fn test_slop_queries() -> anyhow::Result<()> {
+    let index_id = "slop-query";
+    let doc_mapping_yaml = r#"
+            field_mappings:
+              - name: title
+                type: text
+              - name: body
+                type: text
+                record: position
+        "#;
+
+    let test_sandbox = TestSandbox::create(index_id, doc_mapping_yaml, "{}", &["body"]).await?;
+    let docs = vec![
+        json!({"title": "one", "body": "a red bike"}),
+        json!({"title": "two", "body": "a small blue bike"}),
+        json!({"title": "three", "body": "a small, rusty, and yellow bike"}),
+        json!({"title": "four", "body": "fred's small bike"}),
+        json!({"title": "five", "body": "a tiny shelter"}),
+    ];
+    test_sandbox.add_documents(docs.clone()).await?;
+
+    slop_search_and_check(&test_sandbox, index_id, "\"small bird\"~2", 0).await?;
+    slop_search_and_check(&test_sandbox, index_id, "\"red bike\"~2", 1).await?;
+    slop_search_and_check(&test_sandbox, index_id, "\"small blue bike\"~3", 1).await?;
+    slop_search_and_check(&test_sandbox, index_id, "\"small bike\"", 1).await?;
+    slop_search_and_check(&test_sandbox, index_id, "\"small bike\"~1", 2).await?;
+    slop_search_and_check(&test_sandbox, index_id, "\"small bike\"~2", 2).await?;
+    slop_search_and_check(&test_sandbox, index_id, "\"small bike\"~3", 3).await?;
+    slop_search_and_check(&test_sandbox, index_id, "\"tiny shelter\"~3", 1).await?;
+
+    Ok(())
+}
+
 // TODO remove me once `Iterator::is_sorted_by_key` is stabilized.
 fn is_sorted<E, I: Iterator<Item = E>>(mut it: I) -> bool
 where E: Ord {