fvh adaptor to highlight the top boost phrase only

build.gradle

@@ -26,7 +26,7 @@ java {
 }
 
 allprojects {
-    version = '0.42.0'
+    version = '0.42.1'
     group = 'com.yelp.nrtsearch'
 }
 

clientlib/src/main/proto/yelp/nrtsearch/search.proto

@@ -1135,6 +1135,8 @@ message Highlight {
         google.protobuf.UInt32Value  boundary_max_scan = 15;
         // Locale used in boundary scanner when using "word" or "sentence" boundary_scanner. Examples: "en-US", "ch-ZH".
         google.protobuf.StringValue  boundary_scanner_locale = 16;
+        // Only highlight the top matched phrases (with the highest boost value) per fragment. By default, it is false.
+        google.protobuf.BoolValue top_boost_only = 17;
     }
 
     // Highlight settings

grpc-gateway/luceneserver.swagger.json

@@ -1765,6 +1765,10 @@
         "boundaryScannerLocale": {
           "type": "string",
           "description": "Locale used in boundary scanner when using \"word\" or \"sentence\" boundary_scanner. Examples: \"en-US\", \"ch-ZH\"."
+        },
+        "topBoostOnly": {
+          "type": "boolean",
+          "description": "Only highlight the top matched phrases (with the highest boost value) per fragment. By default, it is false."
         }
       }
     },

src/main/java/com/yelp/nrtsearch/server/luceneserver/highlights/HighlightSettings.java

@@ -38,6 +38,7 @@ public class HighlightSettings {
   private final Character[] boundaryChars;
   private final int boundaryMaxScan;
   private final Locale boundaryScannerLocale;
+  private final boolean topBoostOnly;
 
   public HighlightSettings(
       Highlighter highlighter,
@@ -54,6 +55,7 @@ public HighlightSettings(
       Character[] boundaryChars,
       int boundaryMaxScan,
       Locale boundaryScannerLocale,
+      boolean topBoostOnly,
       Map<String, Object> customHighlighterParams) {
     this.highlighter = highlighter;
     this.preTags = preTags;
@@ -69,6 +71,7 @@ public HighlightSettings(
     this.boundaryChars = boundaryChars;
     this.boundaryMaxScan = boundaryMaxScan;
     this.boundaryScannerLocale = boundaryScannerLocale;
+    this.topBoostOnly = topBoostOnly;
     this.customHighlighterParams = customHighlighterParams;
   }
 
@@ -88,6 +91,7 @@ public Builder toBuilder() {
         .withBoundaryChars(this.boundaryChars)
         .withBoundaryMaxScan(this.boundaryMaxScan)
         .withBoundaryScannerLocale(this.boundaryScannerLocale)
+        .withTopBoostOnly(this.topBoostOnly)
         .withCustomHighlighterParams(this.customHighlighterParams);
   }
 
@@ -147,6 +151,10 @@ public Locale getBoundaryScannerLocale() {
     return boundaryScannerLocale;
   }
 
+  public boolean getTopBoostOnly() {
+    return topBoostOnly;
+  }
+
   public Map<String, Object> getCustomHighlighterParams() {
     return customHighlighterParams;
   }
@@ -182,10 +190,12 @@ public String toString() {
         + '\''
         + ", boundaryChars="
         + Arrays.toString(boundaryChars)
-        + ", boundaryCharsMaxScan="
+        + ", boundaryMaxScan="
         + boundaryMaxScan
         + ", boundaryScannerLocale="
-        + boundaryScannerLocale.toLanguageTag()
+        + boundaryScannerLocale
+        + ", topBoostOnly="
+        + topBoostOnly
         + '}';
   }
 
@@ -205,6 +215,7 @@ public static final class Builder {
     private Character[] boundaryChars;
     private int boundaryMaxScan;
     private Locale boundaryScannerLocale;
+    private boolean topBoostOnly;
     private Map<String, Object> customHighlighterParams;
 
     public Builder() {}
@@ -279,6 +290,11 @@ public Builder withBoundaryScannerLocale(Locale boundaryScannerLocale) {
       return this;
     }
 
+    public Builder withTopBoostOnly(boolean topBoostOnly) {
+      this.topBoostOnly = topBoostOnly;
+      return this;
+    }
+
     public Builder withCustomHighlighterParams(Map<String, Object> customHighlighterParams) {
       this.customHighlighterParams = customHighlighterParams;
       return this;
@@ -300,6 +316,7 @@ public HighlightSettings build() {
           boundaryChars,
           boundaryMaxScan,
           boundaryScannerLocale,
+          topBoostOnly,
           customHighlighterParams);
     }
   }

src/main/java/com/yelp/nrtsearch/server/luceneserver/highlights/HighlightUtils.java

@@ -44,6 +44,7 @@ public class HighlightUtils {
   private static final boolean DEFAULT_DISCRETE_MULTIVALUE = false;
   private static final Character[] DEFAULT_BOUNDARY_CHARS =
       SimpleBoundaryScanner.DEFAULT_BOUNDARY_CHARS;
+  private static final boolean DEFAULT_TOP_PHRASE_ONCE = false;
   private static final int DEFAULT_BOUNDARY_MAX_SCAN = SimpleBoundaryScanner.DEFAULT_MAX_SCAN;
   private static final Locale DEFAULT_BOUNDARY_SCANNER_LOCALE = Locale.ROOT;
   private static final QueryNodeMapper QUERY_NODE_MAPPER = QueryNodeMapper.getInstance();
@@ -132,6 +133,10 @@ static Map<String, HighlightSettings> createPerFieldSettings(
                     settings.hasBoundaryScannerLocale()
                         ? Locale.forLanguageTag(settings.getBoundaryScannerLocale().getValue())
                         : globalSettings.getBoundaryScannerLocale())
+                .withTopBoostOnly(
+                    settings.hasTopBoostOnly()
+                        ? settings.getTopBoostOnly().getValue()
+                        : globalSettings.getTopBoostOnly())
                 .withCustomHighlighterParams(
                     settings.hasCustomHighlighterParams()
                         ? StructValueTransformer.transformStruct(
@@ -212,6 +217,10 @@ private static HighlightSettings createGlobalFieldSettings(
             settings.hasBoundaryScannerLocale()
                 ? Locale.forLanguageTag(settings.getBoundaryScannerLocale().getValue())
                 : DEFAULT_BOUNDARY_SCANNER_LOCALE)
+        .withTopBoostOnly(
+            settings.hasTopBoostOnly()
+                ? settings.getTopBoostOnly().getValue()
+                : DEFAULT_TOP_PHRASE_ONCE)
         .withCustomHighlighterParams(
             settings.hasCustomHighlighterParams()
                 ? StructValueTransformer.transformStruct(settings.getCustomHighlighterParams())

src/main/java/com/yelp/nrtsearch/server/luceneserver/highlights/NRTFastVectorHighlighter.java

@@ -119,12 +119,13 @@ public String[] getHighlights(
           "Unknown boundary scanner: " + settings.getBoundaryScanner());
     }
 
-    BaseFragmentsBuilder fragmentsBuilder;
-    if (settings.isScoreOrdered()) {
-      fragmentsBuilder = new ScoreOrderFragmentsBuilder(boundaryScanner);
-    } else {
-      fragmentsBuilder = new SimpleFragmentsBuilder(boundaryScanner);
-    }
+    BaseFragmentsBuilder fragmentsBuilder =
+        new TopBoostOnlyFragmentsBuilderAdaptor(
+            settings.isScoreOrdered()
+                ? new ScoreOrderFragmentsBuilder()
+                : new SimpleFragmentsBuilder(),
+            boundaryScanner,
+            settings.getTopBoostOnly());
     fragmentsBuilder.setDiscreteMultiValueHighlighting(settings.getDiscreteMultivalue());
 
     try {

src/main/java/com/yelp/nrtsearch/server/luceneserver/highlights/TopBoostOnlyFragmentsBuilderAdaptor.java

@@ -0,0 +1,89 @@
+/*
+ * Copyright 2024 Yelp Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.yelp.nrtsearch.server.luceneserver.highlights;
+
+import java.util.List;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.search.highlight.Encoder;
+import org.apache.lucene.search.vectorhighlight.BaseFragmentsBuilder;
+import org.apache.lucene.search.vectorhighlight.BoundaryScanner;
+import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo;
+import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo.SubInfo;
+import org.apache.lucene.search.vectorhighlight.FieldPhraseList.WeightedPhraseInfo.Toffs;
+
+public class TopBoostOnlyFragmentsBuilderAdaptor extends BaseFragmentsBuilder {
+  private final BaseFragmentsBuilder innerBaseFragmentsBuilder;
+  private final boolean topBoostOnly;
+
+  /** a constructor. */
+  public TopBoostOnlyFragmentsBuilderAdaptor(
+      BaseFragmentsBuilder baseFragmentsBuilder,
+      BoundaryScanner boundaryScanner,
+      boolean topBoostOnly) {
+    super(boundaryScanner);
+    this.innerBaseFragmentsBuilder = baseFragmentsBuilder;
+    this.topBoostOnly = topBoostOnly;
+  }
+
+  @Override
+  public List<WeightedFragInfo> getWeightedFragInfoList(List<WeightedFragInfo> src) {
+    return innerBaseFragmentsBuilder.getWeightedFragInfoList(src);
+  }
+
+  @Override
+  protected String makeFragment(
+      StringBuilder buffer,
+      int[] index,
+      Field[] values,
+      WeightedFragInfo fragInfo,
+      String[] preTags,
+      String[] postTags,
+      Encoder encoder) {
+    if (!topBoostOnly) {
+      return super.makeFragment(buffer, index, values, fragInfo, preTags, postTags, encoder);
+    }
+    StringBuilder fragment = new StringBuilder();
+    final int s = fragInfo.getStartOffset();
+    int[] modifiedStartOffset = {s};
+    String src =
+        getFragmentSourceMSO(
+            buffer, index, values, s, fragInfo.getEndOffset(), modifiedStartOffset);
+    int srcIndex = 0;
+    double topBoostValue =
+        fragInfo.getSubInfos().stream().map(SubInfo::getBoost).max(Float::compare).orElse(0f);
+    for (SubInfo subInfo : fragInfo.getSubInfos()) {
+      if (subInfo.getBoost() < topBoostValue) {
+        continue;
+      }
+      for (Toffs to : subInfo.getTermsOffsets()) {
+        fragment
+            .append(
+                encoder.encodeText(
+                    src.substring(srcIndex, to.getStartOffset() - modifiedStartOffset[0])))
+            .append(getPreTag(preTags, subInfo.getSeqnum()))
+            .append(
+                encoder.encodeText(
+                    src.substring(
+                        to.getStartOffset() - modifiedStartOffset[0],
+                        to.getEndOffset() - modifiedStartOffset[0])))
+            .append(getPostTag(postTags, subInfo.getSeqnum()));
+        srcIndex = to.getEndOffset() - modifiedStartOffset[0];
+      }
+    }
+    fragment.append(encoder.encodeText(src.substring(srcIndex)));
+    return fragment.toString();
+  }
+}

src/test/java/com/yelp/nrtsearch/server/luceneserver/highlights/NRTFastVectorHighlighterTest.java

@@ -24,6 +24,8 @@
 import com.google.protobuf.UInt32Value;
 import com.yelp.nrtsearch.server.grpc.AddDocumentRequest;
 import com.yelp.nrtsearch.server.grpc.AddDocumentRequest.MultiValuedField;
+import com.yelp.nrtsearch.server.grpc.BooleanClause;
+import com.yelp.nrtsearch.server.grpc.BooleanQuery;
 import com.yelp.nrtsearch.server.grpc.FieldDefRequest;
 import com.yelp.nrtsearch.server.grpc.Highlight;
 import com.yelp.nrtsearch.server.grpc.Highlight.Settings;
@@ -78,8 +80,9 @@ protected void initIndex(String name) throws Exception {
                     .addAllValue(
                         List.of(
                             "The food is good there, but the service is terrible.",
-                            "I personally don't like the staff at this place",
-                            "Not all food are good."))
+                            "I personally don't like the staff at this place.",
+                            "Not all food are good.",
+                            "The margarita pizza and the marinara pizza in this pizzeria are yummy and inexpensive."))
                     .build())
             .putFields(
                 "boundary_scanner_field",
@@ -190,6 +193,73 @@ public void testHighlightMultivalueField() {
     assertThat(response.getDiagnostics().getHighlightTimeMs()).isGreaterThan(0);
   }
 
+  @Test
+  public void testHighlightMultivalueFieldWithTopBoostOnly() {
+    Highlight highlight =
+        Highlight.newBuilder()
+            .addFields("comment_multivalue")
+            .setSettings(
+                Settings.newBuilder()
+                    .setHighlightQuery(
+                        Query.newBuilder()
+                            .setBooleanQuery(
+                                BooleanQuery.newBuilder()
+                                    .addClauses(
+                                        BooleanClause.newBuilder()
+                                            .setQuery(
+                                                Query.newBuilder()
+                                                    .setPhraseQuery(
+                                                        PhraseQuery.newBuilder()
+                                                            .setField("comment_multivalue")
+                                                            .addAllTerms(
+                                                                List.of("margarita", "pizza")))
+                                                    .setBoost(3))
+                                            .setOccurValue(BooleanClause.Occur.SHOULD_VALUE))
+                                    .addClauses(
+                                        BooleanClause.newBuilder()
+                                            .setQuery(
+                                                Query.newBuilder()
+                                                    .setPhraseQuery(
+                                                        PhraseQuery.newBuilder()
+                                                            .setField("comment_multivalue")
+                                                            .addAllTerms(
+                                                                List.of("marinara", "pizza")))
+                                                    .setBoost(3))
+                                            .setOccurValue(BooleanClause.Occur.SHOULD_VALUE))
+                                    .addClauses(
+                                        BooleanClause.newBuilder()
+                                            .setQuery(
+                                                Query.newBuilder()
+                                                    .setTermQuery(
+                                                        TermQuery.newBuilder()
+                                                            .setField("comment_multivalue")
+                                                            .setTextValue("delicious"))
+                                                    .setBoost(4)))
+                                    .addClauses(
+                                        BooleanClause.newBuilder()
+                                            .setQuery(
+                                                Query.newBuilder()
+                                                    .setTermQuery(
+                                                        TermQuery.newBuilder()
+                                                            .setField("comment_multivalue")
+                                                            .setTextValue("yummy"))
+                                                    .setBoost(2)))))
+                    .setMaxNumberOfFragments(UInt32Value.of(1))
+                    .setFragmentSize(UInt32Value.of(250))
+                    .setTopBoostOnly(BoolValue.of(true))
+                    .setScoreOrdered(BoolValue.of(true))
+                    .setDiscreteMultivalue(BoolValue.of(true)))
+            .build();
+    SearchResponse response = doHighlightQuery(highlight);
+
+    assertFields(response);
+
+    assertThat(response.getHits(0).getHighlightsMap().get("comment_multivalue").getFragmentsList())
+        .containsExactly(
+            "The <em>margarita pizza</em> and the <em>marinara pizza</em> in this pizzeria are yummy and inexpensive.");
+    assertThat(response.getDiagnostics().getHighlightTimeMs()).isGreaterThan(0);
+  }
+
   @Test
   public void testHighlightGlobalSettings() {
     Settings settings =