Update SolrSpout.java

Solr cloud results "Collapse/Expand" bug apache#1049
syefimov · Apr 6, 2023 · 51acb61 · 51acb61
1 parent fecdd06
commit 51acb61
Showing 1 changed file with 25 additions and 19 deletions.
diff --git a/external/solr/src/main/java/com/digitalpebble/stormcrawler/solr/persistence/SolrSpout.java b/external/solr/src/main/java/com/digitalpebble/stormcrawler/solr/persistence/SolrSpout.java
@@ -24,6 +24,9 @@
 import java.util.Map;
 import org.apache.commons.lang.StringUtils;
 import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrQuery.ORDER;
+import org.apache.solr.client.solrj.response.Group;
+import org.apache.solr.client.solrj.response.GroupCommand;
 import org.apache.solr.client.solrj.response.QueryResponse;
 import org.apache.solr.common.SolrDocument;
 import org.apache.solr.common.SolrDocumentList;
@@ -129,16 +132,14 @@ else if (resetFetchDateAfterNSecs != -1) {
         query.setQuery("*:*")
                 .addFilterQuery("nextFetchDate:[* TO " + lastNextFetchDate + "]")
                 .setStart(lastStartOffset)
-                .setRows(this.maxNumResults);
+                .setRows(this.maxNumResults).setSort("nextFetchDate", ORDER.asc);
 
         if (StringUtils.isNotBlank(diversityField) && diversityBucketSize > 0) {
-            query.addFilterQuery(
-                    String.format("{!collapse field=%s sort='nextFetchDate asc'}", diversityField));
-            query.set("expand", "true").set("expand.rows", diversityBucketSize);
-            query.set("expand.sort", "nextFetchDate asc");
+        	query.set("indent", "true").set("group", "true").set("group.field", diversityField)
+			.set("group.limit", diversityBucketSize).set("group.sort", "nextFetchDate asc");
         }
 
-        LOG.debug("QUERY => {}", query.toString());
+        LOG.debug("QUERY => {}", query);
 
         try {
             long startQuery = System.currentTimeMillis();
@@ -149,20 +150,25 @@ else if (resetFetchDateAfterNSecs != -1) {
 
             SolrDocumentList docs = new SolrDocumentList();
 
-            LOG.debug("Response : {}", response.toString());
+            LOG.debug("Response : {}", response);
 
             // add the main results
-            docs.addAll(response.getResults());
-
-            // Add the documents collapsed by the CollapsingQParser
-            Map<String, SolrDocumentList> expandedResults = response.getExpandedResults();
-            if (StringUtils.isNotBlank(diversityField) && expandedResults != null) {
-                for (String key : expandedResults.keySet()) {
-                    docs.addAll(expandedResults.get(key));
-                }
-            }
-
-            int numhits = response.getResults().size();
+			if (response.getResults() != null) {
+				docs.addAll(response.getResults());
+			}
+			int groupsTotal = 0;
+            // get groups
+			if (response.getGroupResponse() != null) {
+				for (GroupCommand groupCommand : response.getGroupResponse().getValues()) {
+					for (Group group : groupCommand.getValues()) {
+						groupsTotal++;
+						LOG.debug("Group : {}", group);
+						docs.addAll(group.getResult());
+					}
+				}
+			}
+
+            int numhits = (response.getResults()!=null)?response.getResults().size():groupsTotal;
 
             // no more results?
             if (numhits == 0) {
@@ -176,7 +182,7 @@ else if (resetFetchDateAfterNSecs != -1) {
 
             int alreadyProcessed = 0;
             int docReturned = 0;
-
+            
             for (SolrDocument doc : docs) {
                 String url = (String) doc.get("url");