Merge pull request #4189 from aduffeck/add-total-to-search

Search: Return the number of total matches

Makefile

@@ -181,7 +181,7 @@ go-coverage:
 
 .PHONY: protobuf
 protobuf:
-	@for mod in $(OCIS_MODULES); do \
+	@for mod in ./services/thumbnails ./services/store ./services/settings; do \
         echo -n "% protobuf $$mod: "; $(MAKE) --no-print-directory -C $$mod protobuf || exit 1; \
     done
 

changelog/unreleased/add-total-to-search-results.md

@@ -0,0 +1,6 @@
+Enhancement: Add number of total matches to the search result
+
+The search service now returns the number of total matches alongside the
+results.
+
+https://github.com/owncloud/ocis/issues/4189

protogen/gen/ocis/services/search/v0/search.swagger.json

@@ -280,6 +280,10 @@
         "nextPageToken": {
           "type": "string",
           "title": "Token to retrieve the next page of results, or empty if there are no\nmore results in the list"
+        },
+        "totalMatches": {
+          "type": "integer",
+          "format": "int32"
         }
       }
     },
@@ -312,6 +316,10 @@
         "nextPageToken": {
           "type": "string",
           "title": "Token to retrieve the next page of results, or empty if there are no\nmore results in the list"
+        },
+        "totalMatches": {
+          "type": "integer",
+          "format": "int32"
         }
       }
     }

protogen/proto/ocis/services/search/v0/search.proto

@@ -76,6 +76,7 @@ message SearchResponse {
   // Token to retrieve the next page of results, or empty if there are no
   // more results in the list
   string next_page_token = 2;
+  int32 total_matches = 3;
 }
 
 message SearchIndexRequest {
@@ -91,11 +92,12 @@ message SearchIndexRequest {
 }
 
 message SearchIndexResponse {
-    repeated ocis.messages.search.v0.Match matches = 1;
+  repeated ocis.messages.search.v0.Match matches = 1;
 
-    // Token to retrieve the next page of results, or empty if there are no
-    // more results in the list
-    string next_page_token = 2;
+  // Token to retrieve the next page of results, or empty if there are no
+  // more results in the list
+  string next_page_token = 2;
+  int32 total_matches = 3;
 }
 
 message IndexSpaceRequest {

services/search/pkg/search/index/index.go

@@ -27,12 +27,13 @@ import (
 	"strings"
 	"time"
 
-	"github.com/blevesearch/bleve/v2"
+	bleve "github.com/blevesearch/bleve/v2"
 	"github.com/blevesearch/bleve/v2/analysis/analyzer/custom"
 	"github.com/blevesearch/bleve/v2/analysis/analyzer/keyword"
 	"github.com/blevesearch/bleve/v2/analysis/token/lowercase"
 	"github.com/blevesearch/bleve/v2/analysis/tokenizer/single"
 	"github.com/blevesearch/bleve/v2/mapping"
+	"github.com/blevesearch/bleve/v2/search"
 	"github.com/cs3org/reva/v2/pkg/storagespace"
 	"google.golang.org/protobuf/types/known/timestamppb"
 
@@ -239,15 +240,16 @@ func (i *Index) Search(ctx context.Context, req *searchsvc.SearchIndexRequest) (
 
 	matches := []*searchmsg.Match{}
 	for _, h := range res.Hits {
-		match, err := fromFields(h.Fields)
+		match, err := fromDocumentMatch(h)
 		if err != nil {
 			return nil, err
 		}
 		matches = append(matches, match)
 	}
 
 	return &searchsvc.SearchIndexResponse{
-		Matches: matches,
+		Matches:      matches,
+		TotalMatches: int32(res.Total),
 	}, nil
 }
 
@@ -311,32 +313,33 @@ func fieldsToEntity(fields map[string]interface{}) *indexDocument {
 	return doc
 }
 
-func fromFields(fields map[string]interface{}) (*searchmsg.Match, error) {
-	rootID, err := storagespace.ParseID(fields["RootID"].(string))
+func fromDocumentMatch(hit *search.DocumentMatch) (*searchmsg.Match, error) {
+	rootID, err := storagespace.ParseID(hit.Fields["RootID"].(string))
 	if err != nil {
 		return nil, err
 	}
-	rID, err := storagespace.ParseID(fields["ID"].(string))
+	rID, err := storagespace.ParseID(hit.Fields["ID"].(string))
 	if err != nil {
 		return nil, err
 	}
 
 	match := &searchmsg.Match{
+		Score: float32(hit.Score),
 		Entity: &searchmsg.Entity{
 			Ref: &searchmsg.Reference{
 				ResourceId: resourceIDtoSearchID(rootID),
-				Path:       fields["Path"].(string),
+				Path:       hit.Fields["Path"].(string),
 			},
 			Id:       resourceIDtoSearchID(rID),
-			Name:     fields["Name"].(string),
-			Size:     uint64(fields["Size"].(float64)),
-			Type:     uint64(fields["Type"].(float64)),
-			MimeType: fields["MimeType"].(string),
-			Deleted:  fields["Deleted"].(bool),
+			Name:     hit.Fields["Name"].(string),
+			Size:     uint64(hit.Fields["Size"].(float64)),
+			Type:     uint64(hit.Fields["Type"].(float64)),
+			MimeType: hit.Fields["MimeType"].(string),
+			Deleted:  hit.Fields["Deleted"].(bool),
 		},
 	}
 
-	if mtime, err := time.Parse(time.RFC3339, fields["Mtime"].(string)); err == nil {
+	if mtime, err := time.Parse(time.RFC3339, hit.Fields["Mtime"].(string)); err == nil {
 		match.Entity.LastModifiedTime = &timestamppb.Timestamp{Seconds: mtime.Unix(), Nanos: int32(mtime.Nanosecond())}
 	}
 

services/search/pkg/search/index/index_test.go

@@ -190,6 +190,19 @@ var _ = Describe("Index", func() {
 					assertDocCount(ref.ResourceId, "Name:*"+ref.ResourceId.OpaqueId+"*", 0)
 				})
 
+				It("returns the total number of hits", func() {
+					res, err := i.Search(ctx, &searchsvc.SearchIndexRequest{
+						Query: "Name:foo.pdf",
+						Ref: &searchmsg.Reference{
+							ResourceId: &searchmsg.ResourceID{
+								StorageId: ref.ResourceId.StorageId,
+								OpaqueId:  ref.ResourceId.OpaqueId,
+							},
+						},
+					})
+					Expect(err).ToNot(HaveOccurred())
+					Expect(res.TotalMatches).To(Equal(int32(1)))
+				})
 				It("returns all desired fields", func() {
 					matches := assertDocCount(ref.ResourceId, "Name:foo.pdf", 1)
 					match := matches[0]
@@ -201,6 +214,7 @@ var _ = Describe("Index", func() {
 					Expect(match.Entity.Type).To(Equal(uint64(ri.Type)))
 					Expect(match.Entity.MimeType).To(Equal(ri.MimeType))
 					Expect(match.Entity.Deleted).To(BeFalse())
+					Expect(match.Score > 0).To(BeTrue())
 					Expect(uint64(match.Entity.LastModifiedTime.AsTime().Unix())).To(Equal(ri.Mtime.Seconds))
 				})
 

services/search/pkg/search/provider/searchprovider.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"path/filepath"
+	"sort"
 	"strings"
 	"time"
 
@@ -44,6 +45,18 @@ type Provider struct {
 	machineAuthAPIKey string
 }
 
+type MatchArray []*searchmsg.Match
+
+func (s MatchArray) Len() int {
+	return len(s)
+}
+func (s MatchArray) Swap(i, j int) {
+	s[i], s[j] = s[j], s[i]
+}
+func (s MatchArray) Less(i, j int) bool {
+	return s[i].Score > s[j].Score
+}
+
 func New(gwClient gateway.GatewayAPIClient, indexClient search.IndexClient, machineAuthAPIKey string, eventsChan <-chan interface{}, logger log.Logger) *Provider {
 	p := &Provider{
 		gwClient:          gwClient,
@@ -98,7 +111,8 @@ func (p *Provider) Search(ctx context.Context, req *searchsvc.SearchRequest) (*s
 		mountpointMap[grantSpaceId] = space.Id.OpaqueId
 	}
 
-	matches := []*searchmsg.Match{}
+	matches := MatchArray{}
+	total := int32(0)
 	for _, space := range listSpacesRes.StorageSpaces {
 		var mountpointRootId *searchmsg.ResourceID
 		mountpointPrefix := ""
@@ -154,6 +168,7 @@ func (p *Provider) Search(ctx context.Context, req *searchsvc.SearchRequest) (*s
 		}
 		p.logger.Debug().Str("space", space.Id.OpaqueId).Int("hits", len(res.Matches)).Msg("space search done")
 
+		total += res.TotalMatches
 		for _, match := range res.Matches {
 			if mountpointPrefix != "" {
 				match.Entity.Ref.Path = utils.MakeRelativePath(strings.TrimPrefix(match.Entity.Ref.Path, mountpointPrefix))
@@ -165,8 +180,19 @@ func (p *Provider) Search(ctx context.Context, req *searchsvc.SearchRequest) (*s
 		}
 	}
 
+	// compile one sorted list of matches from all spaces and apply the limit if needed
+	sort.Sort(matches)
+	limit := req.PageSize
+	if limit == 0 {
+		limit = 200
+	}
+	if int32(len(matches)) > limit {
+		matches = matches[0:limit]
+	}
+
 	return &searchsvc.SearchResponse{
-		Matches: matches,
+		Matches:      matches,
+		TotalMatches: total,
 	}, nil
 }