bleve v2.0.0 proposal PR

analysis/analyzer/custom/custom.go

@@ -17,8 +17,8 @@ package custom
 import (
 	"fmt"
 
-	"github.com/blevesearch/bleve/analysis"
-	"github.com/blevesearch/bleve/registry"
+	"github.com/blevesearch/bleve/v2/analysis"
+	"github.com/blevesearch/bleve/v2/registry"
 )
 
 const Name = "custom"

analysis/analyzer/keyword/keyword.go

@@ -15,9 +15,9 @@
 package keyword
 
 import (
-	"github.com/blevesearch/bleve/analysis"
-	"github.com/blevesearch/bleve/analysis/tokenizer/single"
-	"github.com/blevesearch/bleve/registry"
+	"github.com/blevesearch/bleve/v2/analysis"
+	"github.com/blevesearch/bleve/v2/analysis/tokenizer/single"
+	"github.com/blevesearch/bleve/v2/registry"
 )
 
 const Name = "keyword"

analysis/analyzer/simple/simple.go

@@ -15,10 +15,10 @@
 package simple
 
 import (
-	"github.com/blevesearch/bleve/analysis"
-	"github.com/blevesearch/bleve/analysis/token/lowercase"
-	"github.com/blevesearch/bleve/analysis/tokenizer/letter"
-	"github.com/blevesearch/bleve/registry"
+	"github.com/blevesearch/bleve/v2/analysis"
+	"github.com/blevesearch/bleve/v2/analysis/token/lowercase"
+	"github.com/blevesearch/bleve/v2/analysis/tokenizer/letter"
+	"github.com/blevesearch/bleve/v2/registry"
 )
 
 const Name = "simple"

analysis/analyzer/standard/standard.go

@@ -15,11 +15,11 @@
 package standard
 
 import (
-	"github.com/blevesearch/bleve/analysis"
-	"github.com/blevesearch/bleve/analysis/lang/en"
-	"github.com/blevesearch/bleve/analysis/token/lowercase"
-	"github.com/blevesearch/bleve/analysis/tokenizer/unicode"
-	"github.com/blevesearch/bleve/registry"
+	"github.com/blevesearch/bleve/v2/analysis"
+	"github.com/blevesearch/bleve/v2/analysis/lang/en"
+	"github.com/blevesearch/bleve/v2/analysis/token/lowercase"
+	"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
+	"github.com/blevesearch/bleve/v2/registry"
 )
 
 const Name = "standard"

analysis/analyzer/web/web.go

@@ -15,11 +15,11 @@
 package web
 
 import (
-	"github.com/blevesearch/bleve/analysis"
-	"github.com/blevesearch/bleve/analysis/lang/en"
-	"github.com/blevesearch/bleve/analysis/token/lowercase"
-	"github.com/blevesearch/bleve/analysis/tokenizer/web"
-	"github.com/blevesearch/bleve/registry"
+	"github.com/blevesearch/bleve/v2/analysis"
+	"github.com/blevesearch/bleve/v2/analysis/lang/en"
+	"github.com/blevesearch/bleve/v2/analysis/token/lowercase"
+	"github.com/blevesearch/bleve/v2/analysis/tokenizer/web"
+	"github.com/blevesearch/bleve/v2/registry"
 )
 
 const Name = "web"

analysis/benchmark_test.go

@@ -15,11 +15,12 @@
 package analysis_test
 
 import (
+	index "github.com/blevesearch/bleve_index_api"
 	"testing"
 
-	"github.com/blevesearch/bleve/analysis"
-	"github.com/blevesearch/bleve/analysis/analyzer/standard"
-	"github.com/blevesearch/bleve/registry"
+	"github.com/blevesearch/bleve/v2/analysis"
+	"github.com/blevesearch/bleve/v2/analysis/analyzer/standard"
+	"github.com/blevesearch/bleve/v2/registry"
 )
 
 func BenchmarkAnalysis(b *testing.B) {
@@ -32,7 +33,7 @@ func BenchmarkAnalysis(b *testing.B) {
 		}
 
 		ts := analyzer.Analyze(bleveWikiArticle)
-		freqs := analysis.TokenFrequency(ts, nil, true)
+		freqs := analysis.TokenFrequency(ts, nil, index.IncludeTermVectors)
 		if len(freqs) != 511 {
 			b.Errorf("expected %d freqs, got %d", 511, len(freqs))
 		}

analysis/char/asciifolding/asciifolding.go

@@ -18,8 +18,8 @@
 package asciifolding
 
 import (
-	"github.com/blevesearch/bleve/analysis"
-	"github.com/blevesearch/bleve/registry"
+	"github.com/blevesearch/bleve/v2/analysis"
+	"github.com/blevesearch/bleve/v2/registry"
 )
 
 const Name = "asciifolding"

analysis/char/html/html.go

@@ -17,9 +17,9 @@ package html
 import (
 	"regexp"
 
-	"github.com/blevesearch/bleve/analysis"
-	regexpCharFilter "github.com/blevesearch/bleve/analysis/char/regexp"
-	"github.com/blevesearch/bleve/registry"
+	"github.com/blevesearch/bleve/v2/analysis"
+	regexpCharFilter "github.com/blevesearch/bleve/v2/analysis/char/regexp"
+	"github.com/blevesearch/bleve/v2/registry"
 )
 
 const Name = "html"

analysis/char/regexp/regexp.go

@@ -18,8 +18,8 @@ import (
 	"fmt"
 	"regexp"
 
-	"github.com/blevesearch/bleve/analysis"
-	"github.com/blevesearch/bleve/registry"
+	"github.com/blevesearch/bleve/v2/analysis"
+	"github.com/blevesearch/bleve/v2/registry"
 )
 
 const Name = "regexp"

analysis/char/zerowidthnonjoiner/zerowidthnonjoiner.go

@@ -17,9 +17,9 @@ package zerowidthnonjoiner
 import (
 	"regexp"
 
-	"github.com/blevesearch/bleve/analysis"
-	regexpCharFilter "github.com/blevesearch/bleve/analysis/char/regexp"
-	"github.com/blevesearch/bleve/registry"
+	"github.com/blevesearch/bleve/v2/analysis"
+	regexpCharFilter "github.com/blevesearch/bleve/v2/analysis/char/regexp"
+	"github.com/blevesearch/bleve/v2/registry"
 )
 
 const Name = "zero_width_spaces"

analysis/datetime/flexible/flexible.go

@@ -18,8 +18,8 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/blevesearch/bleve/analysis"
-	"github.com/blevesearch/bleve/registry"
+	"github.com/blevesearch/bleve/v2/analysis"
+	"github.com/blevesearch/bleve/v2/registry"
 )
 
 const Name = "flexiblego"

analysis/datetime/flexible/flexible_test.go

@@ -19,7 +19,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/blevesearch/bleve/analysis"
+	"github.com/blevesearch/bleve/v2/analysis"
 )
 
 func TestFlexibleDateTimeParser(t *testing.T) {

analysis/datetime/optional/optional.go

@@ -17,9 +17,9 @@ package optional
 import (
 	"time"
 
-	"github.com/blevesearch/bleve/analysis"
-	"github.com/blevesearch/bleve/analysis/datetime/flexible"
-	"github.com/blevesearch/bleve/registry"
+	"github.com/blevesearch/bleve/v2/analysis"
+	"github.com/blevesearch/bleve/v2/analysis/datetime/flexible"
+	"github.com/blevesearch/bleve/v2/registry"
 )
 
 const Name = "dateTimeOptional"

analysis/freq.go

@@ -15,105 +15,18 @@
 package analysis
 
 import (
-	"reflect"
-
-	"github.com/blevesearch/bleve/size"
+	index "github.com/blevesearch/bleve_index_api"
 )
 
-var reflectStaticSizeTokenLocation int
-var reflectStaticSizeTokenFreq int
-
-func init() {
-	var tl TokenLocation
-	reflectStaticSizeTokenLocation = int(reflect.TypeOf(tl).Size())
-	var tf TokenFreq
-	reflectStaticSizeTokenFreq = int(reflect.TypeOf(tf).Size())
-}
-
-// TokenLocation represents one occurrence of a term at a particular location in
-// a field. Start, End and Position have the same meaning as in analysis.Token.
-// Field and ArrayPositions identify the field value in the source document.
-// See document.Field for details.
-type TokenLocation struct {
-	Field          string
-	ArrayPositions []uint64
-	Start          int
-	End            int
-	Position       int
-}
-
-func (tl *TokenLocation) Size() int {
-	rv := reflectStaticSizeTokenLocation
-	rv += len(tl.ArrayPositions) * size.SizeOfUint64
-	return rv
-}
-
-// TokenFreq represents all the occurrences of a term in all fields of a
-// document.
-type TokenFreq struct {
-	Term      []byte
-	Locations []*TokenLocation
-	frequency int
-}
-
-func (tf *TokenFreq) Size() int {
-	rv := reflectStaticSizeTokenFreq
-	rv += len(tf.Term)
-	for _, loc := range tf.Locations {
-		rv += loc.Size()
-	}
-	return rv
-}
-
-func (tf *TokenFreq) Frequency() int {
-	return tf.frequency
-}
-
-// TokenFrequencies maps document terms to their combined frequencies from all
-// fields.
-type TokenFrequencies map[string]*TokenFreq
+func TokenFrequency(tokens TokenStream, arrayPositions []uint64, options index.FieldIndexingOptions) index.TokenFrequencies {
+	rv := make(map[string]*index.TokenFreq, len(tokens))
 
-func (tfs TokenFrequencies) Size() int {
-	rv := size.SizeOfMap
-	rv += len(tfs) * (size.SizeOfString + size.SizeOfPtr)
-	for k, v := range tfs {
-		rv += len(k)
-		rv += v.Size()
-	}
-	return rv
-}
-
-func (tfs TokenFrequencies) MergeAll(remoteField string, other TokenFrequencies) {
-	// walk the new token frequencies
-	for tfk, tf := range other {
-		// set the remoteField value in incoming token freqs
-		for _, l := range tf.Locations {
-			l.Field = remoteField
-		}
-		existingTf, exists := tfs[tfk]
-		if exists {
-			existingTf.Locations = append(existingTf.Locations, tf.Locations...)
-			existingTf.frequency = existingTf.frequency + tf.frequency
-		} else {
-			tfs[tfk] = &TokenFreq{
-				Term:      tf.Term,
-				frequency: tf.frequency,
-				Locations: make([]*TokenLocation, len(tf.Locations)),
-			}
-			copy(tfs[tfk].Locations, tf.Locations)
-		}
-	}
-}
-
-func TokenFrequency(tokens TokenStream, arrayPositions []uint64, includeTermVectors bool) TokenFrequencies {
-	rv := make(map[string]*TokenFreq, len(tokens))
-
-	if includeTermVectors {
-		tls := make([]TokenLocation, len(tokens))
+	if options.IncludeTermVectors() {
+		tls := make([]index.TokenLocation, len(tokens))
 		tlNext := 0
 
 		for _, token := range tokens {
-			tls[tlNext] = TokenLocation{
+			tls[tlNext] = index.TokenLocation{
 				ArrayPositions: arrayPositions,
 				Start:          token.Start,
 				End:            token.End,
@@ -123,27 +36,32 @@ func TokenFrequency(tokens TokenStream, arrayPositions []uint64, includeTermVect
 			curr, ok := rv[string(token.Term)]
 			if ok {
 				curr.Locations = append(curr.Locations, &tls[tlNext])
-				curr.frequency++
 			} else {
-				rv[string(token.Term)] = &TokenFreq{
+				curr = &index.TokenFreq{
 					Term:      token.Term,
-					Locations: []*TokenLocation{&tls[tlNext]},
-					frequency: 1,
+					Locations: []*index.TokenLocation{&tls[tlNext]},
 				}
+				rv[string(token.Term)] = curr
+			}
+
+			if !options.SkipFreqNorm() {
+				curr.SetFrequency(curr.Frequency() + 1)
 			}
 
 			tlNext++
 		}
 	} else {
 		for _, token := range tokens {
 			curr, exists := rv[string(token.Term)]
-			if exists {
-				curr.frequency++
-			} else {
-				rv[string(token.Term)] = &TokenFreq{
-					Term:      token.Term,
-					frequency: 1,
+			if !exists {
+				curr = &index.TokenFreq{
+					Term: token.Term,
 				}
+				rv[string(token.Term)] = curr
+			}
+
+			if !options.SkipFreqNorm() {
+				curr.SetFrequency(curr.Frequency() + 1)
 			}
 		}
 	}