Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

use the moved indexing options #1521

Merged
merged 2 commits into from
Dec 16, 2020
Merged

use the moved indexing options #1521

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
edada75
use the moved indexing options
mschoch Dec 16, 2020
3e034a5
update analysis.TokenFrequency to use options (#1522)
mschoch Dec 16, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion analysis/benchmark_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
package analysis_test

import (
index "github.com/blevesearch/bleve_index_api"
"testing"

"github.com/blevesearch/bleve/analysis"
Expand All @@ -32,7 +33,7 @@ func BenchmarkAnalysis(b *testing.B) {
}

ts := analyzer.Analyze(bleveWikiArticle)
freqs := analysis.TokenFrequency(ts, nil, true)
freqs := analysis.TokenFrequency(ts, nil, index.IncludeTermVectors)
if len(freqs) != 511 {
b.Errorf("expected %d freqs, got %d", 511, len(freqs))
}
Expand Down
4 changes: 2 additions & 2 deletions analysis/freq.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,10 @@ import (
index "github.com/blevesearch/bleve_index_api"
)

func TokenFrequency(tokens TokenStream, arrayPositions []uint64, includeTermVectors bool) index.TokenFrequencies {
func TokenFrequency(tokens TokenStream, arrayPositions []uint64, options index.FieldIndexingOptions) index.TokenFrequencies {
rv := make(map[string]*index.TokenFreq, len(tokens))

if includeTermVectors {
if options.IncludeTermVectors() {
tls := make([]index.TokenLocation, len(tokens))
tlNext := 0

Expand Down
2 changes: 1 addition & 1 deletion analysis/freq_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ func TestTokenFrequency(t *testing.T) {
},
}
expectedResult["water"].SetFrequency(2)
result := TokenFrequency(tokens, nil, true)
result := TokenFrequency(tokens, nil, index.IncludeTermVectors)
if !reflect.DeepEqual(result, expectedResult) {
t.Errorf("expected %#v, got %#v", expectedResult, result)
}
Expand Down
2 changes: 1 addition & 1 deletion document/document.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ func (d *Document) SetID(id string) {
}

func (d *Document) AddIDField() {
d.AddField(NewTextFieldCustom("_id", nil, []byte(d.ID()), IndexField|StoreField, nil))
d.AddField(NewTextFieldCustom("_id", nil, []byte(d.ID()), index.IndexField|index.StoreField, nil))
}

func (d *Document) VisitFields(visitor index.FieldVisitor) {
Expand Down
6 changes: 1 addition & 5 deletions document/field.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ type Field interface {
// arrays, ArrayPositions returns 2 indices used to resolve "doc2" value in
// "doc1", then "field" in "doc2".
ArrayPositions() []uint64
Options() IndexingOptions
Options() index.FieldIndexingOptions
Analyze()
Value() []byte

Expand All @@ -40,10 +40,6 @@ type Field interface {
Size() int

EncodedFieldType() byte
IsIndexed() bool
IsStored() bool
IncludeDocValues() bool
IncludeTermVectors() bool
AnalyzedLength() int
AnalyzedTokenFrequencies() index.TokenFrequencies
}
26 changes: 5 additions & 21 deletions document/field_boolean.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,12 @@ func init() {
reflectStaticSizeBooleanField = int(reflect.TypeOf(f).Size())
}

const DefaultBooleanIndexingOptions = StoreField | IndexField | DocValues
const DefaultBooleanIndexingOptions = index.StoreField | index.IndexField | index.DocValues

type BooleanField struct {
name string
arrayPositions []uint64
options IndexingOptions
options index.FieldIndexingOptions
value []byte
numPlainTextBytes uint64
length int
Expand All @@ -57,7 +57,7 @@ func (b *BooleanField) ArrayPositions() []uint64 {
return b.arrayPositions
}

func (b *BooleanField) Options() IndexingOptions {
func (b *BooleanField) Options() index.FieldIndexingOptions {
return b.options
}

Expand All @@ -72,7 +72,7 @@ func (b *BooleanField) Analyze() {
})

b.length = len(tokens)
b.frequencies = analysis.TokenFrequency(tokens, b.arrayPositions, b.options.IncludeTermVectors())
b.frequencies = analysis.TokenFrequency(tokens, b.arrayPositions, b.options)
}

func (b *BooleanField) Value() []byte {
Expand All @@ -98,22 +98,6 @@ func (b *BooleanField) EncodedFieldType() byte {
return 'b'
}

func (b *BooleanField) IsIndexed() bool {
return b.options.IsIndexed()
}

func (b *BooleanField) IsStored() bool {
return b.options.IsStored()
}

func (b *BooleanField) IncludeDocValues() bool {
return b.options.IncludeDocValues()
}

func (b *BooleanField) IncludeTermVectors() bool {
return b.options.IncludeTermVectors()
}

func (b *BooleanField) AnalyzedLength() int {
return b.length
}
Expand All @@ -136,7 +120,7 @@ func NewBooleanField(name string, arrayPositions []uint64, b bool) *BooleanField
return NewBooleanFieldWithIndexingOptions(name, arrayPositions, b, DefaultNumericIndexingOptions)
}

func NewBooleanFieldWithIndexingOptions(name string, arrayPositions []uint64, b bool, options IndexingOptions) *BooleanField {
func NewBooleanFieldWithIndexingOptions(name string, arrayPositions []uint64, b bool, options index.FieldIndexingOptions) *BooleanField {
numPlainTextBytes := 5
v := []byte("F")
if b {
Expand Down
24 changes: 4 additions & 20 deletions document/field_composite.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,14 @@ func init() {
reflectStaticSizeCompositeField = int(reflect.TypeOf(cf).Size())
}

const DefaultCompositeIndexingOptions = IndexField
const DefaultCompositeIndexingOptions = index.IndexField

type CompositeField struct {
name string
includedFields map[string]bool
excludedFields map[string]bool
defaultInclude bool
options IndexingOptions
options index.FieldIndexingOptions
totalLength int
compositeFrequencies index.TokenFrequencies
}
Expand All @@ -44,7 +44,7 @@ func NewCompositeField(name string, defaultInclude bool, include []string, exclu
return NewCompositeFieldWithIndexingOptions(name, defaultInclude, include, exclude, DefaultCompositeIndexingOptions)
}

func NewCompositeFieldWithIndexingOptions(name string, defaultInclude bool, include []string, exclude []string, options IndexingOptions) *CompositeField {
func NewCompositeFieldWithIndexingOptions(name string, defaultInclude bool, include []string, exclude []string, options index.FieldIndexingOptions) *CompositeField {
rv := &CompositeField{
name: name,
options: options,
Expand Down Expand Up @@ -87,7 +87,7 @@ func (c *CompositeField) ArrayPositions() []uint64 {
return []uint64{}
}

func (c *CompositeField) Options() IndexingOptions {
func (c *CompositeField) Options() index.FieldIndexingOptions {
return c.options
}

Expand Down Expand Up @@ -126,22 +126,6 @@ func (c *CompositeField) EncodedFieldType() byte {
return 'c'
}

func (c *CompositeField) IsIndexed() bool {
return c.options.IsIndexed()
}

func (c *CompositeField) IsStored() bool {
return c.options.IsStored()
}

func (c *CompositeField) IncludeDocValues() bool {
return c.options.IncludeDocValues()
}

func (c *CompositeField) IncludeTermVectors() bool {
return c.options.IncludeTermVectors()
}

func (c *CompositeField) AnalyzedLength() int {
return c.totalLength
}
Expand Down
26 changes: 5 additions & 21 deletions document/field_datetime.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ func init() {
reflectStaticSizeDateTimeField = int(reflect.TypeOf(f).Size())
}

const DefaultDateTimeIndexingOptions = StoreField | IndexField | DocValues
const DefaultDateTimeIndexingOptions = index.StoreField | index.IndexField | index.DocValues
const DefaultDateTimePrecisionStep uint = 4

var MinTimeRepresentable = time.Unix(0, math.MinInt64)
Expand All @@ -42,7 +42,7 @@ var MaxTimeRepresentable = time.Unix(0, math.MaxInt64)
type DateTimeField struct {
name string
arrayPositions []uint64
options IndexingOptions
options index.FieldIndexingOptions
value numeric.PrefixCoded
numPlainTextBytes uint64
length int
Expand All @@ -63,30 +63,14 @@ func (n *DateTimeField) ArrayPositions() []uint64 {
return n.arrayPositions
}

func (n *DateTimeField) Options() IndexingOptions {
func (n *DateTimeField) Options() index.FieldIndexingOptions {
return n.options
}

func (n *DateTimeField) EncodedFieldType() byte {
return 'd'
}

func (n *DateTimeField) IsIndexed() bool {
return n.options.IsIndexed()
}

func (n *DateTimeField) IsStored() bool {
return n.options.IsStored()
}

func (n *DateTimeField) IncludeDocValues() bool {
return n.options.IncludeDocValues()
}

func (n *DateTimeField) IncludeTermVectors() bool {
return n.options.IncludeTermVectors()
}

func (n *DateTimeField) AnalyzedLength() int {
return n.length
}
Expand Down Expand Up @@ -127,7 +111,7 @@ func (n *DateTimeField) Analyze() {
}

n.length = len(tokens)
n.frequencies = analysis.TokenFrequency(tokens, n.arrayPositions, n.options.IncludeTermVectors())
n.frequencies = analysis.TokenFrequency(tokens, n.arrayPositions, n.options)
}

func (n *DateTimeField) Value() []byte {
Expand Down Expand Up @@ -164,7 +148,7 @@ func NewDateTimeField(name string, arrayPositions []uint64, dt time.Time) (*Date
return NewDateTimeFieldWithIndexingOptions(name, arrayPositions, dt, DefaultDateTimeIndexingOptions)
}

func NewDateTimeFieldWithIndexingOptions(name string, arrayPositions []uint64, dt time.Time, options IndexingOptions) (*DateTimeField, error) {
func NewDateTimeFieldWithIndexingOptions(name string, arrayPositions []uint64, dt time.Time, options index.FieldIndexingOptions) (*DateTimeField, error) {
if canRepresent(dt) {
dtInt64 := dt.UnixNano()
prefixCoded := numeric.MustNewPrefixCodedInt64(dtInt64, 0)
Expand Down
24 changes: 4 additions & 20 deletions document/field_geopoint.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ var GeoPrecisionStep uint = 9
type GeoPointField struct {
name string
arrayPositions []uint64
options IndexingOptions
options index.FieldIndexingOptions
value numeric.PrefixCoded
numPlainTextBytes uint64
length int
Expand All @@ -58,30 +58,14 @@ func (n *GeoPointField) ArrayPositions() []uint64 {
return n.arrayPositions
}

func (n *GeoPointField) Options() IndexingOptions {
func (n *GeoPointField) Options() index.FieldIndexingOptions {
return n.options
}

func (n *GeoPointField) EncodedFieldType() byte {
return 'g'
}

func (n *GeoPointField) IsIndexed() bool {
return n.options.IsIndexed()
}

func (n *GeoPointField) IsStored() bool {
return n.options.IsStored()
}

func (n *GeoPointField) IncludeDocValues() bool {
return n.options.IncludeDocValues()
}

func (n *GeoPointField) IncludeTermVectors() bool {
return n.options.IncludeTermVectors()
}

func (n *GeoPointField) AnalyzedLength() int {
return n.length
}
Expand Down Expand Up @@ -122,7 +106,7 @@ func (n *GeoPointField) Analyze() {
}

n.length = len(tokens)
n.frequencies = analysis.TokenFrequency(tokens, n.arrayPositions, n.options.IncludeTermVectors())
n.frequencies = analysis.TokenFrequency(tokens, n.arrayPositions, n.options)
}

func (n *GeoPointField) Value() []byte {
Expand Down Expand Up @@ -167,7 +151,7 @@ func NewGeoPointField(name string, arrayPositions []uint64, lon, lat float64) *G
return NewGeoPointFieldWithIndexingOptions(name, arrayPositions, lon, lat, DefaultNumericIndexingOptions)
}

func NewGeoPointFieldWithIndexingOptions(name string, arrayPositions []uint64, lon, lat float64, options IndexingOptions) *GeoPointField {
func NewGeoPointFieldWithIndexingOptions(name string, arrayPositions []uint64, lon, lat float64, options index.FieldIndexingOptions) *GeoPointField {
mhash := geo.MortonHash(lon, lat)
prefixCoded := numeric.MustNewPrefixCodedInt64(int64(mhash), 0)
return &GeoPointField{
Expand Down
26 changes: 5 additions & 21 deletions document/field_numeric.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,14 +31,14 @@ func init() {
reflectStaticSizeNumericField = int(reflect.TypeOf(f).Size())
}

const DefaultNumericIndexingOptions = StoreField | IndexField | DocValues
const DefaultNumericIndexingOptions = index.StoreField | index.IndexField | index.DocValues

const DefaultPrecisionStep uint = 4

type NumericField struct {
name string
arrayPositions []uint64
options IndexingOptions
options index.FieldIndexingOptions
value numeric.PrefixCoded
numPlainTextBytes uint64
length int
Expand All @@ -59,30 +59,14 @@ func (n *NumericField) ArrayPositions() []uint64 {
return n.arrayPositions
}

func (n *NumericField) Options() IndexingOptions {
func (n *NumericField) Options() index.FieldIndexingOptions {
return n.options
}

func (n *NumericField) EncodedFieldType() byte {
return 'n'
}

func (n *NumericField) IsIndexed() bool {
return n.options.IsIndexed()
}

func (n *NumericField) IsStored() bool {
return n.options.IsStored()
}

func (n *NumericField) IncludeDocValues() bool {
return n.options.IncludeDocValues()
}

func (n *NumericField) IncludeTermVectors() bool {
return n.options.IncludeTermVectors()
}

func (n *NumericField) AnalyzedLength() int {
return n.length
}
Expand Down Expand Up @@ -123,7 +107,7 @@ func (n *NumericField) Analyze() {
}

n.length = len(tokens)
n.frequencies = analysis.TokenFrequency(tokens, n.arrayPositions, n.options.IncludeTermVectors())
n.frequencies = analysis.TokenFrequency(tokens, n.arrayPositions, n.options)
}

func (n *NumericField) Value() []byte {
Expand Down Expand Up @@ -160,7 +144,7 @@ func NewNumericField(name string, arrayPositions []uint64, number float64) *Nume
return NewNumericFieldWithIndexingOptions(name, arrayPositions, number, DefaultNumericIndexingOptions)
}

func NewNumericFieldWithIndexingOptions(name string, arrayPositions []uint64, number float64, options IndexingOptions) *NumericField {
func NewNumericFieldWithIndexingOptions(name string, arrayPositions []uint64, number float64, options index.FieldIndexingOptions) *NumericField {
numberInt64 := numeric.Float64ToInt64(number)
prefixCoded := numeric.MustNewPrefixCodedInt64(numberInt64, 0)
return &NumericField{
Expand Down
Loading