IPRangeQuery to search for IPs which are members of a subnet. (#1546)

* Support for IPRangeQuery

* make sure we return an error if argument to IPRangeSearch is not a properly formatter CIDR or IP

* Add missing copyright stubs to new files

* Fix test which broke in Go 1.17
Because of golang/go@d3e3d03
which rejects leading zeros in dot-decimal notation.

* rename all instances of `Ip` to `IP` to conform with stdlib

analysis/type.go

@@ -34,6 +34,7 @@ const (
 	Single
 	Double
 	Boolean
+	IP
 )
 
 // Token represents one occurrence of a term at a particular location in a

document/field_ip.go

@@ -0,0 +1,132 @@
+//  Copyright (c) 2021 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package document
+
+import (
+	"fmt"
+	"net"
+	"reflect"
+
+	"github.com/blevesearch/bleve/v2/analysis"
+	"github.com/blevesearch/bleve/v2/size"
+	index "github.com/blevesearch/bleve_index_api"
+)
+
+var reflectStaticSizeIPField int
+
+func init() {
+	var f IPField
+	reflectStaticSizeIPField = int(reflect.TypeOf(f).Size())
+}
+
+const DefaultIPIndexingOptions = index.StoreField | index.IndexField | index.DocValues | index.IncludeTermVectors
+
+type IPField struct {
+	name              string
+	arrayPositions    []uint64
+	options           index.FieldIndexingOptions
+	value             net.IP
+	numPlainTextBytes uint64
+	length            int
+	frequencies       index.TokenFrequencies
+}
+
+func (b *IPField) Size() int {
+	return reflectStaticSizeIPField + size.SizeOfPtr +
+		len(b.name) +
+		len(b.arrayPositions)*size.SizeOfUint64 +
+		len(b.value)
+}
+
+func (b *IPField) Name() string {
+	return b.name
+}
+
+func (b *IPField) ArrayPositions() []uint64 {
+	return b.arrayPositions
+}
+
+func (b *IPField) Options() index.FieldIndexingOptions {
+	return b.options
+}
+
+func (n *IPField) EncodedFieldType() byte {
+	return 'i'
+}
+
+func (n *IPField) AnalyzedLength() int {
+	return n.length
+}
+
+func (n *IPField) AnalyzedTokenFrequencies() index.TokenFrequencies {
+	return n.frequencies
+}
+
+func (b *IPField) Analyze() {
+
+	tokens := analysis.TokenStream{
+		&analysis.Token{
+			Start:    0,
+			End:      len(b.value),
+			Term:     b.value,
+			Position: 1,
+			Type:     analysis.IP,
+		},
+	}
+	b.length = 1
+	b.frequencies = analysis.TokenFrequency(tokens, b.arrayPositions, b.options)
+}
+
+func (b *IPField) Value() []byte {
+	return b.value
+}
+
+func (b *IPField) IP() (net.IP, error) {
+	return net.IP(b.value), nil
+}
+
+func (b *IPField) GoString() string {
+	return fmt.Sprintf("&document.IPField{Name:%s, Options: %s, Value: %s}", b.name, b.options, net.IP(b.value))
+}
+
+func (b *IPField) NumPlainTextBytes() uint64 {
+	return b.numPlainTextBytes
+}
+
+func NewIPFieldFromBytes(name string, arrayPositions []uint64, value []byte) *IPField {
+	return &IPField{
+		name:              name,
+		arrayPositions:    arrayPositions,
+		value:             value,
+		options:           DefaultNumericIndexingOptions,
+		numPlainTextBytes: uint64(len(value)),
+	}
+}
+
+func NewIPField(name string, arrayPositions []uint64, v net.IP) *IPField {
+	return NewIPFieldWithIndexingOptions(name, arrayPositions, v, DefaultIPIndexingOptions)
+}
+
+func NewIPFieldWithIndexingOptions(name string, arrayPositions []uint64, b net.IP, options index.FieldIndexingOptions) *IPField {
+	v := b.To16()
+
+	return &IPField{
+		name:              name,
+		arrayPositions:    arrayPositions,
+		value:             v,
+		options:           options,
+		numPlainTextBytes: net.IPv6len,
+	}
+}

document/field_ip_test.go

@@ -0,0 +1,38 @@
+//  Copyright (c) 2021 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package document
+
+import (
+	"bytes"
+	"net"
+	"testing"
+)
+
+func TestIPField(t *testing.T) {
+	nf := NewIPField("ip", []uint64{}, net.IPv4(192, 168, 1, 1))
+	nf.Analyze()
+	if nf.length != 1 {
+		t.Errorf("expected 1 token")
+	}
+	if len(nf.value) != 16 {
+		t.Errorf("stored value should be in 16 byte ipv6 format")
+	}
+	if !bytes.Equal(nf.value, []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 192, 168, 1, 1}) {
+		t.Errorf("wrong value stored, expected 192.168.1.1, got %q", nf.value.String())
+	}
+	if len(nf.frequencies) != 1 {
+		t.Errorf("expected 1 token freqs")
+	}
+}

index/scorch/snapshot_index.go

@@ -429,6 +429,8 @@ func (i *IndexSnapshot) Document(id string) (rv index.Document, err error) {
 			rvd.AddField(document.NewTextField(name, arrayPos, value))
 		case 'n':
 			rvd.AddField(document.NewNumericFieldFromBytes(name, arrayPos, value))
+		case 'i':
+			rvd.AddField(document.NewIPFieldFromBytes(name, arrayPos, value))
 		case 'd':
 			rvd.AddField(document.NewDateTimeFieldFromBytes(name, arrayPos, value))
 		case 'b':

index/upsidedown/upsidedown.go

@@ -727,6 +727,8 @@ func decodeFieldType(typ byte, name string, pos []uint64, value []byte) document
 		return document.NewBooleanFieldFromBytes(name, pos, value)
 	case 'g':
 		return document.NewGeoPointFieldFromBytes(name, pos, value)
+	case 'i':
+		return document.NewIPFieldFromBytes(name, pos, value)
 	}
 	return nil
 }

mapping.go

@@ -69,3 +69,7 @@ func NewBooleanFieldMapping() *mapping.FieldMapping {
 func NewGeoPointFieldMapping() *mapping.FieldMapping {
 	return mapping.NewGeoPointFieldMapping()
 }
+
+func NewIPFieldMapping() *mapping.FieldMapping {
+	return mapping.NewIPFieldMapping()
+}

mapping/document.go

@@ -18,6 +18,7 @@ import (
 	"encoding"
 	"encoding/json"
 	"fmt"
+	"net"
 	"reflect"
 	"time"
 
@@ -76,7 +77,7 @@ func (dm *DocumentMapping) Validate(cache *registry.Cache) error {
 			}
 		}
 		switch field.Type {
-		case "text", "datetime", "number", "boolean", "geopoint":
+		case "text", "datetime", "number", "boolean", "geopoint", "IP":
 		default:
 			return fmt.Errorf("unknown field type: '%s'", field.Type)
 		}
@@ -517,8 +518,14 @@ func (dm *DocumentMapping) processProperty(property interface{}, path []string,
 	case reflect.Map, reflect.Slice:
 		if subDocMapping != nil {
 			for _, fieldMapping := range subDocMapping.Fields {
-				if fieldMapping.Type == "geopoint" {
+				switch fieldMapping.Type {
+				case "geopoint":
 					fieldMapping.processGeoPoint(property, pathString, path, indexes, context)
+				case "IP":
+					ip, ok := property.(net.IP)
+					if ok {
+						fieldMapping.processIP(ip, pathString, path, indexes, context)
+					}
 				}
 			}
 		}

mapping/field.go

@@ -17,6 +17,7 @@ package mapping
 import (
 	"encoding/json"
 	"fmt"
+	"net"
 	"time"
 
 	"github.com/blevesearch/bleve/v2/analysis/analyzer/keyword"
@@ -171,6 +172,16 @@ func NewGeoPointFieldMapping() *FieldMapping {
 	}
 }
 
+// NewIPFieldMapping returns a default field mapping for IP points
+func NewIPFieldMapping() *FieldMapping {
+	return &FieldMapping{
+		Type:         "IP",
+		Store:        true,
+		Index:        true,
+		IncludeInAll: true,
+	}
+}
+
 // Options returns the indexing options for this field.
 func (fm *FieldMapping) Options() index.FieldIndexingOptions {
 	var rv index.FieldIndexingOptions
@@ -215,6 +226,11 @@ func (fm *FieldMapping) processString(propertyValueString string, pathString str
 				fm.processTime(parsedDateTime, pathString, path, indexes, context)
 			}
 		}
+	} else if fm.Type == "IP" {
+		ip := net.ParseIP(propertyValueString)
+		if ip != nil {
+			fm.processIP(ip, pathString, path, indexes, context)
+		}
 	}
 }
 
@@ -275,6 +291,17 @@ func (fm *FieldMapping) processGeoPoint(propertyMightBeGeoPoint interface{}, pat
 	}
 }
 
+func (fm *FieldMapping) processIP(ip net.IP, pathString string, path []string, indexes []uint64, context *walkContext) {
+	fieldName := getFieldName(pathString, path, fm)
+	options := fm.Options()
+	field := document.NewIPFieldWithIndexingOptions(fieldName, indexes, ip, options)
+	context.doc.AddField(field)
+
+	if !fm.IncludeInAll {
+		context.excludedFromAll = append(context.excludedFromAll, fieldName)
+	}
+}
+
 func (fm *FieldMapping) analyzerForField(path []string, context *walkContext) *analysis.Analyzer {
 	analyzerName := fm.Analyzer
 	if analyzerName == "" {

query.go

@@ -216,3 +216,12 @@ func NewGeoBoundingBoxQuery(topLeftLon, topLeftLat, bottomRightLon, bottomRightL
 func NewGeoDistanceQuery(lon, lat float64, distance string) *query.GeoDistanceQuery {
 	return query.NewGeoDistanceQuery(lon, lat, distance)
 }
+
+// NewIPRangeQuery creates a new Query for matching IP addresses.
+// If the argument is in CIDR format, then the query will match all
+// IP addresses in the network specified. If the argument is an IP address,
+// then the query will return documents which contain that IP.
+// Both ipv4 and ipv6 are supported.
+func NewIPRangeQuery(cidr string) *query.IPRangeQuery {
+	return query.NewIPRangeQuery(cidr)
+}

search/query/ip_range.go

@@ -0,0 +1,84 @@
+//  Copyright (c) 2021 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package query
+
+import (
+	"fmt"
+	"net"
+
+	"github.com/blevesearch/bleve/v2/mapping"
+	"github.com/blevesearch/bleve/v2/search"
+	"github.com/blevesearch/bleve/v2/search/searcher"
+	index "github.com/blevesearch/bleve_index_api"
+)
+
+type IPRangeQuery struct {
+	CIDR     string `json:"cidr, omitempty"`
+	FieldVal string `json:"field,omitempty"`
+	BoostVal *Boost `json:"boost,omitempty"`
+}
+
+func NewIPRangeQuery(cidr string) *IPRangeQuery {
+	return &IPRangeQuery{
+		CIDR: cidr,
+	}
+}
+
+func (q *IPRangeQuery) SetBoost(b float64) {
+	boost := Boost(b)
+	q.BoostVal = &boost
+}
+
+func (q *IPRangeQuery) Boost() float64 {
+	return q.BoostVal.Value()
+}
+
+func (q *IPRangeQuery) SetField(f string) {
+	q.FieldVal = f
+}
+
+func (q *IPRangeQuery) Field() string {
+	return q.FieldVal
+}
+
+func (q *IPRangeQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {
+	field := q.FieldVal
+	if q.FieldVal == "" {
+		field = m.DefaultSearchField()
+	}
+	_, ipNet, err := net.ParseCIDR(q.CIDR)
+	if err != nil {
+		ip := net.ParseIP(q.CIDR)
+		if ip == nil {
+			return nil, err
+		}
+		// If we are searching for a specific ip rather than members of a network, just use a term search.
+		return searcher.NewTermSearcherBytes(i, ip.To16(), field, q.BoostVal.Value(), options)
+	}
+	return searcher.NewIPRangeSearcher(i, ipNet, field, q.BoostVal.Value(), options)
+}
+
+func (q *IPRangeQuery) Validate() error {
+	_, _, err := net.ParseCIDR(q.CIDR)
+	if err == nil {
+		return nil
+	}
+	// We also allow search for a specific IP.
+	ip := net.ParseIP(q.CIDR)
+	if ip != nil {
+		return nil // we have a valid ip
+	}
+	return fmt.Errorf("IPRangeQuery must be for an network or ip address, %q", q.CIDR)
+}