move in more things from the segment package

index/scorch/empty.go

@@ -0,0 +1,33 @@
+//  Copyright (c) 2020 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package scorch
+
+import segment "github.com/blevesearch/scorch_segment_api"
+
+type emptyPostingsIterator struct{}
+
+func (e *emptyPostingsIterator) Next() (segment.Posting, error) {
+	return nil, nil
+}
+
+func (e *emptyPostingsIterator) Advance(uint64) (segment.Posting, error) {
+	return nil, nil
+}
+
+func (e *emptyPostingsIterator) Size() int {
+	return 0
+}
+
+var anEmptyPostingsIterator = &emptyPostingsIterator{}

index/scorch/int.go

@@ -0,0 +1,87 @@
+//  Copyright (c) 2020 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package scorch
+
+import "fmt"
+
+const (
+	// intMin is chosen such that the range of int tags does not overlap the
+	// ascii character set that is frequently used in testing.
+	intMin      = 0x80 // 128
+	intMaxWidth = 8
+	intZero     = intMin + intMaxWidth           // 136
+	intSmall    = intMax - intZero - intMaxWidth // 109
+	// intMax is the maximum int tag value.
+	intMax = 0xfd // 253
+)
+
+// encodeUvarintAscending encodes the uint64 value using a variable length
+// (length-prefixed) representation. The length is encoded as a single
+// byte indicating the number of encoded bytes (-8) to follow. See
+// EncodeVarintAscending for rationale. The encoded bytes are appended to the
+// supplied buffer and the final buffer is returned.
+func encodeUvarintAscending(b []byte, v uint64) []byte {
+	switch {
+	case v <= intSmall:
+		return append(b, intZero+byte(v))
+	case v <= 0xff:
+		return append(b, intMax-7, byte(v))
+	case v <= 0xffff:
+		return append(b, intMax-6, byte(v>>8), byte(v))
+	case v <= 0xffffff:
+		return append(b, intMax-5, byte(v>>16), byte(v>>8), byte(v))
+	case v <= 0xffffffff:
+		return append(b, intMax-4, byte(v>>24), byte(v>>16), byte(v>>8), byte(v))
+	case v <= 0xffffffffff:
+		return append(b, intMax-3, byte(v>>32), byte(v>>24), byte(v>>16), byte(v>>8),
+			byte(v))
+	case v <= 0xffffffffffff:
+		return append(b, intMax-2, byte(v>>40), byte(v>>32), byte(v>>24), byte(v>>16),
+			byte(v>>8), byte(v))
+	case v <= 0xffffffffffffff:
+		return append(b, intMax-1, byte(v>>48), byte(v>>40), byte(v>>32), byte(v>>24),
+			byte(v>>16), byte(v>>8), byte(v))
+	default:
+		return append(b, intMax, byte(v>>56), byte(v>>48), byte(v>>40), byte(v>>32),
+			byte(v>>24), byte(v>>16), byte(v>>8), byte(v))
+	}
+}
+
+// decodeUvarintAscending decodes a varint encoded uint64 from the input
+// buffer. The remainder of the input buffer and the decoded uint64
+// are returned.
+func decodeUvarintAscending(b []byte) ([]byte, uint64, error) {
+	if len(b) == 0 {
+		return nil, 0, fmt.Errorf("insufficient bytes to decode uvarint value")
+	}
+	length := int(b[0]) - intZero
+	b = b[1:] // skip length byte
+	if length <= intSmall {
+		return b, uint64(length), nil
+	}
+	length -= intSmall
+	if length < 0 || length > 8 {
+		return nil, 0, fmt.Errorf("invalid uvarint length of %d", length)
+	} else if len(b) < length {
+		return nil, 0, fmt.Errorf("insufficient bytes to decode uvarint value: %q", b)
+	}
+	var v uint64
+	// It is faster to range over the elements in a slice than to index
+	// into the slice on each loop iteration.
+	for _, t := range b[:length] {
+		v = (v << 8) | uint64(t)
+	}
+	return b[length:], v, nil
+}

index/scorch/int_test.go

@@ -0,0 +1,91 @@
+//  Copyright (c) 2020 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package scorch
+
+import (
+	"bytes"
+	"math"
+	"testing"
+)
+
+type testCaseUint64 struct {
+	value  uint64
+	expEnc []byte
+}
+
+func TestEncodeDecodeUvarint(t *testing.T) {
+	testBasicEncodeDecodeUint64(encodeUvarintAscending, decodeUvarintAscending, false, t)
+	testCases := []testCaseUint64{
+		{0, []byte{0x88}},
+		{1, []byte{0x89}},
+		{109, []byte{0xf5}},
+		{110, []byte{0xf6, 0x6e}},
+		{1 << 8, []byte{0xf7, 0x01, 0x00}},
+		{math.MaxUint64, []byte{0xfd, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}},
+	}
+	testCustomEncodeUint64(testCases, encodeUvarintAscending, t)
+}
+
+func testBasicEncodeDecodeUint64(
+	encFunc func([]byte, uint64) []byte,
+	decFunc func([]byte) ([]byte, uint64, error),
+	descending bool, t *testing.T,
+) {
+	testCases := []uint64{
+		0, 1,
+		1<<8 - 1, 1 << 8,
+		1<<16 - 1, 1 << 16,
+		1<<24 - 1, 1 << 24,
+		1<<32 - 1, 1 << 32,
+		1<<40 - 1, 1 << 40,
+		1<<48 - 1, 1 << 48,
+		1<<56 - 1, 1 << 56,
+		math.MaxUint64 - 1, math.MaxUint64,
+	}
+
+	var lastEnc []byte
+	for i, v := range testCases {
+		enc := encFunc(nil, v)
+		if i > 0 {
+			if (descending && bytes.Compare(enc, lastEnc) >= 0) ||
+				(!descending && bytes.Compare(enc, lastEnc) < 0) {
+				t.Errorf("ordered constraint violated for %d: [% x] vs. [% x]", v, enc, lastEnc)
+			}
+		}
+		b, decode, err := decFunc(enc)
+		if err != nil {
+			t.Error(err)
+			continue
+		}
+		if len(b) != 0 {
+			t.Errorf("leftover bytes: [% x]", b)
+		}
+		if decode != v {
+			t.Errorf("decode yielded different value than input: %d vs. %d", decode, v)
+		}
+		lastEnc = enc
+	}
+}
+
+func testCustomEncodeUint64(
+	testCases []testCaseUint64, encFunc func([]byte, uint64) []byte, t *testing.T,
+) {
+	for _, test := range testCases {
+		enc := encFunc(nil, test.value)
+		if !bytes.Equal(enc, test.expEnc) {
+			t.Errorf("expected [% x]; got [% x] (value: %d)", test.expEnc, enc, test.value)
+		}
+	}
+}

index/scorch/optimize.go

@@ -176,7 +176,7 @@ OUTER:
 		for _, tfr := range o.tfrs {
 			if _, ok := tfr.iterators[i].(*segment.EmptyPostingsIterator); ok {
 				// An empty postings iterator means the entire AND is empty.
-				oTFR.iterators[i] = segment.AnEmptyPostingsIterator
+				oTFR.iterators[i] = anEmptyPostingsIterator
 				continue OUTER
 			}
 
@@ -193,7 +193,7 @@ OUTER:
 				if docNum1HitLastOk && docNum1HitLast != docNum1Hit {
 					// The docNum1Hit doesn't match the previous
 					// docNum1HitLast, so the entire AND is empty.
-					oTFR.iterators[i] = segment.AnEmptyPostingsIterator
+					oTFR.iterators[i] = anEmptyPostingsIterator
 					continue OUTER
 				}
 
@@ -205,7 +205,7 @@ OUTER:
 
 			if itr.ActualBitmap() == nil {
 				// An empty actual bitmap means the entire AND is empty.
-				oTFR.iterators[i] = segment.AnEmptyPostingsIterator
+				oTFR.iterators[i] = anEmptyPostingsIterator
 				continue OUTER
 			}
 
@@ -221,7 +221,7 @@ OUTER:
 				if !bm.Contains(uint32(docNum1HitLast)) {
 					// The docNum1Hit isn't in one of our actual
 					// bitmaps, so the entire AND is empty.
-					oTFR.iterators[i] = segment.AnEmptyPostingsIterator
+					oTFR.iterators[i] = anEmptyPostingsIterator
 					continue OUTER
 				}
 			}
@@ -236,7 +236,7 @@ OUTER:
 		if len(actualBMs) == 0 {
 			// If we've collected no actual bitmaps at this point,
 			// then the entire AND is empty.
-			oTFR.iterators[i] = segment.AnEmptyPostingsIterator
+			oTFR.iterators[i] = anEmptyPostingsIterator
 			continue OUTER
 		}
 

index/scorch/persister.go

@@ -434,7 +434,7 @@ func prepareBoltSnapshot(snapshot *IndexSnapshot, tx *bolt.Tx, path string,
 	if err != nil {
 		return nil, nil, err
 	}
-	newSnapshotKey := segment.EncodeUvarintAscending(nil, snapshot.epoch)
+	newSnapshotKey := encodeUvarintAscending(nil, snapshot.epoch)
 	snapshotBucket, err := snapshotsBucket.CreateBucketIfNotExists(newSnapshotKey)
 	if err != nil {
 		return nil, nil, err
@@ -474,7 +474,7 @@ func prepareBoltSnapshot(snapshot *IndexSnapshot, tx *bolt.Tx, path string,
 
 	// first ensure that each segment in this snapshot has been persisted
 	for _, segmentSnapshot := range snapshot.segment {
-		snapshotSegmentKey := segment.EncodeUvarintAscending(nil, segmentSnapshot.id)
+		snapshotSegmentKey := encodeUvarintAscending(nil, segmentSnapshot.id)
 		snapshotSegmentBucket, err := snapshotBucket.CreateBucketIfNotExists(snapshotSegmentKey)
 		if err != nil {
 			return nil, nil, err
@@ -628,7 +628,7 @@ func (s *Scorch) loadFromBolt() error {
 		foundRoot := false
 		c := snapshots.Cursor()
 		for k, _ := c.Last(); k != nil; k, _ = c.Prev() {
-			_, snapshotEpoch, err := segment.DecodeUvarintAscending(k)
+			_, snapshotEpoch, err := decodeUvarintAscending(k)
 			if err != nil {
 				log.Printf("unable to parse segment epoch %x, continuing", k)
 				continue
@@ -680,7 +680,7 @@ func (s *Scorch) LoadSnapshot(epoch uint64) (rv *IndexSnapshot, err error) {
 		if snapshots == nil {
 			return nil
 		}
-		snapshotKey := segment.EncodeUvarintAscending(nil, epoch)
+		snapshotKey := encodeUvarintAscending(nil, epoch)
 		snapshot := snapshots.Bucket(snapshotKey)
 		if snapshot == nil {
 			return fmt.Errorf("snapshot with epoch: %v - doesn't exist", epoch)
@@ -744,7 +744,7 @@ func (s *Scorch) loadSnapshot(snapshot *bolt.Bucket) (*IndexSnapshot, error) {
 				_ = rv.DecRef()
 				return nil, fmt.Errorf("failed to load segment: %v", err)
 			}
-			_, segmentSnapshot.id, err = segment.DecodeUvarintAscending(k)
+			_, segmentSnapshot.id, err = decodeUvarintAscending(k)
 			if err != nil {
 				_ = rv.DecRef()
 				return nil, fmt.Errorf("failed to decode segment id: %v", err)
@@ -865,7 +865,7 @@ func (s *Scorch) removeOldBoltSnapshots() (numRemoved int, err error) {
 	}
 
 	for _, epochToRemove := range epochsToRemove {
-		k := segment.EncodeUvarintAscending(nil, epochToRemove)
+		k := encodeUvarintAscending(nil, epochToRemove)
 		err = snapshots.DeleteBucket(k)
 		if err == bolt.ErrBucketNotFound {
 			err = nil
@@ -941,7 +941,7 @@ func (s *Scorch) RootBoltSnapshotEpochs() ([]uint64, error) {
 		}
 		sc := snapshots.Cursor()
 		for sk, _ := sc.Last(); sk != nil; sk, _ = sc.Prev() {
-			_, snapshotEpoch, err := segment.DecodeUvarintAscending(sk)
+			_, snapshotEpoch, err := decodeUvarintAscending(sk)
 			if err != nil {
 				continue
 			}

index/scorch/rollback.go

@@ -19,7 +19,6 @@ import (
 	"log"
 	"os"
 
-	segment "github.com/blevesearch/scorch_segment_api"
 	bolt "go.etcd.io/bbolt"
 )
 
@@ -71,7 +70,7 @@ func RollbackPoints(path string) ([]*RollbackPoint, error) {
 
 	c1 := snapshots.Cursor()
 	for k, _ := c1.Last(); k != nil; k, _ = c1.Prev() {
-		_, snapshotEpoch, err := segment.DecodeUvarintAscending(k)
+		_, snapshotEpoch, err := decodeUvarintAscending(k)
 		if err != nil {
 			log.Printf("RollbackPoints:"+
 				" unable to parse segment epoch %x, continuing", k)
@@ -154,7 +153,7 @@ func Rollback(path string, to *RollbackPoint) error {
 		}
 		sc := snapshots.Cursor()
 		for sk, _ := sc.Last(); sk != nil && !found; sk, _ = sc.Prev() {
-			_, snapshotEpoch, err := segment.DecodeUvarintAscending(sk)
+			_, snapshotEpoch, err := decodeUvarintAscending(sk)
 			if err != nil {
 				continue
 			}
@@ -195,7 +194,7 @@ func Rollback(path string, to *RollbackPoint) error {
 		return nil
 	}
 	for _, epoch := range eligibleEpochs {
-		k := segment.EncodeUvarintAscending(nil, epoch)
+		k := encodeUvarintAscending(nil, epoch)
 		if err != nil {
 			continue
 		}