Skip to content

Commit

Permalink
s2: Use sorted search for index (#555)
Browse files Browse the repository at this point in the history
```
cpu: AMD Ryzen 9 3950X 16-Core Processor
BenchmarkIndexFind
BenchmarkIndexFind/blocks-1-32         	194769370	         6.180 ns/op	       0 B/op	       0 allocs/op
BenchmarkIndexFind/blocks-2-32         	199258790	         6.057 ns/op	       0 B/op	       0 allocs/op
BenchmarkIndexFind/blocks-4-32         	204297220	         5.879 ns/op	       0 B/op	       0 allocs/op
BenchmarkIndexFind/blocks-8-32         	190476280	         6.321 ns/op	       0 B/op	       0 allocs/op
BenchmarkIndexFind/blocks-16-32        	158611744	         7.537 ns/op	       0 B/op	       0 allocs/op
BenchmarkIndexFind/blocks-32-32        	100000000	        10.04 ns/op	       0 B/op	       0 allocs/op
BenchmarkIndexFind/blocks-64-32        	79986668	        14.90 ns/op	       0 B/op	       0 allocs/op
BenchmarkIndexFind/blocks-128-32       	74996250	        14.95 ns/op	       0 B/op	       0 allocs/op
BenchmarkIndexFind/blocks-256-32       	19506928	        60.93 ns/op	       0 B/op	       0 allocs/op
BenchmarkIndexFind/blocks-512-32       	 8450894	       142.1 ns/op	       0 B/op	       0 allocs/op
BenchmarkIndexFind/blocks-1024-32      	 3858536	       307.3 ns/op	       0 B/op	       0 allocs/op
BenchmarkIndexFind/blocks-2048-32      	 2556153	       470.4 ns/op	       0 B/op	       0 allocs/op
BenchmarkIndexFind/blocks-4096-32      	  840476	      1457 ns/op	       0 B/op	       0 allocs/op
BenchmarkIndexFind/blocks-8192-32      	  571398	      2091 ns/op	       0 B/op	       0 allocs/op
BenchmarkIndexFind/blocks-16384-32     	  195525	      6022 ns/op	       0 B/op	       0 allocs/op
BenchmarkIndexFind/blocks-32768-32     	  138596	      8665 ns/op	       0 B/op	       0 allocs/op
BenchmarkIndexFind/blocks-65535-32     	   86328	     13993 ns/op	       0 B/op	       0 allocs/op
```

after:

```
cpu: AMD Ryzen 9 3950X 16-Core Processor
BenchmarkIndexFind
BenchmarkIndexFind/blocks-1-32         	195354979	         6.054 ns/op	       0 B/op	       0 allocs/op
BenchmarkIndexFind/blocks-2-32         	193981434	         6.140 ns/op	       0 B/op	       0 allocs/op
BenchmarkIndexFind/blocks-4-32         	201894782	         6.023 ns/op	       0 B/op	       0 allocs/op
BenchmarkIndexFind/blocks-8-32         	180996812	         6.978 ns/op	       0 B/op	       0 allocs/op
BenchmarkIndexFind/blocks-16-32        	148147708	         8.016 ns/op	       0 B/op	       0 allocs/op
BenchmarkIndexFind/blocks-32-32        	100000000	        10.43 ns/op	       0 B/op	       0 allocs/op
BenchmarkIndexFind/blocks-64-32        	75000000	        15.95 ns/op	       0 B/op	       0 allocs/op
BenchmarkIndexFind/blocks-128-32       	75000000	        15.59 ns/op	       0 B/op	       0 allocs/op
BenchmarkIndexFind/blocks-256-32       	49979174	        22.83 ns/op	       0 B/op	       0 allocs/op
BenchmarkIndexFind/blocks-512-32       	46302012	        25.00 ns/op	       0 B/op	       0 allocs/op
BenchmarkIndexFind/blocks-1024-32      	44444937	        27.43 ns/op	       0 B/op	       0 allocs/op
BenchmarkIndexFind/blocks-2048-32      	38709552	        29.62 ns/op	       0 B/op	       0 allocs/op
BenchmarkIndexFind/blocks-4096-32      	37005741	        31.86 ns/op	       0 B/op	       0 allocs/op
BenchmarkIndexFind/blocks-8192-32      	34297766	        34.43 ns/op	       0 B/op	       0 allocs/op
BenchmarkIndexFind/blocks-16384-32     	30763077	        36.89 ns/op	       0 B/op	       0 allocs/op
BenchmarkIndexFind/blocks-32768-32     	29656209	        39.45 ns/op	       0 B/op	       0 allocs/op
BenchmarkIndexFind/blocks-65535-32     	27980180	        42.53 ns/op	       0 B/op	       0 allocs/op
```

Fixes #554
  • Loading branch information
klauspost authored Apr 7, 2022
1 parent 458f435 commit c06ba5f
Show file tree
Hide file tree
Showing 2 changed files with 47 additions and 0 deletions.
37 changes: 37 additions & 0 deletions s2/encode_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -351,6 +351,43 @@ func TestIndex(t *testing.T) {
}
}

func BenchmarkIndexFind(b *testing.B) {
fatalErr := func(t testing.TB, err error) {
if err != nil {
t.Fatal(err)
}
}
for blocks := 1; blocks <= 65536; blocks *= 2 {
if blocks == 65536 {
blocks = 65535
}

var index Index
index.reset(100)
index.TotalUncompressed = int64(blocks) * 100
index.TotalCompressed = int64(blocks) * 100
for i := 0; i < blocks; i++ {
err := index.add(int64(i*100), int64(i*100))
fatalErr(b, err)
}

rng := rand.New(rand.NewSource(0xabeefcafe))
b.Run(fmt.Sprintf("blocks-%d", len(index.info)), func(b *testing.B) {
b.ResetTimer()
b.ReportAllocs()
const prime4bytes = 2654435761
rng2 := rng.Int63()
for i := 0; i < b.N; i++ {
rng2 = ((rng2 + prime4bytes) * prime4bytes) >> 32
// Find offset:
_, _, err := index.Find(rng2 % (int64(blocks) * 100))
fatalErr(b, err)
}
})
}

}

func TestWriterPadding(t *testing.T) {
n := 100
if testing.Short() {
Expand Down
10 changes: 10 additions & 0 deletions s2/index.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"encoding/json"
"fmt"
"io"
"sort"
)

const (
Expand Down Expand Up @@ -100,6 +101,15 @@ func (i *Index) Find(offset int64) (compressedOff, uncompressedOff int64, err er
if offset > i.TotalUncompressed {
return 0, 0, io.ErrUnexpectedEOF
}
if len(i.info) > 200 {
n := sort.Search(len(i.info), func(n int) bool {
return i.info[n].uncompressedOffset > offset
})
if n == 0 {
n = 1
}
return i.info[n-1].compressedOffset, i.info[n-1].uncompressedOffset, nil
}
for _, info := range i.info {
if info.uncompressedOffset > offset {
break
Expand Down

0 comments on commit c06ba5f

Please sign in to comment.