From 862b5cfcbafae3ab3ca53c40ec2f02144bcf153d Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Thu, 7 Apr 2022 12:17:41 +0200 Subject: [PATCH] s2: Use sorted search for index ``` cpu: AMD Ryzen 9 3950X 16-Core Processor BenchmarkIndexFind BenchmarkIndexFind/blocks-1-32 194769370 6.180 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-2-32 199258790 6.057 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-4-32 204297220 5.879 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-8-32 190476280 6.321 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-16-32 158611744 7.537 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-32-32 100000000 10.04 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-64-32 79986668 14.90 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-128-32 74996250 14.95 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-256-32 19506928 60.93 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-512-32 8450894 142.1 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-1024-32 3858536 307.3 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-2048-32 2556153 470.4 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-4096-32 840476 1457 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-8192-32 571398 2091 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-16384-32 195525 6022 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-32768-32 138596 8665 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-65535-32 86328 13993 ns/op 0 B/op 0 allocs/op ``` after: ``` cpu: AMD Ryzen 9 3950X 16-Core Processor BenchmarkIndexFind BenchmarkIndexFind/blocks-1-32 195354979 6.054 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-2-32 193981434 6.140 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-4-32 201894782 6.023 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-8-32 180996812 6.978 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-16-32 148147708 8.016 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-32-32 100000000 10.43 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-64-32 75000000 15.95 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-128-32 75000000 15.59 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-256-32 49979174 22.83 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-512-32 46302012 25.00 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-1024-32 44444937 27.43 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-2048-32 38709552 29.62 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-4096-32 37005741 31.86 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-8192-32 34297766 34.43 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-16384-32 30763077 36.89 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-32768-32 29656209 39.45 ns/op 0 B/op 0 allocs/op BenchmarkIndexFind/blocks-65535-32 27980180 42.53 ns/op 0 B/op 0 allocs/op ``` Fixes #554 --- s2/encode_test.go | 37 +++++++++++++++++++++++++++++++++++++ s2/index.go | 10 ++++++++++ 2 files changed, 47 insertions(+) diff --git a/s2/encode_test.go b/s2/encode_test.go index fcf04191fa..a1ad5ac591 100644 --- a/s2/encode_test.go +++ b/s2/encode_test.go @@ -351,6 +351,43 @@ func TestIndex(t *testing.T) { } } +func BenchmarkIndexFind(b *testing.B) { + fatalErr := func(t testing.TB, err error) { + if err != nil { + t.Fatal(err) + } + } + for blocks := 1; blocks <= 65536; blocks *= 2 { + if blocks == 65536 { + blocks = 65535 + } + + var index Index + index.reset(100) + index.TotalUncompressed = int64(blocks) * 100 + index.TotalCompressed = int64(blocks) * 100 + for i := 0; i < blocks; i++ { + err := index.add(int64(i*100), int64(i*100)) + fatalErr(b, err) + } + + rng := rand.New(rand.NewSource(0xabeefcafe)) + b.Run(fmt.Sprintf("blocks-%d", len(index.info)), func(b *testing.B) { + b.ResetTimer() + b.ReportAllocs() + const prime4bytes = 2654435761 + rng2 := rng.Int63() + for i := 0; i < b.N; i++ { + rng2 = ((rng2 + prime4bytes) * prime4bytes) >> 32 + // Find offset: + _, _, err := index.Find(rng2 % (int64(blocks) * 100)) + fatalErr(b, err) + } + }) + } + +} + func TestWriterPadding(t *testing.T) { n := 100 if testing.Short() { diff --git a/s2/index.go b/s2/index.go index fd857682e4..7b24a0060b 100644 --- a/s2/index.go +++ b/s2/index.go @@ -10,6 +10,7 @@ import ( "encoding/json" "fmt" "io" + "sort" ) const ( @@ -100,6 +101,15 @@ func (i *Index) Find(offset int64) (compressedOff, uncompressedOff int64, err er if offset > i.TotalUncompressed { return 0, 0, io.ErrUnexpectedEOF } + if len(i.info) > 200 { + n := sort.Search(len(i.info), func(n int) bool { + return i.info[n].uncompressedOffset > offset + }) + if n == 0 { + n = 1 + } + return i.info[n-1].compressedOffset, i.info[n-1].uncompressedOffset, nil + } for _, info := range i.info { if info.uncompressedOffset > offset { break