Skip to content
This repository has been archived by the owner on Aug 19, 2022. It is now read-only.

Deterministic benchmark order; Keybook interface benchmarks #43

Merged
merged 3 commits into from
Nov 14, 2018
Merged

Deterministic benchmark order; Keybook interface benchmarks #43

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ee43bc1
prevent timeout by un-nesting tb.Runs
lanzafame Oct 29, 2018
308d998
sort suite so benchmarks always run in the same order
lanzafame Oct 29, 2018
c04ff2d
add keybook benchmarks
lanzafame Oct 29, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 23 additions & 15 deletions pstoreds/ds_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,26 +33,24 @@ func TestDsPeerstore(t *testing.T) {

func TestDsAddrBook(t *testing.T) {
for name, dsFactory := range dstores {
t.Run(name, func(t *testing.T) {
t.Run("Cacheful", func(t *testing.T) {
t.Parallel()
t.Run(name+" Cacheful", func(t *testing.T) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we keep the hierarchical [datastore]/{Cacheful,Cacheless}/[testcase] format in the full test name, or is the slash a reserved char?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry, didn't see this review. I will be adding some more benchmarks soon, so I will add that in then.

t.Parallel()

opts := DefaultOpts()
opts.TTLInterval = 100 * time.Microsecond
opts.CacheSize = 1024
opts := DefaultOpts()
opts.TTLInterval = 100 * time.Microsecond
opts.CacheSize = 1024

pt.TestAddrBook(t, addressBookFactory(t, dsFactory, opts))
})
pt.TestAddrBook(t, addressBookFactory(t, dsFactory, opts))
})

t.Run("Cacheless", func(t *testing.T) {
t.Parallel()
t.Run(name+" Cacheless", func(t *testing.T) {
t.Parallel()

opts := DefaultOpts()
opts.TTLInterval = 100 * time.Microsecond
opts.CacheSize = 0
opts := DefaultOpts()
opts.TTLInterval = 100 * time.Microsecond
opts.CacheSize = 0

pt.TestAddrBook(t, addressBookFactory(t, dsFactory, opts))
})
pt.TestAddrBook(t, addressBookFactory(t, dsFactory, opts))
})
}
}
Expand All @@ -65,6 +63,14 @@ func TestDsKeyBook(t *testing.T) {
}
}

func BenchmarkDsKeyBook(b *testing.B) {
for name, dsFactory := range dstores {
b.Run(name, func(b *testing.B) {
pt.BenchmarkKeyBook(b, keyBookFactory(b, dsFactory, DefaultOpts()))
})
}
}

func BenchmarkDsPeerstore(b *testing.B) {
caching := DefaultOpts()
caching.CacheSize = 1024
Expand All @@ -75,6 +81,8 @@ func BenchmarkDsPeerstore(b *testing.B) {
for name, dsFactory := range dstores {
b.Run(name, func(b *testing.B) {
pt.BenchmarkPeerstore(b, peerstoreFactory(b, dsFactory, caching), "Caching")
})
b.Run(name, func(b *testing.B) {
pt.BenchmarkPeerstore(b, peerstoreFactory(b, dsFactory, cacheless), "Cacheless")
})
}
Expand Down
6 changes: 6 additions & 0 deletions pstoremem/inmem_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,3 +30,9 @@ func BenchmarkInMemoryPeerstore(b *testing.B) {
return NewPeerstore(), nil
}, "InMem")
}

func BenchmarkInMemoryKeyBook(b *testing.B) {
pt.BenchmarkKeyBook(b, func() (pstore.KeyBook, func()) {
return NewKeyBook(), nil
})
}
11 changes: 10 additions & 1 deletion test/benchmarks_suite.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package test
import (
"context"
"fmt"
"sort"
"testing"

pstore "github.com/libp2p/go-libp2p-peerstore"
Expand Down Expand Up @@ -38,7 +39,15 @@ func BenchmarkPeerstore(b *testing.B, factory PeerstoreFactory, variant string)
go addressProducer(ctx, b, p.ch, p.n)
}

for name, bench := range peerstoreBenchmarks {
// So tests are always run in the same order.
ordernames := make([]string, 0, len(peerstoreBenchmarks))
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Makes sense, although we can also change the type of peerstoreBenchmarks to be an array of struct { name string, testFunc func(...) } so the order is deterministic (I think I had it like this early on!).

Then could do away with the sorting here and in the keybook benchmarks, and apply non-lexicographical orders manually.

Not worth changing, but just wanted to point out an alternative.

for name := range peerstoreBenchmarks {
ordernames = append(ordernames, name)
}
sort.Strings(ordernames)

for _, name := range ordernames {
bench := peerstoreBenchmarks[name]
for _, p := range params {
// Create a new peerstore.
ps, closeFunc := factory()
Expand Down
141 changes: 141 additions & 0 deletions test/keybook_suite.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,3 +161,144 @@ func testInlinedPubKeyAddedOnRetrieve(kb pstore.KeyBook) func(t *testing.T) {
}
}
}

var keybookBenchmarkSuite = map[string]func(kb pstore.KeyBook) func(*testing.B){
"PubKey": benchmarkPubKey,
"AddPubKey": benchmarkAddPubKey,
"PrivKey": benchmarkPrivKey,
"AddPrivKey": benchmarkAddPrivKey,
"PeersWithKeys": benchmarkPeersWithKeys,
}

func BenchmarkKeyBook(b *testing.B, factory KeyBookFactory) {
ordernames := make([]string, 0, len(keybookBenchmarkSuite))
for name := range keybookBenchmarkSuite {
ordernames = append(ordernames, name)
}
sort.Strings(ordernames)
for _, name := range ordernames {
bench := keybookBenchmarkSuite[name]
kb, closeFunc := factory()

b.Run(name, bench(kb))

if closeFunc != nil {
closeFunc()
}
}
}

func benchmarkPubKey(kb pstore.KeyBook) func(*testing.B) {
return func(b *testing.B) {
_, pub, err := pt.RandTestKeyPair(512)
if err != nil {
b.Error(err)
}

id, err := peer.IDFromPublicKey(pub)
if err != nil {
b.Error(err)
}

err = kb.AddPubKey(id, pub)
if err != nil {
b.Fatal(err)
}

b.ResetTimer()
for i := 0; i < b.N; i++ {
kb.PubKey(id)
}
}
}

func benchmarkAddPubKey(kb pstore.KeyBook) func(*testing.B) {
return func(b *testing.B) {
_, pub, err := pt.RandTestKeyPair(512)
if err != nil {
b.Error(err)
}

id, err := peer.IDFromPublicKey(pub)
if err != nil {
b.Error(err)
}

b.ResetTimer()
for i := 0; i < b.N; i++ {
kb.AddPubKey(id, pub)
}
}
}

func benchmarkPrivKey(kb pstore.KeyBook) func(*testing.B) {
return func(b *testing.B) {
priv, _, err := pt.RandTestKeyPair(512)
if err != nil {
b.Error(err)
}

id, err := peer.IDFromPrivateKey(priv)
if err != nil {
b.Error(err)
}

err = kb.AddPrivKey(id, priv)
if err != nil {
b.Fatal(err)
}

b.ResetTimer()
for i := 0; i < b.N; i++ {
kb.PrivKey(id)
}
}
}

func benchmarkAddPrivKey(kb pstore.KeyBook) func(*testing.B) {
return func(b *testing.B) {
priv, _, err := pt.RandTestKeyPair(512)
if err != nil {
b.Error(err)
}

id, err := peer.IDFromPrivateKey(priv)
if err != nil {
b.Error(err)
}

b.ResetTimer()
for i := 0; i < b.N; i++ {
kb.AddPrivKey(id, priv)
}
}
}

func benchmarkPeersWithKeys(kb pstore.KeyBook) func(*testing.B) {
return func(b *testing.B) {
for i := 0; i < 10; i++ {
priv, pub, err := pt.RandTestKeyPair(512)
if err != nil {
b.Error(err)
}

id, err := peer.IDFromPublicKey(pub)
if err != nil {
b.Error(err)
}

err = kb.AddPubKey(id, pub)
if err != nil {
b.Fatal(err)
}
err = kb.AddPrivKey(id, priv)
if err != nil {
b.Fatal(err)
}
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
kb.PeersWithKeys()
}
}
}