From 752586acff95f5e1aa138c72b9ab8a51bdfc4314 Mon Sep 17 00:00:00 2001 From: Sergei Drugalev Date: Tue, 7 Mar 2023 23:45:33 +0100 Subject: [PATCH 1/6] LazyLoad utils module from core --- pkg/utils/lazy.go | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 pkg/utils/lazy.go diff --git a/pkg/utils/lazy.go b/pkg/utils/lazy.go new file mode 100644 index 000000000..a317d5611 --- /dev/null +++ b/pkg/utils/lazy.go @@ -0,0 +1,40 @@ +package utils + +import ( + "sync" +) + +type LazyLoad[T any] struct { + f func() (T, error) + state T + lock sync.Mutex + once sync.Once +} + +func NewLazyLoad[T any](f func() (T, error)) *LazyLoad[T] { + return &LazyLoad[T]{ + f: f, + } +} + +func (l *LazyLoad[T]) Get() (out T, err error) { + l.lock.Lock() + defer l.lock.Unlock() + + // fetch only once (or whenever cleared) + l.once.Do(func() { + l.state, err = l.f() + }) + // if err, clear so next get will retry + if err != nil { + l.once = sync.Once{} + } + out = l.state + return +} + +func (l *LazyLoad[T]) Reset() { + l.lock.Lock() + defer l.lock.Unlock() + l.once = sync.Once{} +} From 1d975a1042ea3299890015f0212776474178c638 Mon Sep 17 00:00:00 2001 From: Sergei Drugalev Date: Tue, 7 Mar 2023 23:51:20 +0100 Subject: [PATCH 2/6] BatchSplit utility func from core --- pkg/utils/collection.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 pkg/utils/collection.go diff --git a/pkg/utils/collection.go b/pkg/utils/collection.go new file mode 100644 index 000000000..0de24a2ed --- /dev/null +++ b/pkg/utils/collection.go @@ -0,0 +1,21 @@ +package utils + +import ( + "fmt" +) + +// BatchSplit splits an slices into an slices of slicess with a maximum length +func BatchSplit[T any](list []T, max int) (out [][]T, err error) { + if max == 0 { + return out, fmt.Errorf("max batch length cannot be 0") + } + + // batch list into no more than max each + for len(list) > max { + // assign to list: remaining after taking slice from beginning + // append to out: max length slice from beginning of list + list, out = list[max:], append(out, list[:max]) + } + out = append(out, list) // append remaining to list (slice len < max) + return out, nil +} \ No newline at end of file From dfcfec2069a5962947bcc4fd87de26ed510ebac5 Mon Sep 17 00:00:00 2001 From: Sergei Drugalev Date: Wed, 8 Mar 2023 16:14:08 +0100 Subject: [PATCH 3/6] KeyNotFound error from core --- pkg/config/error.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pkg/config/error.go b/pkg/config/error.go index 008cc1e03..a504654f2 100644 --- a/pkg/config/error.go +++ b/pkg/config/error.go @@ -31,3 +31,12 @@ type ErrEmpty struct { func (e ErrEmpty) Error() string { return fmt.Sprintf("%s: empty: %s", e.Name, e.Msg) } + +type KeyNotFoundError struct { + ID string + KeyType string +} + +func (e KeyNotFoundError) Error() string { + return fmt.Sprintf("unable to find %s key with id %s", e.KeyType, e.ID) +} From 3e696293b6893860cd097a6d996b3dad21854e66 Mon Sep 17 00:00:00 2001 From: Sergei Drugalev Date: Wed, 8 Mar 2023 16:45:39 +0100 Subject: [PATCH 4/6] Context with timeout test utility from core --- pkg/utils/testutils.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 pkg/utils/testutils.go diff --git a/pkg/utils/testutils.go b/pkg/utils/testutils.go new file mode 100644 index 000000000..1d74064c2 --- /dev/null +++ b/pkg/utils/testutils.go @@ -0,0 +1,20 @@ +package utils + +import ( + "context" + "testing" +) + +func Context(t *testing.T) context.Context { + ctx := context.Background() + var cancel func() + + if d, ok := t.Deadline(); ok { + ctx, cancel = context.WithDeadline(ctx, d) + } else { + ctx, cancel = context.WithCancel(ctx) + } + + t.Cleanup(cancel) + return ctx +} \ No newline at end of file From 6390cb50358e1e21a7cc0ca4796f5f9cfc7c87ab Mon Sep 17 00:00:00 2001 From: Sergei Drugalev Date: Wed, 8 Mar 2023 17:25:23 +0100 Subject: [PATCH 5/6] Formatting --- pkg/monitoring/manager_benchmark_test.go | 23 +++++++++++++++-------- pkg/utils/collection.go | 2 +- pkg/utils/testutils.go | 2 +- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/pkg/monitoring/manager_benchmark_test.go b/pkg/monitoring/manager_benchmark_test.go index 71fd5a61e..0a89a8fda 100644 --- a/pkg/monitoring/manager_benchmark_test.go +++ b/pkg/monitoring/manager_benchmark_test.go @@ -13,18 +13,25 @@ import ( // that the chain readers respond immediately with random data and the rdd poller // will generate a new set of 5 random feeds every second. -//goos: darwin -//goarch: amd64 -//pkg: github.com/smartcontractkit/chainlink-relay/pkg/monitoring -//cpu: Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz +// goos: darwin +// goarch: amd64 +// pkg: github.com/smartcontractkit/chainlink-relay/pkg/monitoring +// cpu: Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz // (10 jan 2022) -// 5719 184623 ns/op 91745 B/op 1482 allocs/op +// +// 5719 184623 ns/op 91745 B/op 1482 allocs/op +// // (17 jan 2022) -// 6679 180862 ns/op 92230 B/op 1493 allocs/op +// +// 6679 180862 ns/op 92230 B/op 1493 allocs/op +// // (18 jan 2022) -// 16504 71478 ns/op 77515 B/op 963 allocs/op +// +// 16504 71478 ns/op 77515 B/op 963 allocs/op +// // (3 feb 2022 -// 59468 23180 ns/op 5921 B/op 61 allocs/op +// +// 59468 23180 ns/op 5921 B/op 61 allocs/op func BenchmarkManager(b *testing.B) { var subs utils.Subprocesses defer subs.Wait() diff --git a/pkg/utils/collection.go b/pkg/utils/collection.go index 0de24a2ed..3ab7a806a 100644 --- a/pkg/utils/collection.go +++ b/pkg/utils/collection.go @@ -18,4 +18,4 @@ func BatchSplit[T any](list []T, max int) (out [][]T, err error) { } out = append(out, list) // append remaining to list (slice len < max) return out, nil -} \ No newline at end of file +} diff --git a/pkg/utils/testutils.go b/pkg/utils/testutils.go index 1d74064c2..5c129b18a 100644 --- a/pkg/utils/testutils.go +++ b/pkg/utils/testutils.go @@ -17,4 +17,4 @@ func Context(t *testing.T) context.Context { t.Cleanup(cancel) return ctx -} \ No newline at end of file +} From d13c50d40f3ed5471bedea2468214f9b0e9996b6 Mon Sep 17 00:00:00 2001 From: Sergei Drugalev Date: Thu, 9 Mar 2023 01:33:47 +0100 Subject: [PATCH 6/6] Tests for the utility modules moved from core --- pkg/utils/collection_test.go | 60 ++++++++++++++++++++++++++++++ pkg/utils/lazy_test.go | 71 ++++++++++++++++++++++++++++++++++++ 2 files changed, 131 insertions(+) create mode 100644 pkg/utils/collection_test.go create mode 100644 pkg/utils/lazy_test.go diff --git a/pkg/utils/collection_test.go b/pkg/utils/collection_test.go new file mode 100644 index 000000000..20bfb2c62 --- /dev/null +++ b/pkg/utils/collection_test.go @@ -0,0 +1,60 @@ +package utils + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBatchSplit(t *testing.T) { + list := []int{} + for i := 0; i < 100; i++ { + list = append(list, i) + } + + runs := []struct { + name string + input []int + max int // max per batch + num int // expected number of batches + lastLen int // expected number in last batch + expectErr bool + }{ + {"max=1", list, 1, len(list), 1, false}, + {"max=25", list, 25, 4, 25, false}, + {"max=33", list, 33, 4, 1, false}, + {"max=87", list, 87, 2, 13, false}, + {"max=len", list, len(list), 1, 100, false}, + {"max=len+1", list, len(list) + 1, 1, len(list), false}, // max exceeds len of list + {"zero-list", []int{}, 1, 1, 0, false}, // zero length list + {"zero-max", list, 0, 0, 0, true}, // zero as max input + } + + for _, r := range runs { + t.Run(r.name, func(t *testing.T) { + batch, err := BatchSplit(r.input, r.max) + if r.expectErr { + assert.Error(t, err) + return + } + + assert.NoError(t, err) + assert.Equal(t, r.num, len(batch)) // check number of batches + + temp := []int{} + for i := 0; i < len(batch); i++ { + expectedLen := r.max + if i == len(batch)-1 { + expectedLen = r.lastLen // expect last batch to be less than max + } + assert.Equal(t, expectedLen, len(batch[i])) // check length of batch + + temp = append(temp, batch[i]...) + } + // assert order has not changed when list is reconstructed + assert.Equal(t, r.input, temp) + + }) + } + +} diff --git a/pkg/utils/lazy_test.go b/pkg/utils/lazy_test.go new file mode 100644 index 000000000..88d02eea7 --- /dev/null +++ b/pkg/utils/lazy_test.go @@ -0,0 +1,71 @@ +package utils + +import ( + "sync" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestLazyLoad(t *testing.T) { + var clientwg sync.WaitGroup + + tc := func() (int, error) { + clientwg.Done() + return 10, nil + } + + // Get should only request a client once, use cached afterward + t.Run("get", func(t *testing.T) { + clientwg.Add(1) // expect one call to get client + c := NewLazyLoad(tc) + rw, err := c.Get() + assert.NoError(t, err) + assert.NotNil(t, rw) + assert.NotNil(t, c.state) + + // used cached client + rw, err = c.Get() + assert.NoError(t, err) + assert.NotNil(t, rw) + clientwg.Wait() + }) + + // Clear removes the cached client, should refetch + t.Run("clear", func(t *testing.T) { + clientwg.Add(2) // expect two calls to get client + + c := NewLazyLoad(tc) + rw, err := c.Get() + assert.NotNil(t, rw) + assert.NoError(t, err) + + c.Reset() + + rw, err = c.Get() + assert.NotNil(t, rw) + assert.NoError(t, err) + clientwg.Wait() + }) + + // Race checks a race condition of Getting and Clearing a new client + t.Run("race", func(t *testing.T) { + clientwg.Add(1) // expect one call to get client + + c := NewLazyLoad(tc) + var wg sync.WaitGroup + wg.Add(2) + go func() { + rw, err := c.Get() + assert.NoError(t, err) + assert.NotNil(t, rw) + wg.Done() + }() + go func() { + c.Reset() + wg.Done() + }() + wg.Wait() + clientwg.Wait() + }) +}