Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix flaky OTLP exporter reconnect test #1814

Merged
merged 4 commits into from
Apr 15, 2021
Merged

Fix flaky OTLP exporter reconnect test #1814

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f71d316
Fix flaky OTLP exporter reconnect test
MrAlias Apr 14, 2021
2d5e814
Add changes to changelog
MrAlias Apr 14, 2021
c027ea9
Use time.After instead of Timer
MrAlias Apr 14, 2021
7cb51a1
Remove changelog entry
MrAlias Apr 14, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 36 additions & 5 deletions exporters/otlp/otlpgrpc/otlp_integration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"context"
"fmt"
"net"
"runtime"
"strings"
"testing"
"time"
Expand Down Expand Up @@ -144,7 +145,7 @@ func TestNewExporter_invokeStartThenStopManyTimes(t *testing.T) {
func TestNewExporter_collectorConnectionDiesThenReconnectsWhenInRestMode(t *testing.T) {
mc := runMockCollector(t)

reconnectionPeriod := 2 * time.Second // 2 second + jitter rest time after reconnection
reconnectionPeriod := 20 * time.Millisecond
ctx := context.Background()
exp := newGRPCExporter(t, ctx, mc.endpoint,
otlpgrpc.WithReconnectionPeriod(reconnectionPeriod))
Expand All @@ -165,8 +166,18 @@ func TestNewExporter_collectorConnectionDiesThenReconnectsWhenInRestMode(t *test
mc.endpoint,
)

// give it time for first reconnection
<-time.After(time.Millisecond * 20)
// Give the exporter sometime to reconnect
func() {
timer := time.After(reconnectionPeriod * 10)
for {
select {
case <-timer:
return
default:
runtime.Gosched()
}
}
}()

// second export, it will detect connection issue, change state of exporter to disconnected and
// send message to disconnected channel but this time reconnection gouroutine will be in (rest mode, not listening to the disconnected channel)
Expand All @@ -184,7 +195,17 @@ func TestNewExporter_collectorConnectionDiesThenReconnectsWhenInRestMode(t *test

// make sure reconnection loop hits beginning and goes back to waiting mode
// after hitting beginning of the loop it should reconnect
<-time.After(time.Second * 4)
func() {
timer := time.After(reconnectionPeriod * 10)
for {
select {
case <-timer:
return
default:
runtime.Gosched()
}
}
}()

n := 10
for i := 0; i < n; i++ {
Expand Down Expand Up @@ -240,7 +261,17 @@ func TestNewExporter_collectorConnectionDiesThenReconnects(t *testing.T) {
nmc := runMockCollectorAtEndpoint(t, mc.endpoint)

// Give the exporter sometime to reconnect
<-time.After(reconnectionPeriod * 4)
func() {
timer := time.After(reconnectionPeriod * 10)
for {
select {
case <-timer:
return
default:
runtime.Gosched()
}
}
}()

n := 10
for i := 0; i < n; i++ {
Expand Down