Skip to content

Commit

Permalink
test: make TestHashRing() more reliable (#1467)
Browse files Browse the repository at this point in the history
This might make the test more reliable. fixes
#1446
We may have been hitting a deadlock situation where cronjobs were
setting up their observation of channels and calling `clock.Now()` while
the test tried to increment the mock clock (exact case is detailed more
in #1368).
  • Loading branch information
matt2e authored May 13, 2024
1 parent d9f26b9 commit 8b424bd
Showing 1 changed file with 4 additions and 10 deletions.
14 changes: 4 additions & 10 deletions backend/controller/cronjobs/cronjobs_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,10 +52,6 @@ func TestHashRing(t *testing.T) {
lock: sync.Mutex{},
attemptCountMap: map[string]int{},
}

verbCallCount := map[string]int{}
verbCallCountLock := sync.Mutex{}

moduleName := "initial"
jobsToCreate := newJobs(t, moduleName, "*/10 * * * * * *", mockDal.clock, 100)

Expand All @@ -68,15 +64,13 @@ func TestHashRing(t *testing.T) {
assert.NoError(t, err)

controllers := newControllers(ctx, 20, mockDal, func() clock.Clock { return clock.NewMock() }, func(ctx context.Context, r *connect.Request[ftlv1.CallRequest], o optional.Option[model.RequestKey], s string) (*connect.Response[ftlv1.CallResponse], error) {
verbRef := schema.RefFromProto(r.Msg.Verb)

verbCallCountLock.Lock()
verbCallCount[verbRef.Name]++
verbCallCountLock.Unlock()

return &connect.Response[ftlv1.CallResponse]{}, nil
})

// This should give time for each controller to start watching its own mock clock
// If we don;t wait here, we might hit a race condition outlined in issue #1368
time.Sleep(time.Millisecond * 100)

// progress time for each controller one at a time, noting which verbs got attempted each time
// to build a map of verb to controller keys
controllersForVerbs := map[string][]model.ControllerKey{}
Expand Down

0 comments on commit 8b424bd

Please sign in to comment.